@livestore/sqlite-wasm 0.4.0 → 0.5.0-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/FacadeVFS.d.ts +0 -3
- package/dist/FacadeVFS.d.ts.map +1 -1
- package/dist/FacadeVFS.js +0 -3
- package/dist/FacadeVFS.js.map +1 -1
- package/dist/browser/mod.d.ts.map +1 -1
- package/dist/browser/mod.js +3 -3
- package/dist/browser/mod.js.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts +4 -4
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.js +79 -66
- package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
- package/dist/browser/opfs/index.d.ts.map +1 -1
- package/dist/browser/opfs/index.js +2 -2
- package/dist/browser/opfs/index.js.map +1 -1
- package/dist/cf/CloudflareDurableObjectVFS.d.ts.map +1 -1
- package/dist/cf/CloudflareDurableObjectVFS.js +3 -1
- package/dist/cf/CloudflareDurableObjectVFS.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -1
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +1 -1
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -1
- package/dist/make-sqlite-db.d.ts.map +1 -1
- package/dist/make-sqlite-db.js +1 -1
- package/dist/make-sqlite-db.js.map +1 -1
- package/dist/node/NodeFS.js +1 -1
- package/dist/node/NodeFS.js.map +1 -1
- package/dist/node/mod.js +1 -1
- package/dist/node/mod.js.map +1 -1
- package/package.json +13 -22
- package/src/FacadeVFS.ts +0 -4
- package/src/browser/mod.ts +3 -3
- package/src/browser/opfs/AccessHandlePoolVFS.ts +91 -77
- package/src/browser/opfs/index.ts +2 -2
- package/src/cf/CloudflareDurableObjectVFS.ts +3 -4
- package/src/cf/test/README.md +8 -11
- package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +2 -1
- package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +2 -1
- package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +2 -1
- package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +2 -1
- package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +6 -3
- package/src/make-sqlite-db.ts +16 -21
- package/src/node/NodeFS.ts +1 -1
- package/src/node/mod.ts +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
/// <reference lib="webworker" />
|
|
3
3
|
|
|
4
4
|
import { shouldNeverHappen } from '@livestore/utils'
|
|
5
|
-
import {
|
|
5
|
+
import { type Context, Effect, Schedule, type Scope, Stream } from '@livestore/utils/effect'
|
|
6
6
|
import { Opfs, type WebError } from '@livestore/utils/effect/browser'
|
|
7
7
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
8
8
|
|
|
@@ -67,8 +67,8 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
67
67
|
#directoryPath
|
|
68
68
|
#directoryHandle: FileSystemDirectoryHandle | undefined
|
|
69
69
|
|
|
70
|
-
//
|
|
71
|
-
readonly #
|
|
70
|
+
// Context for executing Effect operations.
|
|
71
|
+
readonly #services: Context.Context<Opfs.Opfs | Scope.Scope>
|
|
72
72
|
|
|
73
73
|
// The OPFS files all have randomly-generated names that do not match
|
|
74
74
|
// the SQLite files whose data they contain. This map links those names
|
|
@@ -84,8 +84,8 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
84
84
|
#mapIdToFile = new Map<number, { path: string; flags: number; accessHandle: FileSystemSyncAccessHandle }>()
|
|
85
85
|
|
|
86
86
|
static create = Effect.fn(function* (name: string, directoryPath: string, module: any) {
|
|
87
|
-
const
|
|
88
|
-
const vfs = new AccessHandlePoolVFS({ name, directoryPath, module,
|
|
87
|
+
const services = yield* Effect.context<Opfs.Opfs | Scope.Scope>()
|
|
88
|
+
const vfs = new AccessHandlePoolVFS({ name, directoryPath, module, services })
|
|
89
89
|
yield* Effect.promise(() => vfs.isReady())
|
|
90
90
|
return vfs
|
|
91
91
|
})
|
|
@@ -94,16 +94,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
94
94
|
name,
|
|
95
95
|
directoryPath,
|
|
96
96
|
module,
|
|
97
|
-
|
|
97
|
+
services,
|
|
98
98
|
}: {
|
|
99
99
|
name: string
|
|
100
100
|
directoryPath: string
|
|
101
101
|
module: any
|
|
102
|
-
|
|
102
|
+
services: Context.Context<Opfs.Opfs | Scope.Scope>
|
|
103
103
|
}) {
|
|
104
104
|
super(name, module)
|
|
105
105
|
this.#directoryPath = directoryPath
|
|
106
|
-
this.#
|
|
106
|
+
this.#services = services
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
/**
|
|
@@ -116,7 +116,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
116
116
|
* string that is not meaningful to the application.
|
|
117
117
|
*/
|
|
118
118
|
getOpfsFileName = Effect.fn((zName: string) =>
|
|
119
|
-
Effect.gen(this, function* () {
|
|
119
|
+
Effect.gen({ self: this }, function* () {
|
|
120
120
|
const path = this.#getPath(zName)
|
|
121
121
|
const accessHandle = this.#mapPathToAccessHandle.get(path)!
|
|
122
122
|
return this.#mapAccessHandleToName.get(accessHandle)!
|
|
@@ -133,7 +133,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
133
133
|
* the file cannot be modified by other threads.
|
|
134
134
|
*/
|
|
135
135
|
readFilePayload = Effect.fn((zName: string) =>
|
|
136
|
-
Effect.gen(this, function* () {
|
|
136
|
+
Effect.gen({ self: this }, function* () {
|
|
137
137
|
const path = this.#getPath(zName)
|
|
138
138
|
const accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
139
139
|
|
|
@@ -141,7 +141,8 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
141
141
|
return shouldNeverHappen('Cannot read payload for untracked OPFS path')
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
-
const
|
|
144
|
+
const opfs = yield* Opfs.Opfs
|
|
145
|
+
const fileSize = yield* opfs.syncGetSize(accessHandle)
|
|
145
146
|
if (fileSize <= HEADER_OFFSET_DATA) {
|
|
146
147
|
return shouldNeverHappen(
|
|
147
148
|
`OPFS file too small to contain header and payload: size ${fileSize} < HEADER_OFFSET_DATA ${HEADER_OFFSET_DATA}`,
|
|
@@ -150,7 +151,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
150
151
|
|
|
151
152
|
const payloadSize = fileSize - HEADER_OFFSET_DATA
|
|
152
153
|
const payload = new Uint8Array(payloadSize)
|
|
153
|
-
const bytesRead = yield*
|
|
154
|
+
const bytesRead = yield* opfs.syncRead(accessHandle, payload, { at: HEADER_OFFSET_DATA })
|
|
154
155
|
if (bytesRead !== payloadSize) {
|
|
155
156
|
return shouldNeverHappen(`Failed to read full payload from OPFS file: read ${bytesRead}/${payloadSize}`)
|
|
156
157
|
}
|
|
@@ -160,17 +161,18 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
160
161
|
)
|
|
161
162
|
|
|
162
163
|
resetAccessHandle = Effect.fn((zName: string) =>
|
|
163
|
-
Effect.gen(this, function* () {
|
|
164
|
+
Effect.gen({ self: this }, function* () {
|
|
164
165
|
const path = this.#getPath(zName)
|
|
165
166
|
const accessHandle = this.#mapPathToAccessHandle.get(path)!
|
|
166
|
-
yield* Opfs.Opfs
|
|
167
|
+
const opfs = yield* Opfs.Opfs
|
|
168
|
+
yield* opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
167
169
|
// accessHandle.write(new Uint8Array(), { at: HEADER_OFFSET_DATA })
|
|
168
170
|
// accessHandle.flush()
|
|
169
171
|
}),
|
|
170
172
|
)
|
|
171
173
|
|
|
172
174
|
override jOpen(zName: string, fileId: number, flags: number, pOutFlags: DataView): number {
|
|
173
|
-
return Effect.gen(this, function* () {
|
|
175
|
+
return Effect.gen({ self: this }, function* () {
|
|
174
176
|
// First try to open a path that already exists in the file system.
|
|
175
177
|
const name = zName as unknown
|
|
176
178
|
const path = typeof name === 'string' && name !== '' ? this.#getPath(name) : Math.random().toString(36)
|
|
@@ -184,10 +186,10 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
184
186
|
} else {
|
|
185
187
|
// Out of unassociated files. This can be fixed by calling
|
|
186
188
|
// addCapacity() from the application.
|
|
187
|
-
return yield* Effect.
|
|
189
|
+
return yield* Effect.die(new Error('cannot create file'))
|
|
188
190
|
}
|
|
189
191
|
}
|
|
190
|
-
if (accessHandle == null) return yield* Effect.
|
|
192
|
+
if (accessHandle == null) return yield* Effect.die(new Error('file not found'))
|
|
191
193
|
|
|
192
194
|
// Subsequent methods are only passed the fileId, so make sure we have
|
|
193
195
|
// a way to get the file resources.
|
|
@@ -198,16 +200,17 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
198
200
|
return VFS.SQLITE_OK
|
|
199
201
|
}).pipe(
|
|
200
202
|
Effect.tapCauseLogPretty,
|
|
201
|
-
Effect.
|
|
202
|
-
|
|
203
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_CANTOPEN)),
|
|
204
|
+
Effect.runSyncWith(this.#services),
|
|
203
205
|
)
|
|
204
206
|
}
|
|
205
207
|
|
|
206
208
|
override jClose(fileId: number): number {
|
|
207
|
-
return Effect.gen(this, function* () {
|
|
209
|
+
return Effect.gen({ self: this }, function* () {
|
|
208
210
|
const file = this.#mapIdToFile.get(fileId)
|
|
209
211
|
if (file !== undefined) {
|
|
210
|
-
yield* Opfs.Opfs
|
|
212
|
+
const opfs = yield* Opfs.Opfs
|
|
213
|
+
yield* opfs.syncFlush(file.accessHandle)
|
|
211
214
|
this.#mapIdToFile.delete(fileId)
|
|
212
215
|
if ((file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) !== 0) {
|
|
213
216
|
yield* this.#deletePath(file.path)
|
|
@@ -216,15 +219,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
216
219
|
return VFS.SQLITE_OK
|
|
217
220
|
}).pipe(
|
|
218
221
|
Effect.tapCauseLogPretty,
|
|
219
|
-
Effect.
|
|
220
|
-
|
|
222
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_CLOSE)),
|
|
223
|
+
Effect.runSyncWith(this.#services),
|
|
221
224
|
)
|
|
222
225
|
}
|
|
223
226
|
|
|
224
227
|
override jRead(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
|
|
225
|
-
return Effect.gen(this, function* () {
|
|
228
|
+
return Effect.gen({ self: this }, function* () {
|
|
226
229
|
const file = this.#mapIdToFile.get(fileId)!
|
|
227
|
-
const
|
|
230
|
+
const opfs = yield* Opfs.Opfs
|
|
231
|
+
const nBytes = yield* opfs.syncRead(file.accessHandle, pData.subarray(), {
|
|
228
232
|
at: HEADER_OFFSET_DATA + iOffset,
|
|
229
233
|
})
|
|
230
234
|
if (nBytes < pData.byteLength) {
|
|
@@ -234,63 +238,67 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
234
238
|
return VFS.SQLITE_OK
|
|
235
239
|
}).pipe(
|
|
236
240
|
Effect.tapCauseLogPretty,
|
|
237
|
-
Effect.
|
|
238
|
-
|
|
241
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_READ)),
|
|
242
|
+
Effect.runSyncWith(this.#services),
|
|
239
243
|
)
|
|
240
244
|
}
|
|
241
245
|
|
|
242
246
|
override jWrite(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
|
|
243
|
-
return Effect.gen(this, function* () {
|
|
247
|
+
return Effect.gen({ self: this }, function* () {
|
|
244
248
|
const file = this.#mapIdToFile.get(fileId)!
|
|
245
|
-
const
|
|
249
|
+
const opfs = yield* Opfs.Opfs
|
|
250
|
+
const nBytes = yield* opfs.syncWrite(file.accessHandle, pData.subarray(), {
|
|
246
251
|
at: HEADER_OFFSET_DATA + iOffset,
|
|
247
252
|
})
|
|
248
253
|
if (nBytes !== pData.byteLength) {
|
|
249
|
-
return yield* Effect.
|
|
254
|
+
return yield* Effect.die(new Error(`Wrote ${nBytes} bytes, expected ${pData.byteLength}`))
|
|
250
255
|
}
|
|
251
256
|
return VFS.SQLITE_OK
|
|
252
257
|
}).pipe(
|
|
253
258
|
Effect.tapCauseLogPretty,
|
|
254
|
-
Effect.
|
|
255
|
-
|
|
259
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_WRITE)),
|
|
260
|
+
Effect.runSyncWith(this.#services),
|
|
256
261
|
)
|
|
257
262
|
}
|
|
258
263
|
|
|
259
264
|
override jTruncate(fileId: number, iSize: number): number {
|
|
260
|
-
return Effect.gen(this, function* () {
|
|
265
|
+
return Effect.gen({ self: this }, function* () {
|
|
261
266
|
const file = this.#mapIdToFile.get(fileId)!
|
|
262
|
-
yield* Opfs.Opfs
|
|
267
|
+
const opfs = yield* Opfs.Opfs
|
|
268
|
+
yield* opfs.syncTruncate(file.accessHandle, HEADER_OFFSET_DATA + iSize)
|
|
263
269
|
return VFS.SQLITE_OK
|
|
264
270
|
}).pipe(
|
|
265
271
|
Effect.tapCauseLogPretty,
|
|
266
|
-
Effect.
|
|
267
|
-
|
|
272
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_TRUNCATE)),
|
|
273
|
+
Effect.runSyncWith(this.#services),
|
|
268
274
|
)
|
|
269
275
|
}
|
|
270
276
|
|
|
271
277
|
override jSync(fileId: number, _flags: number): number {
|
|
272
|
-
return Effect.gen(this, function* () {
|
|
278
|
+
return Effect.gen({ self: this }, function* () {
|
|
273
279
|
const file = this.#mapIdToFile.get(fileId)!
|
|
274
|
-
yield* Opfs.Opfs
|
|
280
|
+
const opfs = yield* Opfs.Opfs
|
|
281
|
+
yield* opfs.syncFlush(file.accessHandle)
|
|
275
282
|
return VFS.SQLITE_OK
|
|
276
283
|
}).pipe(
|
|
277
284
|
Effect.tapCauseLogPretty,
|
|
278
|
-
Effect.
|
|
279
|
-
|
|
285
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSYNC)),
|
|
286
|
+
Effect.runSyncWith(this.#services),
|
|
280
287
|
)
|
|
281
288
|
}
|
|
282
289
|
|
|
283
290
|
override jFileSize(fileId: number, pSize64: DataView): number {
|
|
284
|
-
return Effect.gen(this, function* () {
|
|
291
|
+
return Effect.gen({ self: this }, function* () {
|
|
285
292
|
const file = this.#mapIdToFile.get(fileId)!
|
|
286
|
-
const
|
|
293
|
+
const opfs = yield* Opfs.Opfs
|
|
294
|
+
const opfsFileSize = yield* opfs.syncGetSize(file.accessHandle)
|
|
287
295
|
const size = opfsFileSize - HEADER_OFFSET_DATA
|
|
288
296
|
pSize64.setBigInt64(0, BigInt(size), true)
|
|
289
297
|
return VFS.SQLITE_OK
|
|
290
298
|
}).pipe(
|
|
291
299
|
Effect.tapCauseLogPretty,
|
|
292
|
-
Effect.
|
|
293
|
-
|
|
300
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSTAT)),
|
|
301
|
+
Effect.runSyncWith(this.#services),
|
|
294
302
|
)
|
|
295
303
|
}
|
|
296
304
|
|
|
@@ -303,35 +311,35 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
303
311
|
}
|
|
304
312
|
|
|
305
313
|
override jAccess(zName: string, _flags: number, pResOut: DataView): number {
|
|
306
|
-
return Effect.gen(this, function* () {
|
|
314
|
+
return Effect.gen({ self: this }, function* () {
|
|
307
315
|
const path = this.#getPath(zName)
|
|
308
316
|
pResOut.setInt32(0, this.#mapPathToAccessHandle.has(path) === true ? 1 : 0, true)
|
|
309
317
|
return VFS.SQLITE_OK
|
|
310
318
|
}).pipe(
|
|
311
319
|
Effect.tapCauseLogPretty,
|
|
312
|
-
Effect.
|
|
313
|
-
|
|
320
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_ACCESS)),
|
|
321
|
+
Effect.runSyncWith(this.#services),
|
|
314
322
|
)
|
|
315
323
|
}
|
|
316
324
|
|
|
317
325
|
override jDelete(zName: string, _syncDir: number): number {
|
|
318
|
-
return Effect.gen(this, function* () {
|
|
326
|
+
return Effect.gen({ self: this }, function* () {
|
|
319
327
|
const path = this.#getPath(zName)
|
|
320
328
|
yield* this.#deletePath(path)
|
|
321
329
|
return VFS.SQLITE_OK
|
|
322
330
|
}).pipe(
|
|
323
331
|
Effect.tapCauseLogPretty,
|
|
324
|
-
Effect.
|
|
325
|
-
|
|
332
|
+
Effect.catchCause(() => Effect.succeed(VFS.SQLITE_IOERR_DELETE)),
|
|
333
|
+
Effect.runSyncWith(this.#services),
|
|
326
334
|
)
|
|
327
335
|
}
|
|
328
336
|
|
|
329
337
|
close() {
|
|
330
|
-
this.#releaseAccessHandles().pipe(
|
|
338
|
+
this.#releaseAccessHandles().pipe(Effect.runPromiseWith(this.#services))
|
|
331
339
|
}
|
|
332
340
|
|
|
333
341
|
async isReady() {
|
|
334
|
-
return Effect.gen(this, function* () {
|
|
342
|
+
return Effect.gen({ self: this }, function* () {
|
|
335
343
|
if (this.#directoryHandle == null) {
|
|
336
344
|
// All files are stored in a single directory.
|
|
337
345
|
this.#directoryHandle = yield* Opfs.getDirectoryHandleByPath(this.#directoryPath, { create: true })
|
|
@@ -342,7 +350,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
342
350
|
}
|
|
343
351
|
}
|
|
344
352
|
return true
|
|
345
|
-
}).pipe(
|
|
353
|
+
}).pipe(Effect.runPromiseWith(this.#services))
|
|
346
354
|
}
|
|
347
355
|
|
|
348
356
|
/**
|
|
@@ -373,11 +381,12 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
373
381
|
* Increase the capacity of the file system by n.
|
|
374
382
|
*/
|
|
375
383
|
addCapacity: (n: number) => Effect.Effect<void, WebError.WebError, Opfs.Opfs | Scope.Scope> = Effect.fn((n: number) =>
|
|
376
|
-
Effect.
|
|
377
|
-
Effect.gen(this, function* () {
|
|
384
|
+
Effect.replicateEffect(
|
|
385
|
+
Effect.gen({ self: this }, function* () {
|
|
378
386
|
const name = Math.random().toString(36).replace('0.', '')
|
|
379
|
-
const
|
|
380
|
-
|
|
387
|
+
const opfs = yield* Opfs.Opfs
|
|
388
|
+
const accessHandle = yield* opfs.getFileHandle(this.#directoryHandle!, name, { create: true }).pipe(
|
|
389
|
+
Effect.andThen((handle) => opfs.createSyncAccessHandle(handle)),
|
|
381
390
|
Effect.retry(Schedule.exponentialBackoff10Sec),
|
|
382
391
|
)
|
|
383
392
|
this.#mapAccessHandleToName.set(accessHandle, name)
|
|
@@ -385,6 +394,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
385
394
|
yield* this.#setAssociatedPath(accessHandle, '', 0)
|
|
386
395
|
}),
|
|
387
396
|
n,
|
|
397
|
+
{ discard: true },
|
|
388
398
|
),
|
|
389
399
|
)
|
|
390
400
|
|
|
@@ -394,17 +404,18 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
394
404
|
* file system.
|
|
395
405
|
*/
|
|
396
406
|
removeCapacity = Effect.fn((n: number) =>
|
|
397
|
-
Effect.gen(this, function* () {
|
|
407
|
+
Effect.gen({ self: this }, function* () {
|
|
398
408
|
let nRemoved = 0
|
|
399
409
|
yield* Effect.forEach(
|
|
400
410
|
this.#availableAccessHandles,
|
|
401
411
|
(accessHandle) =>
|
|
402
|
-
Effect.gen(this, function* () {
|
|
412
|
+
Effect.gen({ self: this }, function* () {
|
|
403
413
|
if (nRemoved === n || this.getSize() === this.getCapacity()) return nRemoved
|
|
404
414
|
|
|
405
415
|
const name = this.#mapAccessHandleToName.get(accessHandle)!
|
|
406
416
|
accessHandle.close()
|
|
407
|
-
yield* Opfs.Opfs
|
|
417
|
+
const opfs = yield* Opfs.Opfs
|
|
418
|
+
yield* opfs.removeEntry(this.#directoryHandle!, name)
|
|
408
419
|
this.#mapAccessHandleToName.delete(accessHandle)
|
|
409
420
|
this.#availableAccessHandles.delete(accessHandle)
|
|
410
421
|
++nRemoved
|
|
@@ -417,15 +428,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
417
428
|
)
|
|
418
429
|
|
|
419
430
|
#acquireAccessHandles = Effect.fn(() =>
|
|
420
|
-
Effect.gen(this, function* () {
|
|
421
|
-
const
|
|
431
|
+
Effect.gen({ self: this }, function* () {
|
|
432
|
+
const opfs = yield* Opfs.Opfs
|
|
433
|
+
const handlesStream = opfs.values(this.#directoryHandle!)
|
|
422
434
|
|
|
423
435
|
yield* handlesStream.pipe(
|
|
424
436
|
Stream.filter((handle): handle is FileSystemFileHandle => handle.kind === 'file'),
|
|
425
437
|
Stream.mapEffect(
|
|
426
438
|
(fileHandle) =>
|
|
427
|
-
Effect.gen(this, function* () {
|
|
428
|
-
const accessHandle = yield*
|
|
439
|
+
Effect.gen({ self: this }, function* () {
|
|
440
|
+
const accessHandle = yield* opfs.createSyncAccessHandle(fileHandle)
|
|
429
441
|
return {
|
|
430
442
|
accessHandle,
|
|
431
443
|
opfsFileName: fileHandle.name,
|
|
@@ -435,7 +447,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
435
447
|
{ concurrency: 'unbounded' },
|
|
436
448
|
),
|
|
437
449
|
Stream.runForEach(({ opfsFileName, accessHandle, path }) =>
|
|
438
|
-
Effect.gen(this, function* () {
|
|
450
|
+
Effect.gen({ self: this }, function* () {
|
|
439
451
|
this.#mapAccessHandleToName.set(accessHandle, opfsFileName)
|
|
440
452
|
if (path !== '') {
|
|
441
453
|
this.#mapPathToAccessHandle.set(path, accessHandle)
|
|
@@ -449,7 +461,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
449
461
|
)
|
|
450
462
|
|
|
451
463
|
#releaseAccessHandles = Effect.fn(() =>
|
|
452
|
-
Effect.gen(this, function* () {
|
|
464
|
+
Effect.gen({ self: this }, function* () {
|
|
453
465
|
yield* Effect.forEach(
|
|
454
466
|
this.#mapAccessHandleToName.keys(),
|
|
455
467
|
(accessHandle) => Effect.sync(() => accessHandle.close()),
|
|
@@ -467,10 +479,11 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
467
479
|
* @returns {string} path or empty string
|
|
468
480
|
*/
|
|
469
481
|
#getAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle) =>
|
|
470
|
-
Effect.gen(this, function* () {
|
|
482
|
+
Effect.gen({ self: this }, function* () {
|
|
471
483
|
// Read the path and digest of the path from the file.
|
|
472
484
|
const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
|
|
473
|
-
yield* Opfs.Opfs
|
|
485
|
+
const opfs = yield* Opfs.Opfs
|
|
486
|
+
yield* opfs.syncRead(accessHandle, corpus, { at: 0 })
|
|
474
487
|
|
|
475
488
|
// Delete files not expected to be present.
|
|
476
489
|
const dataView = new DataView(corpus.buffer, corpus.byteOffset)
|
|
@@ -482,7 +495,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
482
495
|
}
|
|
483
496
|
|
|
484
497
|
const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4)
|
|
485
|
-
yield*
|
|
498
|
+
yield* opfs.syncRead(accessHandle, fileDigest, { at: HEADER_OFFSET_DIGEST })
|
|
486
499
|
|
|
487
500
|
// Verify the digest.
|
|
488
501
|
const computedDigest = this.#computeDigest(corpus)
|
|
@@ -494,7 +507,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
494
507
|
// truncated in #setAssociatedPath after the header is written. If
|
|
495
508
|
// an interruption occurs right before the truncation then garbage
|
|
496
509
|
// may remain in the file.
|
|
497
|
-
yield*
|
|
510
|
+
yield* opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
498
511
|
}
|
|
499
512
|
return new TextDecoder().decode(corpus.subarray(0, pathBytes))
|
|
500
513
|
} else {
|
|
@@ -510,12 +523,12 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
510
523
|
* Set the path on an OPFS file header.
|
|
511
524
|
*/
|
|
512
525
|
#setAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle, path: string, flags: number) =>
|
|
513
|
-
Effect.gen(this, function* () {
|
|
526
|
+
Effect.gen({ self: this }, function* () {
|
|
514
527
|
// Convert the path string to UTF-8.
|
|
515
528
|
const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
|
|
516
529
|
const encodedResult = new TextEncoder().encodeInto(path, corpus)
|
|
517
530
|
if (encodedResult.written >= HEADER_MAX_PATH_SIZE) {
|
|
518
|
-
return yield* Effect.
|
|
531
|
+
return yield* Effect.die(new Error('path too long'))
|
|
519
532
|
}
|
|
520
533
|
|
|
521
534
|
// Add the creation flags.
|
|
@@ -524,9 +537,10 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
524
537
|
|
|
525
538
|
// Write the OPFS file header, including the digest.
|
|
526
539
|
const digest = this.#computeDigest(corpus)
|
|
527
|
-
yield* Opfs.Opfs
|
|
528
|
-
yield*
|
|
529
|
-
yield*
|
|
540
|
+
const opfs = yield* Opfs.Opfs
|
|
541
|
+
yield* opfs.syncWrite(accessHandle, corpus, { at: 0 })
|
|
542
|
+
yield* opfs.syncWrite(accessHandle, digest, { at: HEADER_OFFSET_DIGEST })
|
|
543
|
+
yield* opfs.syncFlush(accessHandle)
|
|
530
544
|
|
|
531
545
|
if (path !== '') {
|
|
532
546
|
this.#mapPathToAccessHandle.set(path, accessHandle)
|
|
@@ -534,7 +548,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
534
548
|
} else {
|
|
535
549
|
// This OPFS file doesn't represent any SQLite file so it doesn't
|
|
536
550
|
// need to keep any data.
|
|
537
|
-
yield*
|
|
551
|
+
yield* opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
538
552
|
this.#availableAccessHandles.add(accessHandle)
|
|
539
553
|
}
|
|
540
554
|
}),
|
|
@@ -578,7 +592,7 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
578
592
|
* @param {string} path
|
|
579
593
|
*/
|
|
580
594
|
#deletePath = Effect.fn((path: string) =>
|
|
581
|
-
Effect.gen(this, function* () {
|
|
595
|
+
Effect.gen({ self: this }, function* () {
|
|
582
596
|
const accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
583
597
|
if (accessHandle !== undefined) {
|
|
584
598
|
// Un-associate the SQLite path from the OPFS file.
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { Effect, type Scope } from '@livestore/utils/effect'
|
|
1
|
+
import { Effect, Semaphore, type Scope } from '@livestore/utils/effect'
|
|
2
2
|
import type { Opfs } from '@livestore/utils/effect/browser'
|
|
3
3
|
import type * as WaSqlite from '@livestore/wa-sqlite'
|
|
4
4
|
|
|
5
5
|
import { AccessHandlePoolVFS } from './AccessHandlePoolVFS.ts'
|
|
6
6
|
|
|
7
|
-
const semaphore =
|
|
7
|
+
const semaphore = Semaphore.make(1).pipe(Effect.runSync)
|
|
8
8
|
const opfsVfsMap = new Map<string, AccessHandlePoolVFS>()
|
|
9
9
|
|
|
10
10
|
export const makeOpfsDb = ({
|
|
@@ -223,10 +223,9 @@ export class CloudflareDurableObjectVFS extends FacadeVFS {
|
|
|
223
223
|
override jFileSize(fileId: number, pSize64: DataView): number {
|
|
224
224
|
try {
|
|
225
225
|
const { path } = this.#getOpenFile(fileId)
|
|
226
|
-
const row = this.#sql
|
|
227
|
-
'SELECT MAX(page_no) AS max_page FROM vfs_pages WHERE file_path = ?',
|
|
228
|
-
|
|
229
|
-
).one()
|
|
226
|
+
const row = this.#sql
|
|
227
|
+
.exec<{ max_page: number | null }>('SELECT MAX(page_no) AS max_page FROM vfs_pages WHERE file_path = ?', path)
|
|
228
|
+
.one()
|
|
230
229
|
const fileSize = row.max_page === null ? 0 : (row.max_page + 1) * PAGE_SIZE
|
|
231
230
|
pSize64.setBigInt64(0, BigInt(fileSize), true)
|
|
232
231
|
return VFS.SQLITE_OK
|
package/src/cf/test/README.md
CHANGED
|
@@ -23,9 +23,9 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
|
|
|
23
23
|
|
|
24
24
|
### Testing Framework
|
|
25
25
|
|
|
26
|
-
- **Vitest**
|
|
27
|
-
- **
|
|
28
|
-
- **
|
|
26
|
+
- **Vitest** in the Node test environment
|
|
27
|
+
- **Mocked Cloudflare storage interfaces** - Tests provide in-memory `DurableObjectStorage` and SQL storage doubles
|
|
28
|
+
- **Per-test setup** - Each test creates fresh mock storage and VFS instances
|
|
29
29
|
|
|
30
30
|
## VFS Implementation Comparison
|
|
31
31
|
|
|
@@ -69,11 +69,6 @@ Tests for the CloudflareWorkerVFS implementation, which uses DurableObjectStorag
|
|
|
69
69
|
pnpm install
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
-
2. Ensure Wrangler configuration is correct:
|
|
73
|
-
```bash
|
|
74
|
-
# Check wrangler.toml exists and has correct Durable Object bindings
|
|
75
|
-
```
|
|
76
|
-
|
|
77
72
|
### Test Commands
|
|
78
73
|
|
|
79
74
|
```bash
|
|
@@ -100,8 +95,9 @@ pnpm test --coverage
|
|
|
100
95
|
|
|
101
96
|
### Test Environment
|
|
102
97
|
|
|
103
|
-
- **Runtime**:
|
|
104
|
-
- **Storage**:
|
|
98
|
+
- **Runtime**: Node via Vitest
|
|
99
|
+
- **Storage**: Mocked Durable Object storage and SQL storage instances created per test
|
|
100
|
+
- **Scope**: Unit/integration coverage for VFS behavior against Cloudflare-shaped interfaces, not workerd runtime coverage
|
|
105
101
|
|
|
106
102
|
## Test Structure
|
|
107
103
|
|
|
@@ -185,7 +181,7 @@ describe('Async Storage VFS Test Suite', () => {
|
|
|
185
181
|
|
|
186
182
|
### Automatic Cleanup
|
|
187
183
|
|
|
188
|
-
- **
|
|
184
|
+
- **Fresh mock storage**: Each test creates new in-memory storage instances
|
|
189
185
|
- **Temporary Files**: Cleaned up automatically
|
|
190
186
|
- **Cache Management**: Memory caches cleared between tests
|
|
191
187
|
|
|
@@ -225,6 +221,7 @@ console.log('File metadata:', metadata)
|
|
|
225
221
|
2. **Migration Tests**: Converting between storage backends
|
|
226
222
|
3. **Stress Tests**: High-load scenarios for both implementations
|
|
227
223
|
4. **Real SQLite Integration**: Tests with actual SQLite WASM
|
|
224
|
+
5. **Workers Runtime Coverage**: Add dedicated workerd-based tests if runtime bindings or Wrangler configuration need validation
|
|
228
225
|
|
|
229
226
|
### Implementation Improvements
|
|
230
227
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="vitest/globals" />
|
|
2
2
|
|
|
3
|
+
import { beforeEach, describe, expect, it } from 'vitest'
|
|
4
|
+
|
|
3
5
|
import type { CfTypes } from '@livestore/common-cf'
|
|
4
6
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
5
|
-
import { beforeEach, describe, expect, it } from 'vitest'
|
|
6
7
|
|
|
7
8
|
import { CloudflareWorkerVFS } from '../../mod.ts'
|
|
8
9
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="vitest/globals" />
|
|
2
2
|
|
|
3
|
+
import { beforeEach, describe, expect, it } from 'vitest'
|
|
4
|
+
|
|
3
5
|
import type { CfTypes } from '@livestore/common-cf'
|
|
4
6
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
5
|
-
import { beforeEach, describe, expect, it } from 'vitest'
|
|
6
7
|
|
|
7
8
|
import { CloudflareWorkerVFS } from '../../mod.ts'
|
|
8
9
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="vitest/globals" />
|
|
2
2
|
|
|
3
|
+
import { beforeEach, describe, expect, it } from 'vitest'
|
|
4
|
+
|
|
3
5
|
import type { CfTypes } from '@livestore/common-cf'
|
|
4
6
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
5
|
-
import { beforeEach, describe, expect, it } from 'vitest'
|
|
6
7
|
|
|
7
8
|
import { CloudflareWorkerVFS } from '../../mod.ts'
|
|
8
9
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="vitest/globals" />
|
|
2
2
|
|
|
3
|
+
import { beforeEach, describe, expect, it } from 'vitest'
|
|
4
|
+
|
|
3
5
|
import type { CfTypes } from '@livestore/common-cf'
|
|
4
6
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
5
|
-
import { beforeEach, describe, expect, it } from 'vitest'
|
|
6
7
|
|
|
7
8
|
import { CloudflareWorkerVFS } from '../../mod.ts'
|
|
8
9
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/// <reference types="vitest/globals" />
|
|
2
2
|
|
|
3
|
+
import { beforeEach, describe, expect, it } from 'vitest'
|
|
4
|
+
|
|
3
5
|
import type { CfTypes } from '@livestore/common-cf'
|
|
4
6
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
5
|
-
import { beforeEach, describe, expect, it } from 'vitest'
|
|
6
7
|
|
|
7
8
|
import { CF_SQL_VFS_PAGE_SIZE, CloudflareDurableObjectVFS } from '../../mod.ts'
|
|
8
9
|
|
|
@@ -64,10 +65,12 @@ describe('CloudflareDurableObjectVFS - Core Functionality', () => {
|
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
// SELECT page_data FROM vfs_pages WHERE file_path = ? AND page_no = ?
|
|
67
|
-
if (
|
|
68
|
+
if (
|
|
69
|
+
normalizedQuery.startsWith('SELECT PAGE_DATA FROM VFS_PAGES WHERE FILE_PATH = ? AND PAGE_NO = ?') === true
|
|
70
|
+
) {
|
|
68
71
|
const [filePath, pageNo] = bindings
|
|
69
72
|
const data = mockPages.get(filePath as string)?.get(pageNo as number)
|
|
70
|
-
return createMockCursor(data !== undefined ? [{ page_data: data }] as any : [] as any)
|
|
73
|
+
return createMockCursor(data !== undefined ? ([{ page_data: data }] as any) : ([] as any))
|
|
71
74
|
}
|
|
72
75
|
|
|
73
76
|
// SELECT 1 AS x FROM vfs_pages LIMIT 1
|