@livestore/sqlite-wasm 0.4.0-dev.2 → 0.4.0-dev.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/FacadeVFS.d.ts.map +1 -1
- package/dist/FacadeVFS.js +4 -0
- package/dist/FacadeVFS.js.map +1 -1
- package/dist/browser/mod.d.ts +15 -7
- package/dist/browser/mod.d.ts.map +1 -1
- package/dist/browser/mod.js +49 -43
- package/dist/browser/mod.js.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts +40 -17
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.js +211 -143
- package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
- package/dist/browser/opfs/index.d.ts +3 -2
- package/dist/browser/opfs/index.d.ts.map +1 -1
- package/dist/browser/opfs/index.js +1 -1
- package/dist/browser/opfs/index.js.map +1 -1
- package/dist/browser/opfs/opfs-sah-pool.d.ts +1 -1
- package/dist/browser/opfs/opfs-sah-pool.d.ts.map +1 -1
- package/dist/browser/opfs/opfs-sah-pool.js +1 -1
- package/dist/browser/opfs/opfs-sah-pool.js.map +1 -1
- package/dist/cf/BlockManager.d.ts +61 -0
- package/dist/cf/BlockManager.d.ts.map +1 -0
- package/dist/cf/BlockManager.js +157 -0
- package/dist/cf/BlockManager.js.map +1 -0
- package/dist/cf/CloudflareSqlVFS.d.ts +51 -0
- package/dist/cf/CloudflareSqlVFS.d.ts.map +1 -0
- package/dist/cf/CloudflareSqlVFS.js +351 -0
- package/dist/cf/CloudflareSqlVFS.js.map +1 -0
- package/dist/cf/CloudflareWorkerVFS.d.ts +72 -0
- package/dist/cf/CloudflareWorkerVFS.d.ts.map +1 -0
- package/dist/cf/CloudflareWorkerVFS.js +552 -0
- package/dist/cf/CloudflareWorkerVFS.js.map +1 -0
- package/dist/cf/mod.d.ts +43 -0
- package/dist/cf/mod.d.ts.map +1 -0
- package/dist/cf/mod.js +74 -0
- package/dist/cf/mod.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +314 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +266 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +462 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +334 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts +2 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts.map +1 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +354 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -0
- package/dist/load-wasm/mod.node.d.ts.map +1 -1
- package/dist/load-wasm/mod.node.js +1 -2
- package/dist/load-wasm/mod.node.js.map +1 -1
- package/dist/load-wasm/mod.workerd.d.ts +2 -0
- package/dist/load-wasm/mod.workerd.d.ts.map +1 -0
- package/dist/load-wasm/mod.workerd.js +28 -0
- package/dist/load-wasm/mod.workerd.js.map +1 -0
- package/dist/make-sqlite-db.d.ts +1 -0
- package/dist/make-sqlite-db.d.ts.map +1 -1
- package/dist/make-sqlite-db.js +29 -8
- package/dist/make-sqlite-db.js.map +1 -1
- package/dist/node/NodeFS.d.ts +1 -2
- package/dist/node/NodeFS.d.ts.map +1 -1
- package/dist/node/NodeFS.js +1 -6
- package/dist/node/NodeFS.js.map +1 -1
- package/dist/node/mod.d.ts.map +1 -1
- package/dist/node/mod.js +5 -10
- package/dist/node/mod.js.map +1 -1
- package/package.json +21 -8
- package/src/FacadeVFS.ts +5 -0
- package/src/browser/mod.ts +39 -13
- package/src/browser/opfs/AccessHandlePoolVFS.ts +387 -225
- package/src/browser/opfs/index.ts +4 -3
- package/src/browser/opfs/opfs-sah-pool.ts +1 -1
- package/src/cf/BlockManager.ts +225 -0
- package/src/cf/CloudflareSqlVFS.ts +450 -0
- package/src/cf/CloudflareWorkerVFS.ts +664 -0
- package/src/cf/README.md +60 -0
- package/src/cf/mod.ts +143 -0
- package/src/cf/test/README.md +224 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +389 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +322 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +585 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +403 -0
- package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +433 -0
- package/src/load-wasm/mod.node.ts +1 -2
- package/src/load-wasm/mod.workerd.ts +28 -0
- package/src/make-sqlite-db.ts +39 -8
- package/src/node/NodeFS.ts +1 -9
- package/src/node/mod.ts +5 -12
- package/src/ambient.d.ts +0 -18
|
@@ -1,5 +1,9 @@
|
|
|
1
|
-
import { Effect, Schedule, Schema } from '@livestore/utils/effect'
|
|
2
1
|
// Based on https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js
|
|
2
|
+
/// <reference lib="webworker" />
|
|
3
|
+
|
|
4
|
+
import { shouldNeverHappen } from '@livestore/utils'
|
|
5
|
+
import { Effect, Runtime, Schedule, type Scope, Stream } from '@livestore/utils/effect'
|
|
6
|
+
import { Opfs, type WebError } from '@livestore/utils/effect/browser'
|
|
3
7
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
4
8
|
import { FacadeVFS } from '../../FacadeVFS.ts'
|
|
5
9
|
|
|
@@ -20,7 +24,33 @@ const HEADER_OFFSET_DATA = SECTOR_SIZE
|
|
|
20
24
|
const PERSISTENT_FILE_TYPES =
|
|
21
25
|
VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
|
|
22
26
|
|
|
23
|
-
|
|
27
|
+
// OPFS file pool capacity must be predicted rather than dynamically increased because
|
|
28
|
+
// capacity expansion (addCapacity) is async while SQLite operations are synchronous.
|
|
29
|
+
// We cannot await in the middle of sqlite3.step() calls without making the API async.
|
|
30
|
+
//
|
|
31
|
+
// We over-allocate because:
|
|
32
|
+
// 1. SQLite’s temporary file usage is not part of its API contract.
|
|
33
|
+
// Future SQLite versions may create additional temporary files without notice.
|
|
34
|
+
// See: https://www.sqlite.org/tempfiles.html
|
|
35
|
+
// 2. In the future, we may change how we operate the SQLite DBs,
|
|
36
|
+
// which may increase the number of files needed.
|
|
37
|
+
// e.g. enabling the WAL mode, using multi-DB transactions, etc.
|
|
38
|
+
//
|
|
39
|
+
// TRADEOFF: Higher capacity means the VFS opens and keeps more file handles, consuming
|
|
40
|
+
// browser resources. Lower capacity risks "SQLITE_CANTOPEN" errors during operations.
|
|
41
|
+
//
|
|
42
|
+
// CAPACITY CALCULATION:
|
|
43
|
+
// - 2 main databases (state + eventlog) × 4 files each (main, journal, WAL, shm) = 8 files
|
|
44
|
+
// - Up to 5 SQLite temporary files (super-journal, temp DB, materializations,
|
|
45
|
+
// transient indices, VACUUM temp DB) = 5 files
|
|
46
|
+
// - Transient state database archival operations = 1 file
|
|
47
|
+
// - Safety buffer for future SQLite versions and unpredictable usage = 6 files
|
|
48
|
+
// Total: 20 files
|
|
49
|
+
//
|
|
50
|
+
// References:
|
|
51
|
+
// - https://sqlite.org/forum/info/a3da1e34d8
|
|
52
|
+
// - https://www.sqlite.org/tempfiles.html
|
|
53
|
+
const DEFAULT_CAPACITY = 20
|
|
24
54
|
|
|
25
55
|
/**
|
|
26
56
|
* This VFS uses the updated Access Handle API with all synchronous methods
|
|
@@ -36,6 +66,9 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
36
66
|
#directoryPath
|
|
37
67
|
#directoryHandle: FileSystemDirectoryHandle | undefined
|
|
38
68
|
|
|
69
|
+
// Runtime for executing Effect operations
|
|
70
|
+
readonly #runtime: Runtime.Runtime<Opfs.Opfs | Scope.Scope>
|
|
71
|
+
|
|
39
72
|
// The OPFS files all have randomly-generated names that do not match
|
|
40
73
|
// the SQLite files whose data they contain. This map links those names
|
|
41
74
|
// with their respective OPFS access handles.
|
|
@@ -49,33 +82,89 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
49
82
|
|
|
50
83
|
#mapIdToFile = new Map<number, { path: string; flags: number; accessHandle: FileSystemSyncAccessHandle }>()
|
|
51
84
|
|
|
52
|
-
static
|
|
53
|
-
const
|
|
54
|
-
|
|
85
|
+
static create = Effect.fn(function* (name: string, directoryPath: string, module: any) {
|
|
86
|
+
const runtime = yield* Effect.runtime<Opfs.Opfs | Scope.Scope>()
|
|
87
|
+
const vfs = new AccessHandlePoolVFS({ name, directoryPath, module, runtime })
|
|
88
|
+
yield* Effect.promise(() => vfs.isReady())
|
|
55
89
|
return vfs
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
constructor(
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
constructor({
|
|
93
|
+
name,
|
|
94
|
+
directoryPath,
|
|
95
|
+
module,
|
|
96
|
+
runtime,
|
|
97
|
+
}: { name: string; directoryPath: string; module: any; runtime: Runtime.Runtime<Opfs.Opfs | Scope.Scope> }) {
|
|
59
98
|
super(name, module)
|
|
60
99
|
this.#directoryPath = directoryPath
|
|
100
|
+
this.#runtime = runtime
|
|
61
101
|
}
|
|
62
102
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
103
|
+
/**
|
|
104
|
+
* Get the OPFS file name that contains the data for the given SQLite file.
|
|
105
|
+
*
|
|
106
|
+
* @remarks
|
|
107
|
+
*
|
|
108
|
+
* This would be for one of the files in the pool managed by this VFS.
|
|
109
|
+
* It's not the same as the SQLite file name. It's a randomly-generated
|
|
110
|
+
* string that is not meaningful to the application.
|
|
111
|
+
*/
|
|
112
|
+
getOpfsFileName = Effect.fn((zName: string) =>
|
|
113
|
+
Effect.gen(this, function* () {
|
|
114
|
+
const path = this.#getPath(zName)
|
|
115
|
+
const accessHandle = this.#mapPathToAccessHandle.get(path)!
|
|
116
|
+
return this.#mapAccessHandleToName.get(accessHandle)!
|
|
117
|
+
}),
|
|
118
|
+
)
|
|
68
119
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
120
|
+
/**
|
|
121
|
+
* Reads the SQLite payload (without the OPFS header) for the given file.
|
|
122
|
+
*
|
|
123
|
+
* @privateRemarks
|
|
124
|
+
*
|
|
125
|
+
* Since the file's access handle is a FileSystemSyncAccessHandle — which
|
|
126
|
+
* acquires an exclusive lock — we don't need to handle short reads as
|
|
127
|
+
* the file cannot be modified by other threads.
|
|
128
|
+
*/
|
|
129
|
+
readFilePayload = Effect.fn((zName: string) =>
|
|
130
|
+
Effect.gen(this, function* () {
|
|
131
|
+
const path = this.#getPath(zName)
|
|
132
|
+
const accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
133
|
+
|
|
134
|
+
if (accessHandle === undefined) {
|
|
135
|
+
return shouldNeverHappen('Cannot read payload for untracked OPFS path')
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const fileSize = yield* Opfs.Opfs.syncGetSize(accessHandle)
|
|
139
|
+
if (fileSize <= HEADER_OFFSET_DATA) {
|
|
140
|
+
return shouldNeverHappen(
|
|
141
|
+
`OPFS file too small to contain header and payload: size ${fileSize} < HEADER_OFFSET_DATA ${HEADER_OFFSET_DATA}`,
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const payloadSize = fileSize - HEADER_OFFSET_DATA
|
|
146
|
+
const payload = new Uint8Array(payloadSize)
|
|
147
|
+
const bytesRead = yield* Opfs.Opfs.syncRead(accessHandle, payload, { at: HEADER_OFFSET_DATA })
|
|
148
|
+
if (bytesRead !== payloadSize) {
|
|
149
|
+
return shouldNeverHappen(`Failed to read full payload from OPFS file: read ${bytesRead}/${payloadSize}`)
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return payload.buffer
|
|
153
|
+
}),
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
resetAccessHandle = Effect.fn((zName: string) =>
|
|
157
|
+
Effect.gen(this, function* () {
|
|
158
|
+
const path = this.#getPath(zName)
|
|
159
|
+
const accessHandle = this.#mapPathToAccessHandle.get(path)!
|
|
160
|
+
yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
161
|
+
// accessHandle.write(new Uint8Array(), { at: HEADER_OFFSET_DATA })
|
|
162
|
+
// accessHandle.flush()
|
|
163
|
+
}),
|
|
164
|
+
)
|
|
76
165
|
|
|
77
166
|
jOpen(zName: string, fileId: number, flags: number, pOutFlags: DataView): number {
|
|
78
|
-
|
|
167
|
+
return Effect.gen(this, function* () {
|
|
79
168
|
// First try to open a path that already exists in the file system.
|
|
80
169
|
const path = zName ? this.#getPath(zName) : Math.random().toString(36)
|
|
81
170
|
let accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
@@ -84,16 +173,15 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
84
173
|
if (this.getSize() < this.getCapacity()) {
|
|
85
174
|
// Choose an unassociated OPFS file from the pool.
|
|
86
175
|
;[accessHandle] = this.#availableAccessHandles.keys()
|
|
87
|
-
this.#setAssociatedPath(accessHandle!, path, flags)
|
|
176
|
+
yield* this.#setAssociatedPath(accessHandle!, path, flags)
|
|
88
177
|
} else {
|
|
89
178
|
// Out of unassociated files. This can be fixed by calling
|
|
90
179
|
// addCapacity() from the application.
|
|
91
|
-
|
|
180
|
+
return yield* Effect.dieMessage('cannot create file')
|
|
92
181
|
}
|
|
93
182
|
}
|
|
94
|
-
if (!accessHandle)
|
|
95
|
-
|
|
96
|
-
}
|
|
183
|
+
if (!accessHandle) return yield* Effect.dieMessage('file not found')
|
|
184
|
+
|
|
97
185
|
// Subsequent methods are only passed the fileId, so make sure we have
|
|
98
186
|
// a way to get the file resources.
|
|
99
187
|
const file = { path, flags, accessHandle }
|
|
@@ -101,61 +189,102 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
101
189
|
|
|
102
190
|
pOutFlags.setInt32(0, flags, true)
|
|
103
191
|
return VFS.SQLITE_OK
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
192
|
+
}).pipe(
|
|
193
|
+
Effect.tapCauseLogPretty,
|
|
194
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_CANTOPEN)),
|
|
195
|
+
Runtime.runSync(this.#runtime),
|
|
196
|
+
)
|
|
108
197
|
}
|
|
109
198
|
|
|
110
199
|
jClose(fileId: number): number {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
file
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
200
|
+
return Effect.gen(this, function* () {
|
|
201
|
+
const file = this.#mapIdToFile.get(fileId)
|
|
202
|
+
if (file) {
|
|
203
|
+
yield* Opfs.Opfs.syncFlush(file.accessHandle)
|
|
204
|
+
this.#mapIdToFile.delete(fileId)
|
|
205
|
+
if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
|
|
206
|
+
yield* this.#deletePath(file.path)
|
|
207
|
+
}
|
|
117
208
|
}
|
|
118
|
-
|
|
119
|
-
|
|
209
|
+
return VFS.SQLITE_OK
|
|
210
|
+
}).pipe(
|
|
211
|
+
Effect.tapCauseLogPretty,
|
|
212
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_CLOSE)),
|
|
213
|
+
Runtime.runSync(this.#runtime),
|
|
214
|
+
)
|
|
120
215
|
}
|
|
121
216
|
|
|
122
217
|
jRead(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
218
|
+
return Effect.gen(this, function* () {
|
|
219
|
+
const file = this.#mapIdToFile.get(fileId)!
|
|
220
|
+
const nBytes = yield* Opfs.Opfs.syncRead(file.accessHandle, pData.subarray(), {
|
|
221
|
+
at: HEADER_OFFSET_DATA + iOffset,
|
|
222
|
+
})
|
|
223
|
+
if (nBytes < pData.byteLength) {
|
|
224
|
+
pData.fill(0, nBytes, pData.byteLength)
|
|
225
|
+
return VFS.SQLITE_IOERR_SHORT_READ
|
|
226
|
+
}
|
|
227
|
+
return VFS.SQLITE_OK
|
|
228
|
+
}).pipe(
|
|
229
|
+
Effect.tapCauseLogPretty,
|
|
230
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_READ)),
|
|
231
|
+
Runtime.runSync(this.#runtime),
|
|
232
|
+
)
|
|
132
233
|
}
|
|
133
234
|
|
|
134
235
|
jWrite(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
236
|
+
return Effect.gen(this, function* () {
|
|
237
|
+
const file = this.#mapIdToFile.get(fileId)!
|
|
238
|
+
const nBytes = yield* Opfs.Opfs.syncWrite(file.accessHandle, pData.subarray(), {
|
|
239
|
+
at: HEADER_OFFSET_DATA + iOffset,
|
|
240
|
+
})
|
|
241
|
+
if (nBytes !== pData.byteLength) {
|
|
242
|
+
return yield* Effect.dieMessage(`Wrote ${nBytes} bytes, expected ${pData.byteLength}`)
|
|
243
|
+
}
|
|
244
|
+
return VFS.SQLITE_OK
|
|
245
|
+
}).pipe(
|
|
246
|
+
Effect.tapCauseLogPretty,
|
|
247
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_WRITE)),
|
|
248
|
+
Runtime.runSync(this.#runtime),
|
|
249
|
+
)
|
|
140
250
|
}
|
|
141
251
|
|
|
142
252
|
jTruncate(fileId: number, iSize: number): number {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
253
|
+
return Effect.gen(this, function* () {
|
|
254
|
+
const file = this.#mapIdToFile.get(fileId)!
|
|
255
|
+
yield* Opfs.Opfs.syncTruncate(file.accessHandle, HEADER_OFFSET_DATA + iSize)
|
|
256
|
+
return VFS.SQLITE_OK
|
|
257
|
+
}).pipe(
|
|
258
|
+
Effect.tapCauseLogPretty,
|
|
259
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_TRUNCATE)),
|
|
260
|
+
Runtime.runSync(this.#runtime),
|
|
261
|
+
)
|
|
146
262
|
}
|
|
147
263
|
|
|
148
264
|
jSync(fileId: number, _flags: number): number {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
265
|
+
return Effect.gen(this, function* () {
|
|
266
|
+
const file = this.#mapIdToFile.get(fileId)!
|
|
267
|
+
yield* Opfs.Opfs.syncFlush(file.accessHandle)
|
|
268
|
+
return VFS.SQLITE_OK
|
|
269
|
+
}).pipe(
|
|
270
|
+
Effect.tapCauseLogPretty,
|
|
271
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSYNC)),
|
|
272
|
+
Runtime.runSync(this.#runtime),
|
|
273
|
+
)
|
|
152
274
|
}
|
|
153
275
|
|
|
154
276
|
jFileSize(fileId: number, pSize64: DataView): number {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
277
|
+
return Effect.gen(this, function* () {
|
|
278
|
+
const file = this.#mapIdToFile.get(fileId)!
|
|
279
|
+
const opfsFileSize = yield* Opfs.Opfs.syncGetSize(file.accessHandle)
|
|
280
|
+
const size = opfsFileSize - HEADER_OFFSET_DATA
|
|
281
|
+
pSize64.setBigInt64(0, BigInt(size), true)
|
|
282
|
+
return VFS.SQLITE_OK
|
|
283
|
+
}).pipe(
|
|
284
|
+
Effect.tapCauseLogPretty,
|
|
285
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSTAT)),
|
|
286
|
+
Runtime.runSync(this.#runtime),
|
|
287
|
+
)
|
|
159
288
|
}
|
|
160
289
|
|
|
161
290
|
jSectorSize(_fileId: number): number {
|
|
@@ -167,38 +296,46 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
167
296
|
}
|
|
168
297
|
|
|
169
298
|
jAccess(zName: string, _flags: number, pResOut: DataView): number {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
299
|
+
return Effect.gen(this, function* () {
|
|
300
|
+
const path = this.#getPath(zName)
|
|
301
|
+
pResOut.setInt32(0, this.#mapPathToAccessHandle.has(path) ? 1 : 0, true)
|
|
302
|
+
return VFS.SQLITE_OK
|
|
303
|
+
}).pipe(
|
|
304
|
+
Effect.tapCauseLogPretty,
|
|
305
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_ACCESS)),
|
|
306
|
+
Runtime.runSync(this.#runtime),
|
|
307
|
+
)
|
|
173
308
|
}
|
|
174
309
|
|
|
175
310
|
jDelete(zName: string, _syncDir: number): number {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
311
|
+
return Effect.gen(this, function* () {
|
|
312
|
+
const path = this.#getPath(zName)
|
|
313
|
+
yield* this.#deletePath(path)
|
|
314
|
+
return VFS.SQLITE_OK
|
|
315
|
+
}).pipe(
|
|
316
|
+
Effect.tapCauseLogPretty,
|
|
317
|
+
Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_DELETE)),
|
|
318
|
+
Runtime.runSync(this.#runtime),
|
|
319
|
+
)
|
|
179
320
|
}
|
|
180
321
|
|
|
181
|
-
|
|
182
|
-
this.#releaseAccessHandles()
|
|
322
|
+
close() {
|
|
323
|
+
this.#releaseAccessHandles().pipe(Runtime.runPromise(this.#runtime))
|
|
183
324
|
}
|
|
184
325
|
|
|
185
326
|
async isReady() {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
327
|
+
return Effect.gen(this, function* () {
|
|
328
|
+
if (!this.#directoryHandle) {
|
|
329
|
+
// All files are stored in a single directory.
|
|
330
|
+
this.#directoryHandle = yield* Opfs.getDirectoryHandleByPath(this.#directoryPath, { create: true })
|
|
331
|
+
|
|
332
|
+
yield* this.#acquireAccessHandles()
|
|
333
|
+
if (this.getCapacity() === 0) {
|
|
334
|
+
yield* this.addCapacity(DEFAULT_CAPACITY)
|
|
192
335
|
}
|
|
193
336
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
await this.#acquireAccessHandles()
|
|
197
|
-
if (this.getCapacity() === 0) {
|
|
198
|
-
await this.addCapacity(DEFAULT_CAPACITY)
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
return true
|
|
337
|
+
return true
|
|
338
|
+
}).pipe(Runtime.runPromise(this.#runtime))
|
|
202
339
|
}
|
|
203
340
|
|
|
204
341
|
/**
|
|
@@ -216,156 +353,184 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
216
353
|
}
|
|
217
354
|
|
|
218
355
|
/**
|
|
219
|
-
*
|
|
356
|
+
* Get all currently tracked SQLite file paths.
|
|
357
|
+
* This can be used by higher-level components for file management operations.
|
|
358
|
+
*
|
|
359
|
+
* @returns Array of currently active SQLite file paths
|
|
220
360
|
*/
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
const handle = await this.#directoryHandle!.getFileHandle(name, {
|
|
225
|
-
create: true,
|
|
226
|
-
})
|
|
361
|
+
getTrackedFilePaths(): string[] {
|
|
362
|
+
return Array.from(this.#mapPathToAccessHandle.keys())
|
|
363
|
+
}
|
|
227
364
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
365
|
+
/**
|
|
366
|
+
* Increase the capacity of the file system by n.
|
|
367
|
+
*/
|
|
368
|
+
addCapacity: (n: number) => Effect.Effect<void, WebError.WebError, Opfs.Opfs | Scope.Scope> = Effect.fn((n: number) =>
|
|
369
|
+
Effect.repeatN(
|
|
370
|
+
Effect.gen(this, function* () {
|
|
371
|
+
const name = Math.random().toString(36).replace('0.', '')
|
|
372
|
+
const accessHandle = yield* Opfs.Opfs.getFileHandle(this.#directoryHandle!, name, { create: true }).pipe(
|
|
373
|
+
Effect.andThen((handle) => Opfs.Opfs.createSyncAccessHandle(handle)),
|
|
374
|
+
Effect.retry(Schedule.exponentialBackoff10Sec),
|
|
375
|
+
)
|
|
376
|
+
this.#mapAccessHandleToName.set(accessHandle, name)
|
|
233
377
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
378
|
+
yield* this.#setAssociatedPath(accessHandle, '', 0)
|
|
379
|
+
}),
|
|
380
|
+
n,
|
|
381
|
+
),
|
|
382
|
+
)
|
|
238
383
|
|
|
239
384
|
/**
|
|
240
385
|
* Decrease the capacity of the file system by n. The capacity cannot be
|
|
241
386
|
* decreased to fewer than the current number of SQLite files in the
|
|
242
387
|
* file system.
|
|
243
388
|
*/
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
389
|
+
removeCapacity = Effect.fn((n: number) =>
|
|
390
|
+
Effect.gen(this, function* () {
|
|
391
|
+
let nRemoved = 0
|
|
392
|
+
yield* Effect.forEach(
|
|
393
|
+
this.#availableAccessHandles,
|
|
394
|
+
(accessHandle) =>
|
|
395
|
+
Effect.gen(this, function* () {
|
|
396
|
+
if (nRemoved === n || this.getSize() === this.getCapacity()) return nRemoved
|
|
397
|
+
|
|
398
|
+
const name = this.#mapAccessHandleToName.get(accessHandle)!
|
|
399
|
+
accessHandle.close()
|
|
400
|
+
yield* Opfs.Opfs.removeEntry(this.#directoryHandle!, name)
|
|
401
|
+
this.#mapAccessHandleToName.delete(accessHandle)
|
|
402
|
+
this.#availableAccessHandles.delete(accessHandle)
|
|
403
|
+
++nRemoved
|
|
404
|
+
}),
|
|
405
|
+
{ discard: true },
|
|
406
|
+
)
|
|
407
|
+
return nRemoved
|
|
408
|
+
}),
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
#acquireAccessHandles = Effect.fn(() =>
|
|
412
|
+
Effect.gen(this, function* () {
|
|
413
|
+
const handlesStream = yield* Opfs.Opfs.values(this.#directoryHandle!)
|
|
414
|
+
|
|
415
|
+
yield* handlesStream.pipe(
|
|
416
|
+
Stream.filter((handle): handle is FileSystemFileHandle => handle.kind === 'file'),
|
|
417
|
+
Stream.mapEffect(
|
|
418
|
+
(fileHandle) =>
|
|
419
|
+
Effect.gen(this, function* () {
|
|
420
|
+
const accessHandle = yield* Opfs.Opfs.createSyncAccessHandle(fileHandle)
|
|
421
|
+
return {
|
|
422
|
+
accessHandle,
|
|
423
|
+
opfsFileName: fileHandle.name,
|
|
424
|
+
path: yield* this.#getAssociatedPath(accessHandle),
|
|
425
|
+
} as const
|
|
426
|
+
}),
|
|
427
|
+
{ concurrency: 'unbounded' },
|
|
428
|
+
),
|
|
429
|
+
Stream.runForEach(({ opfsFileName, accessHandle, path }) =>
|
|
430
|
+
Effect.gen(this, function* () {
|
|
431
|
+
this.#mapAccessHandleToName.set(accessHandle, opfsFileName)
|
|
432
|
+
if (path) {
|
|
433
|
+
this.#mapPathToAccessHandle.set(path, accessHandle)
|
|
434
|
+
} else {
|
|
435
|
+
this.#availableAccessHandles.add(accessHandle)
|
|
436
|
+
}
|
|
437
|
+
}),
|
|
438
|
+
),
|
|
439
|
+
)
|
|
440
|
+
}),
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
#releaseAccessHandles = Effect.fn(() =>
|
|
444
|
+
Effect.gen(this, function* () {
|
|
445
|
+
yield* Effect.forEach(
|
|
446
|
+
this.#mapAccessHandleToName.keys(),
|
|
447
|
+
(accessHandle) => Effect.sync(() => accessHandle.close()),
|
|
448
|
+
{ concurrency: 'unbounded', discard: true },
|
|
449
|
+
)
|
|
450
|
+
this.#mapAccessHandleToName.clear()
|
|
451
|
+
this.#mapPathToAccessHandle.clear()
|
|
452
|
+
this.#availableAccessHandles.clear()
|
|
453
|
+
}),
|
|
454
|
+
)
|
|
294
455
|
|
|
295
456
|
/**
|
|
296
457
|
* Read and return the associated path from an OPFS file header.
|
|
297
458
|
* Empty string is returned for an unassociated OPFS file.
|
|
298
459
|
* @returns {string} path or empty string
|
|
299
460
|
*/
|
|
300
|
-
#getAssociatedPath(accessHandle: FileSystemSyncAccessHandle)
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
461
|
+
#getAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle) =>
|
|
462
|
+
Effect.gen(this, function* () {
|
|
463
|
+
// Read the path and digest of the path from the file.
|
|
464
|
+
const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
|
|
465
|
+
yield* Opfs.Opfs.syncRead(accessHandle, corpus, { at: 0 })
|
|
466
|
+
|
|
467
|
+
// Delete files not expected to be present.
|
|
468
|
+
const dataView = new DataView(corpus.buffer, corpus.byteOffset)
|
|
469
|
+
const flags = dataView.getUint32(HEADER_OFFSET_FLAGS)
|
|
470
|
+
if (corpus[0] && (flags & VFS.SQLITE_OPEN_DELETEONCLOSE || (flags & PERSISTENT_FILE_TYPES) === 0)) {
|
|
471
|
+
yield* Effect.logWarning(`Remove file with unexpected flags ${flags.toString(16)}`)
|
|
472
|
+
yield* this.#setAssociatedPath(accessHandle, '', 0)
|
|
473
|
+
return ''
|
|
474
|
+
}
|
|
313
475
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
476
|
+
const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4)
|
|
477
|
+
yield* Opfs.Opfs.syncRead(accessHandle, fileDigest, { at: HEADER_OFFSET_DIGEST })
|
|
478
|
+
|
|
479
|
+
// Verify the digest.
|
|
480
|
+
const computedDigest = this.#computeDigest(corpus)
|
|
481
|
+
if (fileDigest.every((value, i) => value === computedDigest[i])) {
|
|
482
|
+
// Good digest. Decode the null-terminated path string.
|
|
483
|
+
const pathBytes = corpus.indexOf(0)
|
|
484
|
+
if (pathBytes === 0) {
|
|
485
|
+
// Ensure that unassociated files are empty. Unassociated files are
|
|
486
|
+
// truncated in #setAssociatedPath after the header is written. If
|
|
487
|
+
// an interruption occurs right before the truncation then garbage
|
|
488
|
+
// may remain in the file.
|
|
489
|
+
yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
490
|
+
}
|
|
491
|
+
return new TextDecoder().decode(corpus.subarray(0, pathBytes))
|
|
492
|
+
} else {
|
|
493
|
+
// Bad digest. Repair this header.
|
|
494
|
+
yield* Effect.logWarning('Disassociating file with bad digest.')
|
|
495
|
+
yield* this.#setAssociatedPath(accessHandle, '', 0)
|
|
496
|
+
return ''
|
|
328
497
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
// Bad digest. Repair this header.
|
|
332
|
-
console.warn('Disassociating file with bad digest.')
|
|
333
|
-
this.#setAssociatedPath(accessHandle, '', 0)
|
|
334
|
-
return ''
|
|
335
|
-
}
|
|
336
|
-
}
|
|
498
|
+
}),
|
|
499
|
+
)
|
|
337
500
|
|
|
338
501
|
/**
|
|
339
502
|
* Set the path on an OPFS file header.
|
|
340
503
|
*/
|
|
341
|
-
#setAssociatedPath(accessHandle: FileSystemSyncAccessHandle, path: string, flags: number)
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
504
|
+
#setAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle, path: string, flags: number) =>
|
|
505
|
+
Effect.gen(this, function* () {
|
|
506
|
+
// Convert the path string to UTF-8.
|
|
507
|
+
const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
|
|
508
|
+
const encodedResult = new TextEncoder().encodeInto(path, corpus)
|
|
509
|
+
if (encodedResult.written >= HEADER_MAX_PATH_SIZE) {
|
|
510
|
+
return yield* Effect.dieMessage('path too long')
|
|
511
|
+
}
|
|
348
512
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
513
|
+
// Add the creation flags.
|
|
514
|
+
const dataView = new DataView(corpus.buffer, corpus.byteOffset)
|
|
515
|
+
dataView.setUint32(HEADER_OFFSET_FLAGS, flags)
|
|
516
|
+
|
|
517
|
+
// Write the OPFS file header, including the digest.
|
|
518
|
+
const digest = this.#computeDigest(corpus)
|
|
519
|
+
yield* Opfs.Opfs.syncWrite(accessHandle, corpus, { at: 0 })
|
|
520
|
+
yield* Opfs.Opfs.syncWrite(accessHandle, digest, { at: HEADER_OFFSET_DIGEST })
|
|
521
|
+
yield* Opfs.Opfs.syncFlush(accessHandle)
|
|
522
|
+
|
|
523
|
+
if (path) {
|
|
524
|
+
this.#mapPathToAccessHandle.set(path, accessHandle)
|
|
525
|
+
this.#availableAccessHandles.delete(accessHandle)
|
|
526
|
+
} else {
|
|
527
|
+
// This OPFS file doesn't represent any SQLite file so it doesn't
|
|
528
|
+
// need to keep any data.
|
|
529
|
+
yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
|
|
530
|
+
this.#availableAccessHandles.add(accessHandle)
|
|
531
|
+
}
|
|
532
|
+
}),
|
|
533
|
+
)
|
|
369
534
|
|
|
370
535
|
/**
|
|
371
536
|
* We need a synchronous digest function so can't use WebCrypto.
|
|
@@ -404,17 +569,14 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
404
569
|
* Remove the association between a path and an OPFS file.
|
|
405
570
|
* @param {string} path
|
|
406
571
|
*/
|
|
407
|
-
#deletePath(path: string)
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
572
|
+
#deletePath = Effect.fn((path: string) =>
|
|
573
|
+
Effect.gen(this, function* () {
|
|
574
|
+
const accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
575
|
+
if (accessHandle) {
|
|
576
|
+
// Un-associate the SQLite path from the OPFS file.
|
|
577
|
+
this.#mapPathToAccessHandle.delete(path)
|
|
578
|
+
yield* this.#setAssociatedPath(accessHandle, '', 0)
|
|
579
|
+
}
|
|
580
|
+
}),
|
|
581
|
+
)
|
|
415
582
|
}
|
|
416
|
-
|
|
417
|
-
export class OpfsError extends Schema.TaggedError<OpfsError>()('OpfsError', {
|
|
418
|
-
cause: Schema.Defect,
|
|
419
|
-
path: Schema.String,
|
|
420
|
-
}) {}
|