@livestore/sqlite-wasm 0.4.0-dev.22 → 0.4.0-dev.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/FacadeVFS.d.ts +0 -1
- package/dist/FacadeVFS.d.ts.map +1 -1
- package/dist/FacadeVFS.js +9 -14
- package/dist/FacadeVFS.js.map +1 -1
- package/dist/browser/mod.d.ts +2 -2
- package/dist/browser/mod.d.ts.map +1 -1
- package/dist/browser/mod.js +2 -2
- package/dist/browser/mod.js.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.js +15 -13
- package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
- package/dist/browser/opfs/opfs-sah-pool.js +3 -3
- package/dist/browser/opfs/opfs-sah-pool.js.map +1 -1
- package/dist/cf/CloudflareDurableObjectVFS.d.ts +104 -0
- package/dist/cf/CloudflareDurableObjectVFS.d.ts.map +1 -0
- package/dist/cf/CloudflareDurableObjectVFS.js +281 -0
- package/dist/cf/CloudflareDurableObjectVFS.js.map +1 -0
- package/dist/cf/CloudflareWorkerVFS.d.ts.map +1 -1
- package/dist/cf/CloudflareWorkerVFS.js +29 -28
- package/dist/cf/CloudflareWorkerVFS.js.map +1 -1
- package/dist/cf/mod.d.ts +3 -4
- package/dist/cf/mod.d.ts.map +1 -1
- package/dist/cf/mod.js +4 -12
- package/dist/cf/mod.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +5 -4
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +3 -3
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +3 -3
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -1
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +3 -3
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -1
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +194 -179
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -1
- package/dist/in-memory-vfs.d.ts.map +1 -1
- package/dist/in-memory-vfs.js +0 -1
- package/dist/in-memory-vfs.js.map +1 -1
- package/dist/load-wasm/mod.node.js +1 -1
- package/dist/load-wasm/mod.node.js.map +1 -1
- package/dist/load-wasm/mod.workerd.d.ts.map +1 -1
- package/dist/load-wasm/mod.workerd.js.map +1 -1
- package/dist/make-sqlite-db.d.ts.map +1 -1
- package/dist/make-sqlite-db.js +16 -4
- package/dist/make-sqlite-db.js.map +1 -1
- package/dist/node/NodeFS.d.ts.map +1 -1
- package/dist/node/NodeFS.js +13 -13
- package/dist/node/NodeFS.js.map +1 -1
- package/package.json +54 -15
- package/src/FacadeVFS.ts +9 -14
- package/src/browser/mod.ts +1 -1
- package/src/browser/opfs/AccessHandlePoolVFS.ts +33 -25
- package/src/browser/opfs/opfs-sah-pool.ts +3 -3
- package/src/cf/CloudflareDurableObjectVFS.ts +325 -0
- package/src/cf/CloudflareWorkerVFS.ts +41 -39
- package/src/cf/README.md +3 -3
- package/src/cf/mod.ts +10 -15
- package/src/cf/test/README.md +55 -26
- package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +6 -4
- package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +4 -3
- package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +4 -3
- package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +4 -3
- package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +228 -197
- package/src/in-memory-vfs.ts +0 -1
- package/src/load-wasm/mod.node.ts +1 -1
- package/src/load-wasm/mod.workerd.ts +0 -1
- package/src/make-sqlite-db.ts +24 -4
- package/src/node/NodeFS.ts +24 -23
- package/dist/cf/BlockManager.d.ts +0 -61
- package/dist/cf/BlockManager.d.ts.map +0 -1
- package/dist/cf/BlockManager.js +0 -157
- package/dist/cf/BlockManager.js.map +0 -1
- package/dist/cf/CloudflareSqlVFS.d.ts +0 -51
- package/dist/cf/CloudflareSqlVFS.d.ts.map +0 -1
- package/dist/cf/CloudflareSqlVFS.js +0 -351
- package/dist/cf/CloudflareSqlVFS.js.map +0 -1
- package/src/cf/BlockManager.ts +0 -225
- package/src/cf/CloudflareSqlVFS.ts +0 -450
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import type { CfTypes } from '@livestore/common-cf'
|
|
2
|
+
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
3
|
+
|
|
4
|
+
import { FacadeVFS } from '../FacadeVFS.ts'
|
|
5
|
+
|
|
6
|
+
// Page size for SQL-based storage. Matches dbState page size (PRAGMA page_size=8192)
|
|
7
|
+
// so each SQLite page write maps to exactly one vfs_pages row write — no read-merge-write.
|
|
8
|
+
export const PAGE_SIZE = 8 * 1024 // 8 KiB
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The {@link CloudflareDurableObjectVFS} VFS assumes SQLite is configured with these pragmas.
|
|
12
|
+
* These pragmas are required both for correctness and for minimizing SQLite writes
|
|
13
|
+
* to the underlying storage.
|
|
14
|
+
*/
|
|
15
|
+
export const REQUIRED_PRAGMAS = [
|
|
16
|
+
// Must match the fixed VFS page size so each SQLite page maps to one vfs_pages row.
|
|
17
|
+
`page_size=${PAGE_SIZE}`,
|
|
18
|
+
// The rollback journal is the largest source of VFS writes. Keeping it
|
|
19
|
+
// in WASM memory avoids writing journal pages through the VFS entirely.
|
|
20
|
+
// This is acceptable because the state database is rebuildable from the
|
|
21
|
+
// eventlog in case of a crash. We still, however, need the journal for
|
|
22
|
+
// transaction rollbacks.
|
|
23
|
+
'journal_mode=MEMORY',
|
|
24
|
+
// Skips jSync VFS calls (already a no-op, but avoids the dispatch).
|
|
25
|
+
'synchronous=OFF',
|
|
26
|
+
// A Durable Object is single-threaded with a single connection, so shared
|
|
27
|
+
// locking is unnecessary. Exclusive mode skips per-transaction lock/unlock
|
|
28
|
+
// VFS calls and hot-journal jAccess checks.
|
|
29
|
+
'locking_mode=EXCLUSIVE',
|
|
30
|
+
// Temp tables and indices stay in WASM memory, preventing temp-file VFS writes.
|
|
31
|
+
'temp_store=MEMORY',
|
|
32
|
+
// Keeps all dirty pages in the WASM page cache until commit. Without this,
|
|
33
|
+
// SQLite may spill pages to the VFS mid-transaction, and if those pages are
|
|
34
|
+
// dirtied again before commit, the same page gets written twice.
|
|
35
|
+
'cache_spill=OFF',
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* wa-sqlite VFS for the LiveStore **state database** (dbState), backed by
|
|
40
|
+
* {@link https://developers.cloudflare.com/durable-objects/api/sql-storage/ | SQLite in Durable Objects}.
|
|
41
|
+
*
|
|
42
|
+
* ## Why wa-sqlite on top of SQLite in Durable Objects
|
|
43
|
+
*
|
|
44
|
+
* LiveStore's state database depends on SQLite APIs that SQLite in
|
|
45
|
+
* Durable Objects does not expose. This VFS persists wa-sqlite's pages as
|
|
46
|
+
* rows in the DO's SQLite storage, giving the state database durable
|
|
47
|
+
* persistence while retaining full access to the SQLite API.
|
|
48
|
+
*
|
|
49
|
+
* ## Write Optimization
|
|
50
|
+
*
|
|
51
|
+
* Every VFS write becomes a (billable) row write in the DO's SQLite. To
|
|
52
|
+
* reduce costs, this VFS assumes a wa-sqlite configured with {@link REQUIRED_PRAGMAS}
|
|
53
|
+
* for minimizing writes.
|
|
54
|
+
*
|
|
55
|
+
* ## File Identity
|
|
56
|
+
*
|
|
57
|
+
* LiveStore currently only persists the main database file through this VFS,
|
|
58
|
+
* but pages still stay keyed by file path, so schema-hashed filenames open
|
|
59
|
+
* isolated databases across process restarts.
|
|
60
|
+
*/
|
|
61
|
+
export class CloudflareDurableObjectVFS extends FacadeVFS {
|
|
62
|
+
#sql: CfTypes.SqlStorage
|
|
63
|
+
/**
|
|
64
|
+
* Tracks the path and open flags for each SQLite file handle.
|
|
65
|
+
*
|
|
66
|
+
* @remarks
|
|
67
|
+
*
|
|
68
|
+
* After `jOpen`, SQLite calls the VFS with `fileId` for reads, writes,
|
|
69
|
+
* truncates, size checks, and close, so we need this in-memory map to get
|
|
70
|
+
* back to the persisted `file_path` key and honor handle-specific flags
|
|
71
|
+
* such as `SQLITE_OPEN_DELETEONCLOSE`.
|
|
72
|
+
*/
|
|
73
|
+
#openFiles = new Map<number, { path: string; flags: number }>()
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* @param name - VFS name registered with wa-sqlite (must be unique per database).
|
|
77
|
+
* @param sql - The Durable Object's {@link CfTypes.SqlStorage} handle.
|
|
78
|
+
* @param module - The wa-sqlite WASM module instance.
|
|
79
|
+
*/
|
|
80
|
+
constructor(name: string, sql: CfTypes.SqlStorage, module: any) {
|
|
81
|
+
super(name, module)
|
|
82
|
+
this.#sql = sql
|
|
83
|
+
this.#sql.exec(
|
|
84
|
+
`CREATE TABLE IF NOT EXISTS vfs_pages (
|
|
85
|
+
file_path TEXT NOT NULL,
|
|
86
|
+
page_no INTEGER NOT NULL,
|
|
87
|
+
page_data BLOB NOT NULL,
|
|
88
|
+
PRIMARY KEY (file_path, page_no)
|
|
89
|
+
)`,
|
|
90
|
+
)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Accepts any open request unconditionally and records which path each fileId refers to.
|
|
95
|
+
*
|
|
96
|
+
* @remarks
|
|
97
|
+
*
|
|
98
|
+
* SQLite's fileId is only an in-memory open-handle identifier. Persisted
|
|
99
|
+
* data must stay keyed by filename so schema-hashed state DB filenames
|
|
100
|
+
* remain isolated from each other across restarts.
|
|
101
|
+
*/
|
|
102
|
+
override jOpen(path: string | null, fileId: number, flags: number, pOutFlags: DataView): number {
|
|
103
|
+
const resolvedPath = this.#resolveOpenPath(path)
|
|
104
|
+
this.#openFiles.set(fileId, { path: resolvedPath, flags })
|
|
105
|
+
pOutFlags.setInt32(0, flags, true)
|
|
106
|
+
return VFS.SQLITE_OK
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
override jClose(fileId: number): number {
|
|
110
|
+
try {
|
|
111
|
+
const openFile = this.#openFiles.get(fileId)
|
|
112
|
+
if (openFile === undefined) return VFS.SQLITE_OK
|
|
113
|
+
|
|
114
|
+
this.#openFiles.delete(fileId)
|
|
115
|
+
|
|
116
|
+
if ((openFile.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) !== 0) {
|
|
117
|
+
this.#deleteFilePages(openFile.path)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return VFS.SQLITE_OK
|
|
121
|
+
} catch (error) {
|
|
122
|
+
console.error('jClose error:', error)
|
|
123
|
+
return VFS.SQLITE_IOERR_CLOSE
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Reads a single page from `vfs_pages` and copies the requested byte range
|
|
129
|
+
* into {@link buffer}.
|
|
130
|
+
*
|
|
131
|
+
* @remarks
|
|
132
|
+
*
|
|
133
|
+
* Missing pages are zero-filled and still return `SQLITE_OK` (not `SQLITE_IOERR_SHORT_READ`)
|
|
134
|
+
* because wa-sqlite expects this during initial database creation.
|
|
135
|
+
*/
|
|
136
|
+
override jRead(fileId: number, buffer: Uint8Array, offset: number): number {
|
|
137
|
+
try {
|
|
138
|
+
const { path } = this.#getOpenFile(fileId)
|
|
139
|
+
const pageNo = Math.floor(offset / PAGE_SIZE)
|
|
140
|
+
const cursor = this.#sql.exec<{ page_data: ArrayBuffer }>(
|
|
141
|
+
'SELECT page_data FROM vfs_pages WHERE file_path = ? AND page_no = ?',
|
|
142
|
+
path,
|
|
143
|
+
pageNo,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
const rows = cursor.toArray()
|
|
147
|
+
|
|
148
|
+
if (rows.length === 0) {
|
|
149
|
+
buffer.fill(0)
|
|
150
|
+
return VFS.SQLITE_OK
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const src = new Uint8Array(rows[0]!.page_data)
|
|
154
|
+
const pageOffset = offset % PAGE_SIZE
|
|
155
|
+
const available = src.byteLength - pageOffset
|
|
156
|
+
|
|
157
|
+
if (available >= buffer.byteLength) {
|
|
158
|
+
buffer.set(src.subarray(pageOffset, pageOffset + buffer.byteLength))
|
|
159
|
+
return VFS.SQLITE_OK
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
buffer.set(src.subarray(pageOffset))
|
|
163
|
+
buffer.fill(0, available)
|
|
164
|
+
return VFS.SQLITE_OK
|
|
165
|
+
} catch (error) {
|
|
166
|
+
console.error('jRead error:', error)
|
|
167
|
+
return VFS.SQLITE_IOERR
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Writes a single page to `vfs_pages`.
|
|
173
|
+
*
|
|
174
|
+
* @remarks
|
|
175
|
+
*
|
|
176
|
+
* The VFS only supports canonical SQLite page writes. Rejecting any other
|
|
177
|
+
* shape turns page-size mismatches into a hard failure instead of silently
|
|
178
|
+
* aliasing multiple SQLite pages onto the same `vfs_pages` row.
|
|
179
|
+
*
|
|
180
|
+
* The data is copied out of the WASM heap via `data.slice()` because
|
|
181
|
+
* Cloudflare's SQL storage cannot persist the {@link FacadeVFS}
|
|
182
|
+
* Proxy-wrapped buffer across DO restarts.
|
|
183
|
+
*/
|
|
184
|
+
override jWrite(fileId: number, data: Uint8Array, offset: number): number {
|
|
185
|
+
try {
|
|
186
|
+
const { path } = this.#getOpenFile(fileId)
|
|
187
|
+
this.#assertCanonicalPageWrite(data, offset)
|
|
188
|
+
const pageNo = Math.floor(offset / PAGE_SIZE)
|
|
189
|
+
// data.slice() copies out of the WASM heap Proxy so CF SQL storage can persist the BLOB correctly.
|
|
190
|
+
this.#sql.exec(
|
|
191
|
+
'INSERT OR REPLACE INTO vfs_pages (file_path, page_no, page_data) VALUES (?, ?, ?)',
|
|
192
|
+
path,
|
|
193
|
+
pageNo,
|
|
194
|
+
data.slice(),
|
|
195
|
+
)
|
|
196
|
+
return VFS.SQLITE_OK
|
|
197
|
+
} catch (error) {
|
|
198
|
+
console.error('jWrite error:', error)
|
|
199
|
+
return VFS.SQLITE_IOERR
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Deletes all pages beyond the new file size boundary. */
|
|
204
|
+
override jTruncate(fileId: number, size: number): number {
|
|
205
|
+
try {
|
|
206
|
+
const { path } = this.#getOpenFile(fileId)
|
|
207
|
+
const lastPageNo = Math.ceil(size / PAGE_SIZE)
|
|
208
|
+
this.#sql.exec('DELETE FROM vfs_pages WHERE file_path = ? AND page_no >= ?', path, lastPageNo)
|
|
209
|
+
return VFS.SQLITE_OK
|
|
210
|
+
} catch (error) {
|
|
211
|
+
console.error('jTruncate error:', error)
|
|
212
|
+
return VFS.SQLITE_IOERR
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Derives file size from `MAX(page_no)`.
|
|
218
|
+
*
|
|
219
|
+
* @remarks
|
|
220
|
+
*
|
|
221
|
+
* Possible because all writes are page-aligned (page size matches `PRAGMA page_size`).
|
|
222
|
+
*/
|
|
223
|
+
override jFileSize(fileId: number, pSize64: DataView): number {
|
|
224
|
+
try {
|
|
225
|
+
const { path } = this.#getOpenFile(fileId)
|
|
226
|
+
const row = this.#sql.exec<{ max_page: number | null }>(
|
|
227
|
+
'SELECT MAX(page_no) AS max_page FROM vfs_pages WHERE file_path = ?',
|
|
228
|
+
path,
|
|
229
|
+
).one()
|
|
230
|
+
const fileSize = row.max_page === null ? 0 : (row.max_page + 1) * PAGE_SIZE
|
|
231
|
+
pSize64.setBigInt64(0, BigInt(fileSize), true)
|
|
232
|
+
return VFS.SQLITE_OK
|
|
233
|
+
} catch (error) {
|
|
234
|
+
console.error('jFileSize error:', error)
|
|
235
|
+
return VFS.SQLITE_IOERR
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Wipes all pages for the requested file path. */
|
|
240
|
+
override jDelete(path: string, _syncDir: number): number {
|
|
241
|
+
try {
|
|
242
|
+
this.#deleteFilePages(this.#getPath(path))
|
|
243
|
+
return VFS.SQLITE_OK
|
|
244
|
+
} catch (error) {
|
|
245
|
+
console.error('jDelete error:', error)
|
|
246
|
+
return VFS.SQLITE_IOERR
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Always reports "not found".
|
|
251
|
+
*
|
|
252
|
+
* @remarks
|
|
253
|
+
*
|
|
254
|
+
* With `locking_mode=EXCLUSIVE` and `journal_mode=MEMORY`, the only `jAccess` call SQLite makes is for
|
|
255
|
+
* hot-journal detection, which does not apply. Returning 1 (found) would
|
|
256
|
+
* trigger hot-journal recovery writes on every cold start.
|
|
257
|
+
*/
|
|
258
|
+
override jAccess(_path: string, _flags: number, pResOut: DataView): number {
|
|
259
|
+
pResOut.setInt32(0, 0, true)
|
|
260
|
+
return VFS.SQLITE_OK
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Returns page count and total stored bytes for diagnostics. */
|
|
264
|
+
getStats(): {
|
|
265
|
+
pageSize: number
|
|
266
|
+
totalPages: number
|
|
267
|
+
totalStoredBytes: number
|
|
268
|
+
} {
|
|
269
|
+
try {
|
|
270
|
+
const cursor = this.#sql.exec<{ total_pages: number; total_bytes: number }>(
|
|
271
|
+
'SELECT COUNT(*) AS total_pages, COALESCE(SUM(LENGTH(page_data)), 0) AS total_bytes FROM vfs_pages',
|
|
272
|
+
)
|
|
273
|
+
const stats = cursor.one()
|
|
274
|
+
|
|
275
|
+
return {
|
|
276
|
+
pageSize: PAGE_SIZE,
|
|
277
|
+
totalPages: stats.total_pages,
|
|
278
|
+
totalStoredBytes: stats.total_bytes,
|
|
279
|
+
}
|
|
280
|
+
} catch {
|
|
281
|
+
return {
|
|
282
|
+
pageSize: PAGE_SIZE,
|
|
283
|
+
totalPages: 0,
|
|
284
|
+
totalStoredBytes: 0,
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Convert a bare filename, path, or URL to a UNIX-style path.
|
|
291
|
+
*/
|
|
292
|
+
#getPath(nameOrURL: string | URL): string {
|
|
293
|
+
const url = typeof nameOrURL === 'string' ? new URL(nameOrURL, 'file://localhost/') : nameOrURL
|
|
294
|
+
return url.pathname
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
#resolveOpenPath(path: string | null): string {
|
|
298
|
+
const pathOrHandleId = path !== null && path !== '' ? path : Math.random().toString(36).slice(2)
|
|
299
|
+
return this.#getPath(pathOrHandleId)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
#getOpenFile(fileId: number): { path: string; flags: number } {
|
|
303
|
+
const openFile = this.#openFiles.get(fileId)
|
|
304
|
+
if (openFile === undefined) throw new Error(`Unknown fileId ${fileId}`)
|
|
305
|
+
return openFile
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
#assertCanonicalPageWrite(data: Uint8Array, offset: number) {
|
|
309
|
+
if (offset % PAGE_SIZE !== 0) {
|
|
310
|
+
throw new Error(
|
|
311
|
+
`CloudflareDurableObjectVFS expected page-aligned writes, got offset=${offset} for pageSize=${PAGE_SIZE}`,
|
|
312
|
+
)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
if (data.byteLength !== PAGE_SIZE) {
|
|
316
|
+
throw new Error(
|
|
317
|
+
`CloudflareDurableObjectVFS expected single-page writes of ${PAGE_SIZE} bytes, got ${data.byteLength}. Check that SQLite PRAGMA page_size matches the VFS page size.`,
|
|
318
|
+
)
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
#deleteFilePages(path: string) {
|
|
323
|
+
this.#sql.exec('DELETE FROM vfs_pages WHERE file_path = ?', path)
|
|
324
|
+
}
|
|
325
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { CfTypes } from '@livestore/common-cf'
|
|
2
2
|
import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
|
|
3
|
+
|
|
3
4
|
import { FacadeVFS } from '../FacadeVFS.ts'
|
|
4
5
|
|
|
5
6
|
const SECTOR_SIZE = 4096
|
|
@@ -106,7 +107,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
106
107
|
}
|
|
107
108
|
}
|
|
108
109
|
|
|
109
|
-
if (oldestKey) {
|
|
110
|
+
if (oldestKey !== '') {
|
|
110
111
|
this.#chunkCache.delete(oldestKey)
|
|
111
112
|
}
|
|
112
113
|
}
|
|
@@ -114,7 +115,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
114
115
|
#getCachedChunk(path: string, chunkIndex: number): Uint8Array | undefined {
|
|
115
116
|
const key = this.#getCacheKey(path, chunkIndex)
|
|
116
117
|
const entry = this.#chunkCache.get(key)
|
|
117
|
-
if (entry) {
|
|
118
|
+
if (entry !== undefined) {
|
|
118
119
|
entry.lastAccessed = Date.now()
|
|
119
120
|
return entry.data
|
|
120
121
|
}
|
|
@@ -142,7 +143,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
142
143
|
const key = `write:${path}`
|
|
143
144
|
|
|
144
145
|
// Cancel any pending write for this path
|
|
145
|
-
if (this.#writePromises.has(key)) {
|
|
146
|
+
if (this.#writePromises.has(key) === true) {
|
|
146
147
|
this.#pendingWrites.delete(key)
|
|
147
148
|
}
|
|
148
149
|
|
|
@@ -163,10 +164,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
163
164
|
|
|
164
165
|
async #loadMetadata(path: string): Promise<FileMetadata | undefined> {
|
|
165
166
|
const cached = this.#metadataCache.get(path)
|
|
166
|
-
if (cached) return cached
|
|
167
|
+
if (cached !== undefined) return cached
|
|
167
168
|
|
|
168
169
|
const metadata = await this.#storage.get<FileMetadata>(this.#getMetadataKey(path))
|
|
169
|
-
if (metadata) {
|
|
170
|
+
if (metadata !== undefined) {
|
|
170
171
|
this.#metadataCache.set(path, metadata)
|
|
171
172
|
}
|
|
172
173
|
return metadata
|
|
@@ -182,10 +183,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
182
183
|
|
|
183
184
|
async #loadChunk(path: string, chunkIndex: number): Promise<Uint8Array | undefined> {
|
|
184
185
|
const cached = this.#getCachedChunk(path, chunkIndex)
|
|
185
|
-
if (cached) return cached
|
|
186
|
+
if (cached !== undefined) return cached
|
|
186
187
|
|
|
187
188
|
const chunk = await this.#storage.get<Uint8Array>(this.#getChunkKey(path, chunkIndex))
|
|
188
|
-
if (chunk) {
|
|
189
|
+
if (chunk !== undefined) {
|
|
189
190
|
this.#setCachedChunk(path, chunkIndex, chunk)
|
|
190
191
|
}
|
|
191
192
|
return chunk
|
|
@@ -198,7 +199,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
198
199
|
|
|
199
200
|
async #deleteFile(path: string): Promise<void> {
|
|
200
201
|
const metadata = await this.#loadMetadata(path)
|
|
201
|
-
if (
|
|
202
|
+
if (metadata == null) return
|
|
202
203
|
|
|
203
204
|
// Delete all chunks and metadata atomically
|
|
204
205
|
const keysToDelete = [this.#getMetadataKey(path)]
|
|
@@ -220,12 +221,13 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
220
221
|
await this.#updateFileIndex()
|
|
221
222
|
}
|
|
222
223
|
|
|
223
|
-
jOpen(zName: string, fileId: number, flags: number, pOutFlags: DataView): number {
|
|
224
|
+
override jOpen(zName: string, fileId: number, flags: number, pOutFlags: DataView): number {
|
|
224
225
|
try {
|
|
225
|
-
const
|
|
226
|
+
const name = zName as unknown
|
|
227
|
+
const path = typeof name === 'string' && name !== '' ? this.#getPath(name) : Math.random().toString(36)
|
|
226
228
|
const metadata = this.#metadataCache.get(path)
|
|
227
229
|
|
|
228
|
-
if (
|
|
230
|
+
if (metadata == null && (flags & VFS.SQLITE_OPEN_CREATE) !== 0) {
|
|
229
231
|
// Create new file
|
|
230
232
|
if (this.#activeFiles.size >= this.#maxFiles) {
|
|
231
233
|
throw new Error('cannot create file: capacity exceeded')
|
|
@@ -246,7 +248,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
246
248
|
this.#scheduleWrite(path, () => this.#saveMetadata(path, newMetadata))
|
|
247
249
|
}
|
|
248
250
|
|
|
249
|
-
if (
|
|
251
|
+
if (this.#metadataCache.has(path) === false) {
|
|
250
252
|
throw new Error('file not found')
|
|
251
253
|
}
|
|
252
254
|
|
|
@@ -265,11 +267,11 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
265
267
|
}
|
|
266
268
|
}
|
|
267
269
|
|
|
268
|
-
jClose(fileId: number): number {
|
|
270
|
+
override jClose(fileId: number): number {
|
|
269
271
|
const handle = this.#openFiles.get(fileId)
|
|
270
|
-
if (handle) {
|
|
272
|
+
if (handle !== undefined) {
|
|
271
273
|
this.#openFiles.delete(fileId)
|
|
272
|
-
if (handle.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
|
|
274
|
+
if ((handle.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) !== 0) {
|
|
273
275
|
// Schedule async delete
|
|
274
276
|
this.#scheduleWrite(handle.path, () => this.#deleteFile(handle.path))
|
|
275
277
|
}
|
|
@@ -277,10 +279,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
277
279
|
return VFS.SQLITE_OK
|
|
278
280
|
}
|
|
279
281
|
|
|
280
|
-
jRead(fileId: number, pData: Uint8Array, iOffset: number): number {
|
|
282
|
+
override jRead(fileId: number, pData: Uint8Array, iOffset: number): number {
|
|
281
283
|
try {
|
|
282
284
|
const handle = this.#openFiles.get(fileId)
|
|
283
|
-
if (
|
|
285
|
+
if (handle == null) return VFS.SQLITE_IOERR
|
|
284
286
|
|
|
285
287
|
const fileSize = handle.metadata.size
|
|
286
288
|
const requestedBytes = pData.byteLength
|
|
@@ -303,7 +305,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
303
305
|
|
|
304
306
|
for (let chunkIndex = startChunk; chunkIndex <= endChunk; chunkIndex++) {
|
|
305
307
|
const chunk = this.#getCachedChunk(handle.path, chunkIndex)
|
|
306
|
-
if (
|
|
308
|
+
if (chunk == null) {
|
|
307
309
|
// Cache miss - this is a problem for synchronous operation
|
|
308
310
|
// We should have preloaded chunks during initialization
|
|
309
311
|
console.warn(`Cache miss for chunk ${chunkIndex} of ${handle.path}`)
|
|
@@ -338,10 +340,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
338
340
|
}
|
|
339
341
|
}
|
|
340
342
|
|
|
341
|
-
jWrite(fileId: number, pData: Uint8Array, iOffset: number): number {
|
|
343
|
+
override jWrite(fileId: number, pData: Uint8Array, iOffset: number): number {
|
|
342
344
|
try {
|
|
343
345
|
const handle = this.#openFiles.get(fileId)
|
|
344
|
-
if (
|
|
346
|
+
if (handle == null) return VFS.SQLITE_IOERR
|
|
345
347
|
|
|
346
348
|
const bytesToWrite = pData.byteLength
|
|
347
349
|
const startChunk = Math.floor(iOffset / CHUNK_SIZE)
|
|
@@ -356,7 +358,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
356
358
|
const writeEnd = Math.min(CHUNK_SIZE, iOffset + bytesToWrite - chunkOffset)
|
|
357
359
|
|
|
358
360
|
let chunk = this.#getCachedChunk(handle.path, chunkIndex)
|
|
359
|
-
if (
|
|
361
|
+
if (chunk == null) {
|
|
360
362
|
// Create new chunk
|
|
361
363
|
chunk = new Uint8Array(CHUNK_SIZE)
|
|
362
364
|
} else {
|
|
@@ -399,10 +401,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
399
401
|
}
|
|
400
402
|
}
|
|
401
403
|
|
|
402
|
-
jTruncate(fileId: number, iSize: number): number {
|
|
404
|
+
override jTruncate(fileId: number, iSize: number): number {
|
|
403
405
|
try {
|
|
404
406
|
const handle = this.#openFiles.get(fileId)
|
|
405
|
-
if (
|
|
407
|
+
if (handle == null) return VFS.SQLITE_IOERR
|
|
406
408
|
|
|
407
409
|
// const oldSize = handle.metadata.size
|
|
408
410
|
const newChunkCount = Math.ceil(iSize / CHUNK_SIZE)
|
|
@@ -435,7 +437,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
435
437
|
|
|
436
438
|
if (lastChunkSize < CHUNK_SIZE) {
|
|
437
439
|
const lastChunk = this.#getCachedChunk(handle.path, lastChunkIndex)
|
|
438
|
-
if (lastChunk) {
|
|
440
|
+
if (lastChunk !== undefined) {
|
|
439
441
|
const truncatedChunk = new Uint8Array(CHUNK_SIZE)
|
|
440
442
|
truncatedChunk.set(lastChunk.subarray(0, lastChunkSize))
|
|
441
443
|
this.#setCachedChunk(handle.path, lastChunkIndex, truncatedChunk)
|
|
@@ -454,10 +456,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
454
456
|
}
|
|
455
457
|
}
|
|
456
458
|
|
|
457
|
-
jSync(fileId: number, _flags: number): number {
|
|
459
|
+
override jSync(fileId: number, _flags: number): number {
|
|
458
460
|
try {
|
|
459
461
|
const handle = this.#openFiles.get(fileId)
|
|
460
|
-
if (
|
|
462
|
+
if (handle == null) return VFS.SQLITE_IOERR
|
|
461
463
|
|
|
462
464
|
// Force sync all pending writes for this file
|
|
463
465
|
// Note: DurableObjectStorage operations are already synchronous
|
|
@@ -469,10 +471,10 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
469
471
|
}
|
|
470
472
|
}
|
|
471
473
|
|
|
472
|
-
jFileSize(fileId: number, pSize64: DataView): number {
|
|
474
|
+
override jFileSize(fileId: number, pSize64: DataView): number {
|
|
473
475
|
try {
|
|
474
476
|
const handle = this.#openFiles.get(fileId)
|
|
475
|
-
if (
|
|
477
|
+
if (handle == null) return VFS.SQLITE_IOERR
|
|
476
478
|
|
|
477
479
|
pSize64.setBigInt64(0, BigInt(handle.metadata.size), true)
|
|
478
480
|
return VFS.SQLITE_OK
|
|
@@ -482,19 +484,19 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
482
484
|
}
|
|
483
485
|
}
|
|
484
486
|
|
|
485
|
-
jSectorSize(_fileId: number): number {
|
|
487
|
+
override jSectorSize(_fileId: number): number {
|
|
486
488
|
return SECTOR_SIZE
|
|
487
489
|
}
|
|
488
490
|
|
|
489
|
-
jDeviceCharacteristics(_fileId: number): number {
|
|
491
|
+
override jDeviceCharacteristics(_fileId: number): number {
|
|
490
492
|
return VFS.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN
|
|
491
493
|
}
|
|
492
494
|
|
|
493
|
-
jAccess(zName: string, _flags: number, pResOut: DataView): number {
|
|
495
|
+
override jAccess(zName: string, _flags: number, pResOut: DataView): number {
|
|
494
496
|
try {
|
|
495
497
|
const path = this.#getPath(zName)
|
|
496
498
|
const exists = this.#activeFiles.has(path)
|
|
497
|
-
pResOut.setInt32(0, exists ? 1 : 0, true)
|
|
499
|
+
pResOut.setInt32(0, exists === true ? 1 : 0, true)
|
|
498
500
|
return VFS.SQLITE_OK
|
|
499
501
|
} catch (e: any) {
|
|
500
502
|
console.error('jAccess error:', e.message)
|
|
@@ -502,7 +504,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
502
504
|
}
|
|
503
505
|
}
|
|
504
506
|
|
|
505
|
-
jDelete(zName: string, _syncDir: number): number {
|
|
507
|
+
override jDelete(zName: string, _syncDir: number): number {
|
|
506
508
|
try {
|
|
507
509
|
const path = this.#getPath(zName)
|
|
508
510
|
|
|
@@ -526,7 +528,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
526
528
|
}
|
|
527
529
|
|
|
528
530
|
async isReady() {
|
|
529
|
-
if (
|
|
531
|
+
if (this.#initialized === false) {
|
|
530
532
|
await this.#initializeStorage()
|
|
531
533
|
this.#initialized = true
|
|
532
534
|
}
|
|
@@ -536,7 +538,7 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
536
538
|
async #initializeStorage() {
|
|
537
539
|
// Load list of existing files
|
|
538
540
|
const fileList = await this.#storage.get<string[]>('index:files')
|
|
539
|
-
if (fileList) {
|
|
541
|
+
if (fileList !== undefined) {
|
|
540
542
|
for (const path of fileList) {
|
|
541
543
|
this.#activeFiles.add(path)
|
|
542
544
|
// Preload metadata for all files
|
|
@@ -557,8 +559,8 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
557
559
|
for (const path of this.#activeFiles) {
|
|
558
560
|
const metadata = this.#metadataCache.get(path)
|
|
559
561
|
if (
|
|
560
|
-
metadata &&
|
|
561
|
-
(metadata.flags & VFS.SQLITE_OPEN_DELETEONCLOSE || (metadata.flags & PERSISTENT_FILE_TYPES) === 0)
|
|
562
|
+
metadata !== undefined &&
|
|
563
|
+
((metadata.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) !== 0 || (metadata.flags & PERSISTENT_FILE_TYPES) === 0)
|
|
562
564
|
) {
|
|
563
565
|
console.warn(`Cleaning up temporary file: ${path}`)
|
|
564
566
|
await this.#deleteFile(path)
|
|
@@ -615,13 +617,13 @@ export class CloudflareWorkerVFS extends FacadeVFS {
|
|
|
615
617
|
*/
|
|
616
618
|
async #preloadChunks(path: string, startChunk: number, count = 3) {
|
|
617
619
|
const metadata = this.#metadataCache.get(path)
|
|
618
|
-
if (
|
|
620
|
+
if (metadata == null) return
|
|
619
621
|
|
|
620
622
|
const endChunk = Math.min(startChunk + count, metadata.chunkCount)
|
|
621
623
|
const promises: Promise<void>[] = []
|
|
622
624
|
|
|
623
625
|
for (let i = startChunk; i < endChunk; i++) {
|
|
624
|
-
if (
|
|
626
|
+
if (this.#getCachedChunk(path, i) == null) {
|
|
625
627
|
promises.push(this.#loadChunk(path, i).then(() => {}))
|
|
626
628
|
}
|
|
627
629
|
}
|
package/src/cf/README.md
CHANGED
|
@@ -37,7 +37,7 @@ CREATE INDEX IF NOT EXISTS idx_vfs_blocks_range ON vfs_blocks(file_path, block_i
|
|
|
37
37
|
CREATE INDEX IF NOT EXISTS idx_vfs_files_modified ON vfs_files(modified_at);
|
|
38
38
|
|
|
39
39
|
-- Trigger to update modified_at timestamp when file size changes
|
|
40
|
-
CREATE TRIGGER IF NOT EXISTS trg_vfs_files_update_modified
|
|
40
|
+
CREATE TRIGGER IF NOT EXISTS trg_vfs_files_update_modified
|
|
41
41
|
AFTER UPDATE OF file_size ON vfs_files
|
|
42
42
|
BEGIN
|
|
43
43
|
UPDATE vfs_files SET modified_at = unixepoch() WHERE file_path = NEW.file_path;
|
|
@@ -45,7 +45,7 @@ CREATE TRIGGER IF NOT EXISTS trg_vfs_files_update_modified
|
|
|
45
45
|
|
|
46
46
|
-- View for file statistics (useful for debugging and monitoring)
|
|
47
47
|
CREATE VIEW IF NOT EXISTS vfs_file_stats AS
|
|
48
|
-
SELECT
|
|
48
|
+
SELECT
|
|
49
49
|
f.file_path,
|
|
50
50
|
f.file_size,
|
|
51
51
|
f.flags,
|
|
@@ -57,4 +57,4 @@ SELECT
|
|
|
57
57
|
FROM vfs_files f
|
|
58
58
|
LEFT JOIN vfs_blocks b ON f.file_path = b.file_path
|
|
59
59
|
GROUP BY f.file_path, f.file_size, f.flags, f.created_at, f.modified_at;
|
|
60
|
-
```
|
|
60
|
+
```
|
package/src/cf/mod.ts
CHANGED
|
@@ -5,12 +5,16 @@ import type { CfTypes } from '@livestore/common-cf'
|
|
|
5
5
|
import { Effect } from '@livestore/utils/effect'
|
|
6
6
|
import type * as WaSqlite from '@livestore/wa-sqlite'
|
|
7
7
|
import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
|
|
8
|
+
|
|
8
9
|
import { makeInMemoryDb } from '../in-memory-vfs.ts'
|
|
9
10
|
import { makeSqliteDb } from '../make-sqlite-db.ts'
|
|
10
|
-
import {
|
|
11
|
+
import { CloudflareDurableObjectVFS } from './CloudflareDurableObjectVFS.ts'
|
|
11
12
|
|
|
12
|
-
export {
|
|
13
|
-
|
|
13
|
+
export {
|
|
14
|
+
CloudflareDurableObjectVFS,
|
|
15
|
+
PAGE_SIZE as CF_SQL_VFS_PAGE_SIZE,
|
|
16
|
+
REQUIRED_PRAGMAS as CF_SQL_VFS_REQUIRED_PRAGMAS,
|
|
17
|
+
} from './CloudflareDurableObjectVFS.ts'
|
|
14
18
|
export { CloudflareWorkerVFS } from './CloudflareWorkerVFS.ts'
|
|
15
19
|
|
|
16
20
|
export type CloudflareDatabaseMetadataInMemory = {
|
|
@@ -24,8 +28,7 @@ export type CloudflareDatabaseMetadataInMemory = {
|
|
|
24
28
|
|
|
25
29
|
export type CloudflareDatabaseMetadataFs = {
|
|
26
30
|
_tag: 'storage'
|
|
27
|
-
|
|
28
|
-
vfs: CloudflareSqlVFS
|
|
31
|
+
vfs: CloudflareDurableObjectVFS
|
|
29
32
|
dbPointer: number
|
|
30
33
|
persistenceInfo: PersistenceInfo
|
|
31
34
|
deleteDb: () => void
|
|
@@ -115,18 +118,10 @@ const makeCloudflareFsDb = ({
|
|
|
115
118
|
// If this is becoming a problem, we can use a hashed version of the directory name
|
|
116
119
|
const vfsName = `cf-do-sqlite-${fileName}`
|
|
117
120
|
if (sqlite3.vfs_registered.has(vfsName) === false) {
|
|
118
|
-
|
|
119
|
-
// const nodeFsVfs = new CloudflareWorkerVFS(vfsName, storage, (sqlite3 as any).module)
|
|
120
|
-
const nodeFsVfs = new CloudflareSqlVFS(vfsName, storage.sql, (sqlite3 as any).module)
|
|
121
|
-
|
|
122
|
-
// Initialize the VFS schema before registering it
|
|
123
|
-
const isReady = yield* Effect.promise(() => nodeFsVfs.isReady())
|
|
124
|
-
if (!isReady) {
|
|
125
|
-
throw new Error(`Failed to initialize CloudflareSqlVFS for ${vfsName}`)
|
|
126
|
-
}
|
|
121
|
+
const vfs = new CloudflareDurableObjectVFS(vfsName, storage.sql, (sqlite3 as any).module)
|
|
127
122
|
|
|
128
123
|
// @ts-expect-error TODO fix types
|
|
129
|
-
sqlite3.vfs_register(
|
|
124
|
+
sqlite3.vfs_register(vfs, false)
|
|
130
125
|
}
|
|
131
126
|
|
|
132
127
|
// yield* fs.makeDirectory(directory, { recursive: true })
|