@threadedstack/wmd-web 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/RELEASE.json +39 -54
- package/guest/arm64/Image +2 -2
- package/guest/arm64/Image-4k +1 -1
- package/guest/arm64/initramfs.cpio.gz +0 -0
- package/guest/arm64/vm.wasm +0 -0
- package/package.json +4 -4
- package/relay/hosted.ts +27 -11
- package/relay/main.ts +3 -3
- package/src/boot.js +25 -0
- package/src/build/builder.js +791 -0
- package/src/build/context.js +349 -0
- package/src/build/dockerfile.js +1055 -0
- package/src/build/dockerignore.js +270 -0
- package/src/build/fast.js +1187 -0
- package/src/build/inspect.js +78 -0
- package/src/build/pin.js +6 -0
- package/src/build/tar.js +304 -0
- package/src/host/ctl.js +17 -0
- package/src/host/disk-map.js +330 -0
- package/src/host/disk-worker.js +125 -0
- package/src/host/disk.js +219 -0
- package/src/host/wmd-imports.js +3 -3
- package/src/image-files.js +2 -1
- package/src/registry/hub.js +113 -44
- package/src/registry/load-tar.js +473 -0
- package/src/registry/pull.js +5 -3
- package/src/worker.js +2 -0
- package/wmdpack.wasm +0 -0
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
// A thin disk in two OPFS files (docs/plans/web-buildkit-impl.md 5.9).
|
|
2
|
+
//
|
|
3
|
+
// vm.wasm's virtio-blk device (guest/vm/src/dev/blk.c) reads and writes a disk of a fixed size through
|
|
4
|
+
// wmd.blk_rw. Firefox charges a file's whole length against the quota even where nothing was written, so
|
|
5
|
+
// the disk is not one file of its size. The guest's disk is cut into 64 KiB extents; the data file holds
|
|
6
|
+
// only the extents the guest wrote, packed into slots, and the map file says which slot holds which
|
|
7
|
+
// extent:
|
|
8
|
+
//
|
|
9
|
+
// map file header (64 bytes): 'WMDDISK1', u32 version, u32 extent shift, f64 disk bytes, u32 crc32
|
|
10
|
+
// of bytes 0..23; then one 8-byte record per extent: u32 slot+1 (0: never written or
|
|
11
|
+
// discarded, reads as zeros), u32 crc32 of (extent index, slot+1)
|
|
12
|
+
// data file slot s at s * 64 KiB
|
|
13
|
+
//
|
|
14
|
+
// Order (a disk write cache; the guest's flush is the commit): a write puts its data in the data file and
|
|
15
|
+
// updates the in-memory map only; the map records of the writes and discards since the last flush are written
|
|
16
|
+
// at the flush: flush the data file, write the records, flush the map file, then shrink the data file past
|
|
17
|
+
// the last live slot. A crash before a flush loses the writes since the last flush, as a disk's write cache
|
|
18
|
+
// does. A record whose crc does not match (torn by a crash) is dropped on open, so its extent reads as zeros.
|
|
19
|
+
// Because no record names a slot before the data in it is durable, a slot freed by a discard can be taken
|
|
20
|
+
// by another extent at once: a stale record can only point at bytes written for another extent that was
|
|
21
|
+
// itself discarded. ENOSPC comes only when every slot is held by a live extent.
|
|
22
|
+
//
|
|
23
|
+
// A handle is an OPFS FileSystemSyncAccessHandle or anything with the same read/write/getSize/truncate/
|
|
24
|
+
// flush (the unit tests pass a fake).
|
|
25
|
+
|
|
26
|
+
export class DiskError extends Error {
|
|
27
|
+
constructor(message, reason) {
|
|
28
|
+
super(message)
|
|
29
|
+
this.name = 'DiskError'
|
|
30
|
+
this.reason = reason // 'locked' | 'quota' | 'open' | 'corrupt' | 'timeout'
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export const EXT_SHIFT = 16
|
|
35
|
+
export const EXT = 1 << EXT_SHIFT
|
|
36
|
+
export const MAP_HEADER = 64
|
|
37
|
+
const REC = 8
|
|
38
|
+
const MAGIC = 'WMDDISK1'
|
|
39
|
+
const VERSION = 1
|
|
40
|
+
|
|
41
|
+
const CRC_TABLE = (() => {
|
|
42
|
+
const t = new Uint32Array(256)
|
|
43
|
+
for (let n = 0; n < 256; n++) {
|
|
44
|
+
let c = n
|
|
45
|
+
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1
|
|
46
|
+
t[n] = c >>> 0
|
|
47
|
+
}
|
|
48
|
+
return t
|
|
49
|
+
})()
|
|
50
|
+
|
|
51
|
+
export function crc32(bytes, crc = 0) {
|
|
52
|
+
let c = ~crc >>> 0
|
|
53
|
+
for (let i = 0; i < bytes.length; i++) c = CRC_TABLE[(c ^ bytes[i]) & 0xff] ^ (c >>> 8)
|
|
54
|
+
return ~c >>> 0
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function recCrc(index, slot1) {
|
|
58
|
+
const b = new Uint8Array(8)
|
|
59
|
+
const v = new DataView(b.buffer)
|
|
60
|
+
v.setUint32(0, index, true)
|
|
61
|
+
v.setUint32(4, slot1, true)
|
|
62
|
+
return crc32(b)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const ENOSPC = -28
|
|
66
|
+
const EIO = -5
|
|
67
|
+
const EINVAL = -22
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* True when createSyncAccessHandle failed because another holder has the file open: Chromium and Firefox throw
|
|
71
|
+
* NoModificationAllowedError, WebKit throws InvalidStateError.
|
|
72
|
+
*/
|
|
73
|
+
export function heldElsewhere(err) {
|
|
74
|
+
return err?.name === 'NoModificationAllowedError' || err?.name === 'InvalidStateError'
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The errno a failed OPFS call stands for: ENOSPC when the quota ran out, else EIO. */
|
|
78
|
+
export function errnoOf(err) {
|
|
79
|
+
return err?.name === 'QuotaExceededError' ? ENOSPC : EIO
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export class DiskMap {
|
|
83
|
+
/**
|
|
84
|
+
* Opens the disk in `data` and `map`, or makes a new one of `bytes` (a multiple of 64 KiB) when the data
|
|
85
|
+
* file is empty and the map has no valid header. A bad header over a non-empty data file throws DiskError
|
|
86
|
+
* 'corrupt' (the data is not wiped). An existing disk keeps the size it was made with; `bytes` is then ignored.
|
|
87
|
+
* @returns {DiskMap} with .created and .dropped (records dropped for a bad crc or a slot claimed twice)
|
|
88
|
+
*/
|
|
89
|
+
static open(data, map, bytes) {
|
|
90
|
+
const d = new DiskMap(data, map)
|
|
91
|
+
const hdr = new Uint8Array(MAP_HEADER)
|
|
92
|
+
const got = map.getSize() >= MAP_HEADER ? map.read(hdr, { at: 0 }) : 0
|
|
93
|
+
const hv = new DataView(hdr.buffer)
|
|
94
|
+
const valid =
|
|
95
|
+
got === MAP_HEADER &&
|
|
96
|
+
new TextDecoder().decode(hdr.subarray(0, 8)) === MAGIC &&
|
|
97
|
+
hv.getUint32(8, true) === VERSION &&
|
|
98
|
+
hv.getUint32(12, true) === EXT_SHIFT &&
|
|
99
|
+
hv.getUint32(24, true) === crc32(hdr.subarray(0, 24))
|
|
100
|
+
if (valid) d.#load(hv.getFloat64(16, true))
|
|
101
|
+
else if (data.getSize() > 0) throw new DiskError("the disk's map is damaged (bad header over existing data); delete the disk with deleteDisk to start over", 'corrupt')
|
|
102
|
+
else d.#create(bytes)
|
|
103
|
+
return d
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
constructor(data, map) {
|
|
107
|
+
this.data = data
|
|
108
|
+
this.map = map
|
|
109
|
+
this.created = false
|
|
110
|
+
this.dropped = 0
|
|
111
|
+
this.dirty = new Set() // extents whose record changed since the last flush
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
#create(bytes) {
|
|
115
|
+
if (!(bytes > 0) || bytes % EXT) throw new RangeError(`disk size ${bytes} is not a positive multiple of ${EXT}`)
|
|
116
|
+
this.bytes = bytes
|
|
117
|
+
this.n = bytes / EXT
|
|
118
|
+
this.slots = new Uint32Array(this.n) // extent -> slot+1
|
|
119
|
+
this.used = new Uint8Array(this.n) // slot -> 1 when a live extent holds it
|
|
120
|
+
this.lowFree = 0
|
|
121
|
+
this.top = 0 // slots in the data file
|
|
122
|
+
this.data.truncate(0)
|
|
123
|
+
this.map.truncate(0)
|
|
124
|
+
this.map.truncate(MAP_HEADER + this.n * REC) // zeros: every extent unwritten
|
|
125
|
+
const hdr = new Uint8Array(MAP_HEADER)
|
|
126
|
+
hdr.set(new TextEncoder().encode(MAGIC))
|
|
127
|
+
const hv = new DataView(hdr.buffer)
|
|
128
|
+
hv.setUint32(8, VERSION, true)
|
|
129
|
+
hv.setUint32(12, EXT_SHIFT, true)
|
|
130
|
+
hv.setFloat64(16, bytes, true)
|
|
131
|
+
hv.setUint32(24, crc32(hdr.subarray(0, 24)), true)
|
|
132
|
+
this.map.write(hdr, { at: 0 })
|
|
133
|
+
this.map.flush()
|
|
134
|
+
this.data.flush()
|
|
135
|
+
this.created = true
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
#load(bytes) {
|
|
139
|
+
this.bytes = bytes
|
|
140
|
+
this.n = Math.floor(bytes / EXT)
|
|
141
|
+
this.slots = new Uint32Array(this.n)
|
|
142
|
+
this.used = new Uint8Array(this.n)
|
|
143
|
+
const recs = new Uint8Array(this.n * REC)
|
|
144
|
+
const got = this.map.read(recs, { at: MAP_HEADER })
|
|
145
|
+
const rv = new DataView(recs.buffer)
|
|
146
|
+
const fix = []
|
|
147
|
+
for (let i = 0; i < this.n; i++) {
|
|
148
|
+
if ((i + 1) * REC > got) break
|
|
149
|
+
const slot1 = rv.getUint32(i * REC, true)
|
|
150
|
+
if (slot1 === 0) continue
|
|
151
|
+
const ok = rv.getUint32(i * REC + 4, true) === recCrc(i, slot1) && slot1 <= this.n && !this.used[slot1 - 1]
|
|
152
|
+
if (!ok) {
|
|
153
|
+
this.dropped++
|
|
154
|
+
fix.push(i)
|
|
155
|
+
continue
|
|
156
|
+
}
|
|
157
|
+
this.slots[i] = slot1
|
|
158
|
+
this.used[slot1 - 1] = 1
|
|
159
|
+
}
|
|
160
|
+
for (const i of fix) this.#writeRec(i, 0)
|
|
161
|
+
if (fix.length) this.map.flush()
|
|
162
|
+
this.lowFree = 0
|
|
163
|
+
this.top = 0
|
|
164
|
+
for (let s = this.n - 1; s >= 0; s--)
|
|
165
|
+
if (this.used[s]) {
|
|
166
|
+
this.top = s + 1
|
|
167
|
+
break
|
|
168
|
+
}
|
|
169
|
+
this.#shrink()
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
#writeRec(i, slot1) {
|
|
173
|
+
const b = new Uint8Array(REC)
|
|
174
|
+
const v = new DataView(b.buffer)
|
|
175
|
+
v.setUint32(0, slot1, true)
|
|
176
|
+
v.setUint32(4, slot1 ? recCrc(i, slot1) : 0, true)
|
|
177
|
+
if (this.map.write(b, { at: MAP_HEADER + i * REC }) !== REC) throw Object.assign(new Error('short map write'), { errno: ENOSPC })
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
#release(s) {
|
|
181
|
+
this.used[s] = 0
|
|
182
|
+
if (s < this.lowFree) this.lowFree = s
|
|
183
|
+
while (this.top > 0 && !this.used[this.top - 1]) this.top--
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
#alloc() {
|
|
187
|
+
let s = this.lowFree
|
|
188
|
+
while (s < this.n && this.used[s]) s++
|
|
189
|
+
if (s >= this.n) throw Object.assign(new Error('no free slot'), { errno: ENOSPC }) // cannot happen: n slots for n extents
|
|
190
|
+
this.used[s] = 1
|
|
191
|
+
this.lowFree = s + 1
|
|
192
|
+
if (s + 1 > this.top) this.top = s + 1
|
|
193
|
+
return s
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
#check(off, len) {
|
|
197
|
+
return Number.isInteger(off) && off >= 0 && len >= 0 && off + len <= this.bytes
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** The disk's [off, off+len) into buf. 0 or a negative errno. */
|
|
201
|
+
read(off, buf) {
|
|
202
|
+
if (!this.#check(off, buf.length)) return EINVAL
|
|
203
|
+
try {
|
|
204
|
+
for (let done = 0; done < buf.length; ) {
|
|
205
|
+
const pos = off + done
|
|
206
|
+
const i = Math.floor(pos / EXT)
|
|
207
|
+
const within = pos - i * EXT
|
|
208
|
+
const n = Math.min(EXT - within, buf.length - done)
|
|
209
|
+
const part = buf.subarray(done, done + n)
|
|
210
|
+
const slot1 = this.slots[i]
|
|
211
|
+
if (!slot1) part.fill(0)
|
|
212
|
+
else {
|
|
213
|
+
const got = this.data.read(part, { at: (slot1 - 1) * EXT + within })
|
|
214
|
+
if (got < n) part.fill(0, got)
|
|
215
|
+
}
|
|
216
|
+
done += n
|
|
217
|
+
}
|
|
218
|
+
return 0
|
|
219
|
+
} catch (e) {
|
|
220
|
+
return e.errno || errnoOf(e)
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** buf onto the disk at off. 0 or a negative errno. */
|
|
225
|
+
write(off, buf) {
|
|
226
|
+
if (!this.#check(off, buf.length)) return EINVAL
|
|
227
|
+
try {
|
|
228
|
+
for (let done = 0; done < buf.length; ) {
|
|
229
|
+
const pos = off + done
|
|
230
|
+
const i = Math.floor(pos / EXT)
|
|
231
|
+
const within = pos - i * EXT
|
|
232
|
+
const n = Math.min(EXT - within, buf.length - done)
|
|
233
|
+
const part = buf.subarray(done, done + n)
|
|
234
|
+
let slot1 = this.slots[i]
|
|
235
|
+
if (slot1) {
|
|
236
|
+
if (this.data.write(part, { at: (slot1 - 1) * EXT + within }) !== n) return ENOSPC
|
|
237
|
+
} else {
|
|
238
|
+
const s = this.#alloc()
|
|
239
|
+
// a new slot may hold an old extent's bytes: write the whole extent, zeros around the data
|
|
240
|
+
let whole = part
|
|
241
|
+
if (n !== EXT) {
|
|
242
|
+
whole = new Uint8Array(EXT)
|
|
243
|
+
whole.set(part, within)
|
|
244
|
+
}
|
|
245
|
+
let wrote
|
|
246
|
+
try {
|
|
247
|
+
wrote = this.data.write(whole, { at: s * EXT })
|
|
248
|
+
} catch (e) {
|
|
249
|
+
wrote = e
|
|
250
|
+
}
|
|
251
|
+
if (wrote !== EXT) {
|
|
252
|
+
this.#release(s)
|
|
253
|
+
return typeof wrote === 'number' ? ENOSPC : wrote.errno || errnoOf(wrote)
|
|
254
|
+
}
|
|
255
|
+
this.slots[i] = s + 1
|
|
256
|
+
this.dirty.add(i)
|
|
257
|
+
}
|
|
258
|
+
done += n
|
|
259
|
+
}
|
|
260
|
+
return 0
|
|
261
|
+
} catch (e) {
|
|
262
|
+
return e.errno || errnoOf(e)
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Forgets every whole extent in [off, off+len); parts of extents are left as they are. */
|
|
267
|
+
discard(off, len) {
|
|
268
|
+
if (!this.#check(off, len)) return EINVAL
|
|
269
|
+
try {
|
|
270
|
+
const first = Math.ceil(off / EXT)
|
|
271
|
+
const end = Math.floor((off + len) / EXT)
|
|
272
|
+
for (let i = first; i < end; i++) {
|
|
273
|
+
const slot1 = this.slots[i]
|
|
274
|
+
if (!slot1) continue
|
|
275
|
+
this.slots[i] = 0
|
|
276
|
+
this.dirty.add(i)
|
|
277
|
+
this.used[slot1 - 1] = 0 // reusable at once: its record is rewritten at the flush
|
|
278
|
+
if (slot1 - 1 < this.lowFree) this.lowFree = slot1 - 1
|
|
279
|
+
}
|
|
280
|
+
return 0
|
|
281
|
+
} catch (e) {
|
|
282
|
+
return e.errno || errnoOf(e)
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Makes every earlier write durable: the data file, then the map records, then the map file; then shrinks. */
|
|
287
|
+
flush() {
|
|
288
|
+
try {
|
|
289
|
+
this.data.flush()
|
|
290
|
+
const idx = [...this.dirty].sort((a, b) => a - b)
|
|
291
|
+
for (let k = 0; k < idx.length; ) {
|
|
292
|
+
let e = k + 1
|
|
293
|
+
while (e < idx.length && idx[e] === idx[e - 1] + 1) e++
|
|
294
|
+
this.#writeRecs(idx[k], idx.slice(k, e))
|
|
295
|
+
k = e
|
|
296
|
+
}
|
|
297
|
+
this.map.flush()
|
|
298
|
+
this.dirty.clear()
|
|
299
|
+
this.#shrink()
|
|
300
|
+
return 0
|
|
301
|
+
} catch (e) {
|
|
302
|
+
return e.errno || errnoOf(e) // the dirty set stays: the next flush retries
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
#writeRecs(first, list) {
|
|
307
|
+
const b = new Uint8Array(list.length * REC)
|
|
308
|
+
const v = new DataView(b.buffer)
|
|
309
|
+
list.forEach((i, k) => {
|
|
310
|
+
const slot1 = this.slots[i]
|
|
311
|
+
v.setUint32(k * REC, slot1, true)
|
|
312
|
+
v.setUint32(k * REC + 4, slot1 ? recCrc(i, slot1) : 0, true)
|
|
313
|
+
})
|
|
314
|
+
if (this.map.write(b, { at: MAP_HEADER + first * REC }) !== b.length) throw Object.assign(new Error('short map write'), { errno: ENOSPC })
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
#shrink() {
|
|
318
|
+
let top = this.top
|
|
319
|
+
while (top > 0 && !this.used[top - 1]) top--
|
|
320
|
+
this.top = top
|
|
321
|
+
if (this.data.getSize() > top * EXT) this.data.truncate(top * EXT)
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Bytes the data file holds and the extents mapped. */
|
|
325
|
+
usage() {
|
|
326
|
+
let mapped = 0
|
|
327
|
+
for (let i = 0; i < this.n; i++) if (this.slots[i]) mapped++
|
|
328
|
+
return { bytes: this.bytes, fileBytes: this.data.getSize(), mapped, slots: this.top }
|
|
329
|
+
}
|
|
330
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// The disk Worker (disk.js): owns one disk's two OPFS files through sync access handles and serves the
|
|
2
|
+
// vCPUs' wmd.blk_rw requests from the mailbox. After 'ready' it runs a loop with no event loop, so the page
|
|
3
|
+
// talks to it only through the mailbox (control word i32[5]).
|
|
4
|
+
|
|
5
|
+
import { DiskMap, errnoOf, heldElsewhere } from './disk-map.js'
|
|
6
|
+
import { DATA, HEADER, OP, STATE } from './disk.js'
|
|
7
|
+
|
|
8
|
+
const MiB = 2 ** 20
|
|
9
|
+
|
|
10
|
+
self.onmessage = async e => {
|
|
11
|
+
if (e.data?.t !== 'open') return
|
|
12
|
+
const { mailbox, name, bytes: asked, minBytes } = e.data
|
|
13
|
+
const i32 = new Int32Array(mailbox, 0, 8)
|
|
14
|
+
const fail = (reason, message) => {
|
|
15
|
+
Atomics.store(i32, 1, STATE.GONE)
|
|
16
|
+
Atomics.notify(i32, 1)
|
|
17
|
+
self.postMessage({ t: 'fail', reason, message })
|
|
18
|
+
}
|
|
19
|
+
let data
|
|
20
|
+
let map
|
|
21
|
+
let disk
|
|
22
|
+
let est = null
|
|
23
|
+
let smaller = false
|
|
24
|
+
try {
|
|
25
|
+
const root = await navigator.storage.getDirectory()
|
|
26
|
+
est = await navigator.storage.estimate().catch(() => null)
|
|
27
|
+
const handle = async n => (await root.getFileHandle(n, { create: true })).createSyncAccessHandle()
|
|
28
|
+
try {
|
|
29
|
+
data = await handle(`${name}.disk`)
|
|
30
|
+
map = await handle(`${name}.map`)
|
|
31
|
+
} catch (err) {
|
|
32
|
+
data?.close()
|
|
33
|
+
return fail(heldElsewhere(err) ? 'locked' : 'open', `${name}: ${err?.name || ''} ${err?.message || err}`)
|
|
34
|
+
}
|
|
35
|
+
let bytes = asked
|
|
36
|
+
const fresh = map.getSize() === 0
|
|
37
|
+
if (fresh && est?.quota) {
|
|
38
|
+
// the origin's free room: its quota less its usage, plus what this disk's own files already count; a tenth kept back
|
|
39
|
+
const room = Math.floor(((est.quota - (est.usage || 0) + data.getSize() + map.getSize()) * 0.9) / MiB) * MiB
|
|
40
|
+
if (room < bytes) {
|
|
41
|
+
bytes = room
|
|
42
|
+
smaller = true
|
|
43
|
+
}
|
|
44
|
+
if (room < minBytes) {
|
|
45
|
+
data.close()
|
|
46
|
+
map.close()
|
|
47
|
+
return fail('quota', `${name}: the origin has room for ${Math.round(room / MiB)} MiB, less than ${Math.round(minBytes / MiB)} MiB`)
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
disk = DiskMap.open(data, map, bytes)
|
|
51
|
+
} catch (err) {
|
|
52
|
+
data?.close()
|
|
53
|
+
map?.close()
|
|
54
|
+
return fail(err?.reason === 'corrupt' ? 'corrupt' : errnoOf(err) === -28 ? 'quota' : 'open', `${name}: ${err?.name || ''} ${err?.message || err}`)
|
|
55
|
+
}
|
|
56
|
+
const persisted = await navigator.storage.persisted?.().catch(() => null)
|
|
57
|
+
self.postMessage({ t: 'ready', bytes: disk.bytes, created: disk.created, dropped: disk.dropped, est, persisted, smaller })
|
|
58
|
+
serve(mailbox, disk, () => {
|
|
59
|
+
data.close()
|
|
60
|
+
map.close()
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function serve(mailbox, disk, closeFiles) {
|
|
65
|
+
const i32 = new Int32Array(mailbox, 0, 8)
|
|
66
|
+
const f64 = new Float64Array(mailbox, 0, 8)
|
|
67
|
+
const data = new Uint8Array(mailbox, HEADER, DATA)
|
|
68
|
+
const tmp = new Uint8Array(DATA) // sync access handles take no views of shared memory
|
|
69
|
+
const stats = { reads: 0, writes: 0, flushes: 0, discards: 0, readBytes: 0, writeBytes: 0, ms: 0 }
|
|
70
|
+
let quotaSaid = false
|
|
71
|
+
let ops = 0
|
|
72
|
+
const post = () => self.postMessage({ t: 'stats', stats: { ...stats, ...disk.usage() } })
|
|
73
|
+
for (;;) {
|
|
74
|
+
if (Atomics.load(i32, 5) === 1) break
|
|
75
|
+
const s = Atomics.load(i32, 1)
|
|
76
|
+
if (s === STATE.GONE) break
|
|
77
|
+
if (s !== STATE.POSTED) {
|
|
78
|
+
Atomics.wait(i32, 1, s, 250)
|
|
79
|
+
continue
|
|
80
|
+
}
|
|
81
|
+
const t = performance.now()
|
|
82
|
+
const op = Atomics.load(i32, 2)
|
|
83
|
+
const len = Atomics.load(i32, 3) >>> 0
|
|
84
|
+
const off = f64[3]
|
|
85
|
+
let r
|
|
86
|
+
if (op === OP.READ) {
|
|
87
|
+
r = disk.read(off, tmp.subarray(0, len))
|
|
88
|
+
if (r === 0) data.set(tmp.subarray(0, len))
|
|
89
|
+
stats.reads++
|
|
90
|
+
stats.readBytes += len
|
|
91
|
+
} else if (op === OP.WRITE) {
|
|
92
|
+
tmp.set(data.subarray(0, len))
|
|
93
|
+
r = disk.write(off, tmp.subarray(0, len))
|
|
94
|
+
stats.writes++
|
|
95
|
+
stats.writeBytes += len
|
|
96
|
+
} else if (op === OP.FLUSH) {
|
|
97
|
+
r = disk.flush()
|
|
98
|
+
stats.flushes++
|
|
99
|
+
} else if (op === OP.DISCARD) {
|
|
100
|
+
r = disk.discard(off, len)
|
|
101
|
+
stats.discards++
|
|
102
|
+
} else r = -22
|
|
103
|
+
if (r === -28 && !quotaSaid) {
|
|
104
|
+
quotaSaid = true
|
|
105
|
+
self.postMessage({ t: 'error', reason: 'quota', message: `the origin's storage quota ran out writing ${len} bytes at ${off}` })
|
|
106
|
+
} else if (r !== 0 && r !== -28) self.postMessage({ t: 'error', reason: 'io', message: `op ${op} at ${off} len ${len}: ${r}` })
|
|
107
|
+
stats.ms += performance.now() - t
|
|
108
|
+
Atomics.store(i32, 4, r)
|
|
109
|
+
// a vCPU that gave up marked the disk gone; leave that mark
|
|
110
|
+
if (Atomics.compareExchange(i32, 1, STATE.POSTED, STATE.DONE) === STATE.POSTED) Atomics.notify(i32, 1)
|
|
111
|
+
if (++ops % 4096 === 0 || op === OP.FLUSH) post()
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
disk.flush()
|
|
115
|
+
post()
|
|
116
|
+
} finally {
|
|
117
|
+
try {
|
|
118
|
+
closeFiles()
|
|
119
|
+
} finally {
|
|
120
|
+
Atomics.store(i32, 1, STATE.GONE)
|
|
121
|
+
Atomics.notify(i32, 1)
|
|
122
|
+
self.postMessage({ t: 'closed' }) // the page waits for this: the handles are closed
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
package/src/host/disk.js
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
// A disk for the guest, kept in the origin private file system (docs/plans/web-buildkit-impl.md 5.9).
|
|
2
|
+
//
|
|
3
|
+
// vm.wasm's virtio-blk device (guest/vm/src/dev/blk.c) calls wmd.blk_rw(op, off, ptr, len) on the vCPU
|
|
4
|
+
// that kicked the queue, with ptr in memory 0 (RAM outside memory 0 is bounced through memory 0). An OPFS
|
|
5
|
+
// sync access handle belongs to one Worker, so one disk Worker (disk-worker.js) owns the disk's files
|
|
6
|
+
// (disk-map.js) and the vCPU Workers reach it through a mailbox in a SharedArrayBuffer:
|
|
7
|
+
//
|
|
8
|
+
// i32[0] lock 0 free, 1 held. vm.wasm already serialises requests (notify_lock, virtio.c); this guards
|
|
9
|
+
// i32[1] state 0 idle, 1 request posted, 2 done, 3 the disk is gone (every request: EIO)
|
|
10
|
+
// i32[2] op 0 read, 1 write, 2 flush, 3 discard (blk.c's BLK_OP_*)
|
|
11
|
+
// i32[3] len bytes (u32)
|
|
12
|
+
// i32[4] result 0 or a negative errno
|
|
13
|
+
// i32[5] control 1: the page asks the Worker to flush and close
|
|
14
|
+
// f64[3] off byte offset (bytes 24..31)
|
|
15
|
+
// data from byte 64, DATA bytes; longer reads and writes go in pieces
|
|
16
|
+
//
|
|
17
|
+
// A vCPU that waits WAIT_MS for an answer marks the disk gone and returns EIO, so a Worker that died or
|
|
18
|
+
// hangs gives the guest an I/O error, never a stuck vCPU.
|
|
19
|
+
|
|
20
|
+
import { DiskError } from './disk-map.js'
|
|
21
|
+
|
|
22
|
+
export const HEADER = 64
|
|
23
|
+
export const DATA = 4 << 20
|
|
24
|
+
export const WAIT_MS = 30_000
|
|
25
|
+
export const OP = { READ: 0, WRITE: 1, FLUSH: 2, DISCARD: 3 }
|
|
26
|
+
export const STATE = { IDLE: 0, POSTED: 1, DONE: 2, GONE: 3 }
|
|
27
|
+
const EIO = -5
|
|
28
|
+
|
|
29
|
+
export function createDiskMailbox() {
|
|
30
|
+
return new SharedArrayBuffer(HEADER + DATA)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The vCPU side: wmd.blk_rw over the mailbox. `now` is for tests. */
|
|
34
|
+
export function createDiskImports({ memory, mailbox, waitMs = WAIT_MS, now = () => performance.now() }) {
|
|
35
|
+
const i32 = new Int32Array(mailbox, 0, 8)
|
|
36
|
+
const f64 = new Float64Array(mailbox, 0, 8)
|
|
37
|
+
const data = new Uint8Array(mailbox, HEADER, DATA)
|
|
38
|
+
const gone = () => Atomics.load(i32, 1) === STATE.GONE
|
|
39
|
+
const lock = () => {
|
|
40
|
+
const end = now() + waitMs
|
|
41
|
+
while (Atomics.compareExchange(i32, 0, 0, 1) !== 0) {
|
|
42
|
+
if (gone()) return false
|
|
43
|
+
if (now() > end) {
|
|
44
|
+
Atomics.store(i32, 1, STATE.GONE)
|
|
45
|
+
return false
|
|
46
|
+
}
|
|
47
|
+
Atomics.wait(i32, 0, 1, 50)
|
|
48
|
+
}
|
|
49
|
+
return true
|
|
50
|
+
}
|
|
51
|
+
const unlock = () => {
|
|
52
|
+
Atomics.store(i32, 0, 0)
|
|
53
|
+
Atomics.notify(i32, 0, 1)
|
|
54
|
+
}
|
|
55
|
+
const one = (op, off, len, mem) => {
|
|
56
|
+
if (op === OP.WRITE) data.set(mem)
|
|
57
|
+
Atomics.store(i32, 2, op)
|
|
58
|
+
Atomics.store(i32, 3, len)
|
|
59
|
+
f64[3] = off
|
|
60
|
+
if (Atomics.compareExchange(i32, 1, STATE.IDLE, STATE.POSTED) !== STATE.IDLE) return EIO
|
|
61
|
+
Atomics.notify(i32, 1, 1)
|
|
62
|
+
const end = now() + waitMs
|
|
63
|
+
for (;;) {
|
|
64
|
+
const s = Atomics.load(i32, 1)
|
|
65
|
+
if (s === STATE.DONE) break
|
|
66
|
+
if (s === STATE.GONE) return EIO
|
|
67
|
+
if (now() > end) {
|
|
68
|
+
// the Worker never answered: the disk is gone for good, so a late answer cannot be mistaken for the next one's;
|
|
69
|
+
// but an answer that landed just before the deadline counts (the swap says which came first)
|
|
70
|
+
if (Atomics.compareExchange(i32, 1, STATE.POSTED, STATE.GONE) === STATE.DONE) break
|
|
71
|
+
Atomics.notify(i32, 1)
|
|
72
|
+
return EIO
|
|
73
|
+
}
|
|
74
|
+
Atomics.wait(i32, 1, STATE.POSTED, 1000)
|
|
75
|
+
}
|
|
76
|
+
const r = Atomics.load(i32, 4)
|
|
77
|
+
if (op === OP.READ && r === 0) mem.set(data.subarray(0, len))
|
|
78
|
+
Atomics.compareExchange(i32, 1, STATE.DONE, STATE.IDLE)
|
|
79
|
+
return r
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
blk_rw(op, off, ptr, len) {
|
|
83
|
+
ptr >>>= 0 // an i32 from wasm; memory 0 can pass 2 GiB
|
|
84
|
+
len >>>= 0
|
|
85
|
+
const base = Number(off)
|
|
86
|
+
if ((op === OP.READ || op === OP.WRITE) && ptr + len > memory.buffer.byteLength) return -14 // EFAULT, never a throw into wasm
|
|
87
|
+
if (gone() || !lock()) return EIO
|
|
88
|
+
try {
|
|
89
|
+
if (op === OP.FLUSH || op === OP.DISCARD) return one(op, base, len, null)
|
|
90
|
+
if (op !== OP.READ && op !== OP.WRITE) return -22
|
|
91
|
+
for (let done = 0; done < len; ) {
|
|
92
|
+
const n = Math.min(DATA, len - done)
|
|
93
|
+
const r = one(op, base + done, n, new Uint8Array(memory.buffer, ptr + done, n))
|
|
94
|
+
if (r !== 0) return r
|
|
95
|
+
done += n
|
|
96
|
+
}
|
|
97
|
+
return 0
|
|
98
|
+
} finally {
|
|
99
|
+
unlock()
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export { DiskError }
|
|
106
|
+
|
|
107
|
+
/** A valid disk name: letters, digits, '-', '_' and '.', at most 128, not starting with '.'. */
|
|
108
|
+
export function checkDiskName(name) {
|
|
109
|
+
if (typeof name !== 'string' || !/^[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}$/.test(name)) throw new TypeError(`bad disk name: ${JSON.stringify(name)}`)
|
|
110
|
+
return name
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Opens (or makes) the disk `name` in OPFS: files `<name>.disk` and `<name>.map`. A new disk gets `mib`
|
|
115
|
+
* MiB, made smaller when the origin's quota has no room for it; an existing one keeps its size.
|
|
116
|
+
* @returns {Promise<{mailbox, name, bytes, mib, created, dropped, est, persisted, events: object[],
|
|
117
|
+
* onEvent: Function|null, stats: object|null, close(): Promise<void>}>}
|
|
118
|
+
* Rejects with DiskError: reason 'locked' when another tab has it open, 'quota' when there is no room
|
|
119
|
+
* for even `minMib`, 'corrupt' when the map is damaged, 'timeout' when the worker never answers, 'open' otherwise.
|
|
120
|
+
*/
|
|
121
|
+
export async function openDisk({ name, mib, minMib = 1024, onEvent = null, openTimeoutMs = 30_000, workerUrl = new URL('./disk-worker.js', import.meta.url) }) {
|
|
122
|
+
checkDiskName(name)
|
|
123
|
+
const mailbox = createDiskMailbox()
|
|
124
|
+
const w = new Worker(workerUrl, { type: 'module', name: `wmd-disk-${name}` })
|
|
125
|
+
const disk = { mailbox, name, events: [], onEvent, stats: null, worker: w, closed: null }
|
|
126
|
+
const emit = ev => {
|
|
127
|
+
disk.events.push(ev)
|
|
128
|
+
if (disk.events.length > 200) disk.events.shift()
|
|
129
|
+
try {
|
|
130
|
+
disk.onEvent?.(ev)
|
|
131
|
+
} catch {}
|
|
132
|
+
}
|
|
133
|
+
let closedRes
|
|
134
|
+
disk.closed = new Promise(r => (closedRes = r))
|
|
135
|
+
await new Promise((res, rej) => {
|
|
136
|
+
const timer = setTimeout(() => {
|
|
137
|
+
Atomics.store(new Int32Array(mailbox, 0, 8), 1, STATE.GONE)
|
|
138
|
+
w.terminate()
|
|
139
|
+
rej(new DiskError(`${name}: the disk worker did not answer in ${Math.round(openTimeoutMs / 1000)} s`, 'timeout'))
|
|
140
|
+
}, openTimeoutMs)
|
|
141
|
+
const rej0 = rej
|
|
142
|
+
const res0 = res
|
|
143
|
+
rej = e => (clearTimeout(timer), rej0(e))
|
|
144
|
+
res = () => (clearTimeout(timer), res0())
|
|
145
|
+
w.onmessage = e => {
|
|
146
|
+
const m = e.data
|
|
147
|
+
if (m.t === 'ready') {
|
|
148
|
+
Object.assign(disk, { bytes: m.bytes, mib: m.bytes / 2 ** 20, created: m.created, dropped: m.dropped, est: m.est, persisted: m.persisted })
|
|
149
|
+
if (m.smaller) emit({ t: 'disk', reason: 'quota', mib: disk.mib, asked: mib })
|
|
150
|
+
res()
|
|
151
|
+
} else if (m.t === 'fail') {
|
|
152
|
+
w.terminate()
|
|
153
|
+
rej(new DiskError(m.message, m.reason))
|
|
154
|
+
} else if (m.t === 'stats') disk.stats = m.stats
|
|
155
|
+
else if (m.t === 'error') emit({ t: 'disk', reason: m.reason || 'error', message: m.message })
|
|
156
|
+
else if (m.t === 'closed') closedRes()
|
|
157
|
+
}
|
|
158
|
+
w.onerror = e => {
|
|
159
|
+
Atomics.store(new Int32Array(mailbox, 0, 8), 1, STATE.GONE)
|
|
160
|
+
rej(new DiskError(`disk worker: ${e.message}`, 'open'))
|
|
161
|
+
emit({ t: 'disk', reason: 'error', message: `disk worker: ${e.message}` })
|
|
162
|
+
}
|
|
163
|
+
w.postMessage({ t: 'open', mailbox, name, bytes: Math.floor(mib) * 2 ** 20, minBytes: minMib * 2 ** 20 })
|
|
164
|
+
})
|
|
165
|
+
disk.close = async ({ timeoutMs = 5_000 } = {}) => {
|
|
166
|
+
const i32 = new Int32Array(mailbox, 0, 8)
|
|
167
|
+
Atomics.store(i32, 5, 1)
|
|
168
|
+
Atomics.notify(i32, 1)
|
|
169
|
+
await Promise.race([disk.closed, new Promise(r => setTimeout(r, timeoutMs))])
|
|
170
|
+
Atomics.store(i32, 1, STATE.GONE)
|
|
171
|
+
w.terminate()
|
|
172
|
+
}
|
|
173
|
+
return disk
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Deletes the disk `name` (both files). False when it is open somewhere (locked) or absent. */
|
|
177
|
+
export async function deleteDisk(name, { root } = {}) {
|
|
178
|
+
checkDiskName(name)
|
|
179
|
+
root ||= await navigator.storage.getDirectory()
|
|
180
|
+
// Lock both files first (a writable stream is refused while a worker holds the file's sync access handle,
|
|
181
|
+
// and works on the page's thread), release, then remove: if either is held, remove neither.
|
|
182
|
+
const files = []
|
|
183
|
+
const locks = []
|
|
184
|
+
try {
|
|
185
|
+
for (const f of [`${name}.disk`, `${name}.map`]) {
|
|
186
|
+
let fh
|
|
187
|
+
try {
|
|
188
|
+
fh = await root.getFileHandle(f)
|
|
189
|
+
} catch (e) {
|
|
190
|
+
if (e?.name === 'NotFoundError') continue // a lone file left behind is still removable
|
|
191
|
+
throw e
|
|
192
|
+
}
|
|
193
|
+
locks.push(await fh.createWritable())
|
|
194
|
+
files.push(f)
|
|
195
|
+
}
|
|
196
|
+
} catch {
|
|
197
|
+
files.length = 0
|
|
198
|
+
}
|
|
199
|
+
for (const l of locks) await l.abort().catch(() => {})
|
|
200
|
+
if (!files.length) return false
|
|
201
|
+
let ok = true
|
|
202
|
+
for (const f of files)
|
|
203
|
+
try {
|
|
204
|
+
await root.removeEntry(f)
|
|
205
|
+
} catch {
|
|
206
|
+
ok = false
|
|
207
|
+
}
|
|
208
|
+
return ok
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Deletes every disk whose name starts with `prefix` and that no tab holds open. Returns the names deleted. */
|
|
212
|
+
export async function sweepDisks(prefix, { root } = {}) {
|
|
213
|
+
root ||= await navigator.storage.getDirectory()
|
|
214
|
+
const names = new Set()
|
|
215
|
+
for await (const [n] of root.entries()) if (n.startsWith(prefix) && /\.(disk|map)$/.test(n)) names.add(n.replace(/\.(disk|map)$/, ''))
|
|
216
|
+
const gone = []
|
|
217
|
+
for (const n of names) if (await deleteDisk(n, { root })) gone.push(n)
|
|
218
|
+
return gone
|
|
219
|
+
}
|