@threadedstack/wmd-web 0.1.17 → 0.1.18

package/README.md

@@ -155,6 +155,50 @@ The interface this host implements is in [docs/host-interface.md](docs/host-inte
   too) and shows the last 128 KB. P-B1's merged branch had the same rings at 4 MiB
   (cherry-picked as `17e9ce50`); the merge keeps P-B2's 32 MiB rings and drop markers.
 
+## Building images (`wmd-web/build`)
+
+`createBuilder` runs BuildKit (moby/buildkit v0.33.0) in a privileged builder machine in the page and builds a
+Dockerfile there. The result is `{index, packs}`, the shape `pullImage` returns, so `bootWmd` can run it at once.
+Chromium, Firefox and WebKit all pass the same 18 build steps (plan `docs/plans/web-buildkit-impl.md`, section 10).
+
+```js
+import { createBuilder } from 'wmd-web/build'
+
+const builder = await createBuilder({
+  vmUrl, files,                       // as bootWmd: the engine and the builder arch's guest files
+  arch: 'arm64',                      // or 'amd64'
+  image: { tar: builderTarUrl, sha256 }, // the builder image; must match BUILDER_PINS in src/build/pin.js
+  relay: { url, protocol: 'wisp' },   // the guest's network: base images and RUN downloads
+  disk: { namespace: 'my-app' },      // BuildKit's cache on an ext4 disk in OPFS (default 10240 MiB)
+  converter,                          // wmdpack.wasm
+  pull: ref => pullImage(ref, { fetch, converter, platform: 'linux/arm64' }), // base images for the fast path (off without it)
+})
+const r = await builder.build({ context: { Dockerfile: 'FROM alpine:3.20\nRUN echo hi > /x\n' } })
+// r: {index, packs, log, path: 'fast'|'buildkit', ms, cached}
+await builder.close()
+```
+
+- **The cache stays.** The disk outlives the builder and the visit. After a reload, or after a tab was killed
+  mid-build, the next build mounts the disk again and its steps come back `CACHED`. `close({keep: false})` deletes
+  the disk.
+- **A second tab** that asks for a disk already open gets a temporary disk, deleted when it closes.
+- **Features:** `RUN --mount=type=cache` and `type=secret`, `# syntax=` frontends, build args and target stages,
+  `.dockerignore`, file capabilities (`setcap`), push to a registry (TLS, or plain http in `httpRegistries`), and
+  `tar: true` for a Docker tar that `docker load` takes. `--mount=type=ssh` and another platform are refused before
+  any work.
+- **The fast path.** With `pull` given, a Dockerfile with no `RUN` is built in the page without BuildKit, layer
+  for layer the same as BuildKit's (40 compared cases): under 0.4 s once its base image is pulled, where a warm
+  BuildKit takes 5 to 14 s.
+- **Errors:** `BuildError {step, tail, reason}`: `reason` is `'no-space'`, `'quota'`, `'disk'`, `'timeout'`, `'pin'`
+  or `'secret'`. Cancelling with `signal` stops the step and leaves nothing behind.
+- **The builder images** are built with `make web-builder` (wmd builds them; Docker Desktop is not used) into
+  `web/dist/builder/builder-<arch>.tar`, and their sha256 is pinned in `src/build/pin.js`. `make web-release` puts
+  them next to the release tgz.
+- **Docker Hub's rate limit** applies to base images. `registryMirrors` gives BuildKit a mirror to try first.
+- **Tests:** `bash ../scripts/bootlock.sh run --max 1800 -- bunx playwright test -c test/build --project=<browser>`
+  (`build.spec.js`: the 18 steps; `parity.spec.js`: the fast path against BuildKit). `test/build/seed-mirror.sh`
+  seeds a local mirror so the runs never touch Docker Hub.
+
 ## Running the tests
 
 ```sh

package/RELEASE.json

@@ -1,38 +1,9 @@
 {
   "name": "wmd-web",
-  "version": "0.3.2",
-  "commit": "fa2b21cfd0fd0bf679498e6feeefc26c33f17372",
-  "previous": null,
+  "version": "0.4.0",
+  "commit": "cb2d3874a3d65236337c76c3c59147fe152a010c",
+  "previous": "wmd-web-v0.1.0",
   "decisions": [
-    "D-388",
-    "D-395",
-    "D-408",
-    "D-415",
-    "D-416",
-    "D-417",
-    "D-419",
-    "D-423",
-    "D-428",
-    "D-430",
-    "D-432",
-    "D-433",
-    "D-438",
-    "D-442",
-    "D-443",
-    "D-444",
-    "D-445",
-    "D-446",
-    "D-447",
-    "D-448",
-    "D-449",
-    "D-451",
-    "D-453",
-    "D-454",
-    "D-468",
-    "D-476",
-    "D-477",
-    "D-481",
-    "D-482",
     "D-486",
     "D-488",
     "D-494",
@@ -49,43 +20,53 @@
     "D-539",
     "D-542",
     "D-544",
-    "D-545"
+    "D-545",
+    "D-547",
+    "D-548"
   ],
-  "vmSources": "c7f2793941170d96db37d9a12a8a5ad072fda7de",
+  "vmSources": "7a707b82c43d3f1eb999476cad68e7ce79cc9cb5",
   "guest": {
     "arch": "arm64",
     "files": {
       "vm.wasm": {
-        "sha256": "2504f4071537e205d78766f2c84219fe49ea4709314becff3a314765733e7fdf",
-        "bytes": 1561314
+        "sha256": "40299e37e788189e6d3bea9d9cdaefa241a5acf27931e6aae14117c100b036ec",
+        "bytes": 1561499
       },
       "Image": {
-        "sha256": "a6b3209cf69d5bc1dacf0f3ed67f92c535dbb4108d67b47091c2aef87c6d7b04",
-        "bytes": 39047176
+        "sha256": "1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b",
+        "bytes": 39049224
       },
       "Image-4k": {
-        "sha256": "8ff6e338a0b42d9f774b35423e0e7f9d7f779c635446e8de2a3beb83d3518f8a",
+        "sha256": "6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b",
         "bytes": 38828040
       },
       "initramfs.cpio.gz": {
-        "sha256": "a2260db90e5d8acdf12a2123f696b4512a1465856d5653e20a85504069a3480c",
-        "bytes": 4685323
+        "sha256": "f008aa9e0f6b4e913242f9982d3b194dba7dc39042a24bc89be64630672125af",
+        "bytes": 4685204
       }
     }
   },
   "files": {
-    "README.md": "6873b42beaf44e02eded67fa54aaa2c13d766597048fbc8633909b6d77b049e0",
+    "README.md": "3c12ca29aab5160b955ba3f50035f19be1dbf0278c4fb006e253a37ac9efbe6a",
     "export-page/coi-sw.js": "0300e670b413cb2e0aae35843fc583546bbf6687df971ab6df41cfa3b0d494d7",
     "export-page/index.html": "f5450c89e63601bbbbffef421f03491b231d1f143070012c1c5315d4b0059f43",
     "export-page/run.css": "b57adf1e76553549c1f89ff157b7973409dedc21889cd3932948cd05b51adc6f",
     "export-page/run.js": "717636a632a40469106ddf1383e198eaac5d83651b5866d0699ec04b82462b7f",
-    "guest/arm64/Image": "a6b3209cf69d5bc1dacf0f3ed67f92c535dbb4108d67b47091c2aef87c6d7b04",
-    "guest/arm64/Image-4k": "8ff6e338a0b42d9f774b35423e0e7f9d7f779c635446e8de2a3beb83d3518f8a",
-    "guest/arm64/initramfs.cpio.gz": "a2260db90e5d8acdf12a2123f696b4512a1465856d5653e20a85504069a3480c",
-    "guest/arm64/vm.wasm": "2504f4071537e205d78766f2c84219fe49ea4709314becff3a314765733e7fdf",
-    "package.json": "17aa3a8406e8d2d85b647c426438d3fd75edcb59d83e257882227e7b93cccbc0",
+    "guest/arm64/Image": "1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b",
+    "guest/arm64/Image-4k": "6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b",
+    "guest/arm64/initramfs.cpio.gz": "f008aa9e0f6b4e913242f9982d3b194dba7dc39042a24bc89be64630672125af",
+    "guest/arm64/vm.wasm": "40299e37e788189e6d3bea9d9cdaefa241a5acf27931e6aae14117c100b036ec",
+    "package.json": "778ef2457ab01f302387c811d796b7c61f97feb095725da285655629965f2b5e",
     "src/base.js": "982f8e46dbb727ec542a494ebdb884ccd19a31ff4e091425b2d1fc01b0d4754a",
-    "src/boot.js": "911a40e01fa4833249743653b7698e8fed217d52a34056a92524562f3f8daacc",
+    "src/boot.js": "62f5cf901d797d53bef584b3c6c0c7fb1a4cc8e2450e9570d6289c72036ed025",
+    "src/build/builder.js": "751ea022e1f6f0c6ab79bd4efd6cfe0d705268ff7f699af87dcc64d1871ccc52",
+    "src/build/context.js": "4baf1884d366455f3f76f397575d515671585ebf5deb27bc35bc2b1dd9c6bc11",
+    "src/build/dockerfile.js": "5957fe3d4ac8ff8b312f980ee04bd1d98ad4373a61f9a33e02f1ff1108b6ea53",
+    "src/build/dockerignore.js": "8fc03d09da020f43ea483979b1183891b71d76f012379ab7813838ea0e53c546",
+    "src/build/fast.js": "bf7373a43cce995b21751c317a58d18438f9fd2f670266b314b1151ef3cdb9e8",
+    "src/build/inspect.js": "52aa26d332c08406e8eeaf8f72661d8acc36bbd4f547a7d68e72b4f0fe265a22",
+    "src/build/pin.js": "8c67f017ae3ec560d2638fdfc0bd91ddd758150af16c188bbe3bf2cb4d7e28b2",
+    "src/build/tar.js": "25cf477488d9a60da9015be9ca5830bad3ad24428da1546f0621411cb7f44949",
     "src/calib-worker.js": "3d610591a1b3b58f59c3a2ab457f43c90727d95973cd43055b3b73b76180611d",
     "src/fs/9p/codec.js": "69711eb4ebc84eff39256138169fe18f5a5066d6cd9e4ff2daf61c65380d7ef7",
     "src/fs/9p/server.js": "da0ba00ebb5522161bb0ef7cd94bfc9616e342bf865633dfef0ba5bc67627d7a",
@@ -110,7 +91,10 @@
     "src/host/ckpt-save.js": "e222571033ae74a4439596c5e90e03b2aa9a5516665dd290f8dd243712294523",
     "src/host/ckpt.js": "2155b5c8e6f609478805eb8a0b5d5210d47cdf35a80f79e0b9ae42b2c3170923",
     "src/host/clock.js": "aba6af3a8b1a985ae340892222e4b482d33afe20f5c8b3a4c7403ce87d1e247c",
-    "src/host/ctl.js": "654c55c64efa3df4df8b7b28024a10bd0e7ed3afb89004f8a583b9e62f17f58f",
+    "src/host/ctl.js": "ca2ebbb940c9668cb0573c959cd2ff8982f96d58dfdb8dce39d304b89e033dd7",
+    "src/host/disk-map.js": "3d7e0d7d488ce25fc6a0e4dc0a9b1c8fc2af96bac1c03491a1d30be392af268b",
+    "src/host/disk-worker.js": "9d470cdc72d9e1fbf955e466ec2562abe780f2c6d89174ca3e4fbb1828f8dd76",
+    "src/host/disk.js": "373e23539e3916674b7e8708d2e3ae22578214129e56936765db5c8f465cb343",
     "src/host/errno.js": "26021d85962084be2c43c98fb3accafc2a8f11fbf3c8a5b8d484bd6a088ae4db",
     "src/host/fs.js": "d6753f74db35b7b01baa8709fc7fdcdfc8d638dfe42f454b0a833b22111e8f4a",
     "src/host/gcnudge.js": "4a6569cb73ae24a11fc4ac8c467380bd9efba0bb5fa9867b797bc4f153ab15ab",
@@ -127,9 +111,9 @@
     "src/host/stdio.js": "43973675578eb1d324c2c6d7078e51a6a1f121190ada5ef14a5551364b8c7f5e",
     "src/host/threads.js": "952381940e7f43114867e9a9714528f2f20c8064b6b0f2018ab075128dc601be",
     "src/host/wasi.js": "fbafa239b8fb4083f7e8c910b1adab38901e8e7cffd247284a9abb2847eaf4bd",
-    "src/host/wmd-imports.js": "094f994758c239b75177781c0613decbbd926090b0e8974163ce7caf8a99234d",
+    "src/host/wmd-imports.js": "3a00562f7c9c7787ab3841b14dde5d903a78f43ed19c43a2f5e31fd8a14c09d2",
     "src/host/yamux.js": "f4044f14f7e836224bdb4b1e53e59c565ee478b83b2e428fffb9f7a048768f79",
-    "src/image-files.js": "539856f3991b9f89202dd04ec3816ba8bce7787034ab810534c5170a205ffde7",
+    "src/image-files.js": "7bac16bd278d17ba291dfe4771564627b5efbdda927b429018a75e8b906dd4cd",
     "src/net/dhcp-dns.js": "fdfa7c7d5cbbb4a8774b7dee195bf2df22a0c3fe8f20ac9694b24485fa8f25d5",
     "src/net/net-worker.js": "0a2ce998f69807e386ded7ff1bee53687a1d59ed76d7ef46e2eb6f958b14ad38",
     "src/net/packet.js": "d658f26fa84324a88d7dbc26af9c4a9c10e977c3060f16f7db431496eacd1127",
@@ -143,12 +127,13 @@
     "src/net/wisp.js": "06c50159f416d40a912fe2cce4806388b938f2ab0220b2ee6da890630ce3d113",
     "src/registry/client.js": "fd38550e9afd82c5cb498513d245c33357a71697b3b2e6dd78538934bcc9d879",
     "src/registry/converter.js": "1fb8294fd5275f6772c27130cc9ba70bd46fafaf440884d42448036618229f37",
-    "src/registry/hub.js": "22323e0f799a7078843514c9f87bbcdaa4173fd991ad4e059377befa773762d4",
+    "src/registry/hub.js": "cd672323bd1f96bd1d7e9eed80d4f36891e919e456fc0deb378fa0c7e3b6f249",
+    "src/registry/load-tar.js": "9dd2b5d8884c64247339b141ab5a57b8ab1a2e007c32aa45b1103b37405b0ed7",
     "src/registry/oci.js": "6a51ea16b34a65c0e732730f3e07d7ad16c50f6943a486ec05ca6aa3e98676a1",
     "src/registry/pack-worker.js": "fd6798851cb7815dacfcb67fc78c41654209c5504d8fd343aae524e99de1c741",
-    "src/registry/pull.js": "66edf4602aca8d41cc68393d01a2da01c4667de7c493af5ae400e8f249d4e3ae",
+    "src/registry/pull.js": "24f2e01a76c9f7f27896af69b989f8c97f0ca31712b3e8110aaa5b5e3148b04b",
     "src/registry/ref.js": "24038e0fbcb6ac143cb8e8f7b9fac4dd0803d0891961786dc88dba01a4402e7b",
     "src/registry/wasi-mem.js": "a7f684343010892460db27ee8c1e38e47e797c93dc4ebe4c30185dca6ca56ee2",
-    "src/worker.js": "fe23f1b99acedc65e56f92f7246923fdbdbefd085bae427b57cbe1402b4f5161"
+    "src/worker.js": "c654be359e741d5b1fde1be05f8909968d4e00d1fca61be169e60ea691daa662"
   }
 }

package/guest/arm64/Image

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: package/guest/arm64/Image
-size: 39047176 bytes
-sha256: a6b3209cf69d5bc1dacf0f3ed67f92c535dbb4108d67b47091c2aef87c6d7b04
+size: 39049224 bytes
+sha256: 1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b

package/guest/arm64/Image-4k

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: package/guest/arm64/Image-4k
 size: 38828040 bytes
-sha256: 8ff6e338a0b42d9f774b35423e0e7f9d7f779c635446e8de2a3beb83d3518f8a
+sha256: 6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@threadedstack/wmd-web",
-  "version": "0.1.17",
+  "version": "0.1.18",
   "license": "MIT",
   "homepage": "https://github.com/threadedstack/wmd-releases",
   "repository": {
@@ -9,9 +9,9 @@
   },
   "description": "Browser host for wmd's guest vm.wasm (Web Workers + shared memory + WASI preview1)",
   "wmd": {
-    "tag": "v0.1.17",
-    "commit": "fa2b21cfd0fd0bf679498e6feeefc26c33f17372",
-    "web": "0.3.2"
+    "tag": "v0.1.18",
+    "commit": "cb2d3874a3d65236337c76c3c59147fe152a010c",
+    "web": "0.4.0"
   },
   "type": "module",
   "main": "./src/boot.js",

package/relay/hosted.ts

@@ -26,7 +26,7 @@ export type HostedOptions = {
   now?: () => number // ms since the epoch (tests)
   warn?: (line: string) => void // one-line operator notices (stderr)
   sites?: string[] // A: page origin patterns served (required: none listed serves no page)
-  sitesPerAddrPerHour?: number // B: different sites per address group per hour (200: plan §16)
+  sitesPerAddrPerHour?: number // B: different sites per address group per hour (200: plan §16); a publish identity three times that
   sitesPerNetPerHour?: number // B: per network (800)
   monthBytes?: number // C: bytes relayed per UTC month (the operator's figure from the provider's free outbound data); 0 = no cap
   usageFile?: string // C: the month's counter survives a restart here
@@ -57,6 +57,7 @@ export type Handle = { ip: string; grp: string; origin: string; end: (why: Why)
 /** What a session holds from the governor until it ends. */
 export type Admission = {
   grp: string
+  publish: boolean
   net: string
   handle: Handle
   fair: Bucket | null
@@ -403,7 +404,7 @@ export function createGovernor(o: HostedOptions) {
     return m
   }
   const oldest = (m: Map<string, number>) => Math.min(...m.values()) + HOUR
-  function sites(grp: string, host: string): Why | null {
+  function sites(grp: string, host: string, publish: boolean): Why | null {
     const t = now()
     const site = host.toLowerCase().replace(/\.$/, '')
     const net = coarseKey(grp)
@@ -414,8 +415,9 @@ export function createGovernor(o: HostedOptions) {
     fresh(a, t)
     fresh(n, t)
     if (!a.has(site)) {
-      if (a.size >= c.sitesPerAddrPerHour) {
-        return { rule: 'too-many-sites', words: `${c.sitesPerAddrPerHour} different sites in an hour from this address: a new site waits until one ages out; sites already reached keep working`, until: oldest(a) }
+      const max = publish ? 3 * c.sitesPerAddrPerHour : c.sitesPerAddrPerHour
+      if (a.size >= max) {
+        return { rule: 'too-many-sites', words: `${max} different sites in an hour from this address${publish ? ' while publishing' : ''}: a new site waits until one ages out; sites already reached keep working`, until: oldest(a) }
       }
       if (!n.has(site) && n.size >= c.sitesPerNetPerHour) {
         return { rule: 'too-many-sites', words: `${c.sitesPerNetPerHour} different sites in an hour from this network: a new site waits until one ages out; sites already reached keep working`, until: oldest(n) }
@@ -452,7 +454,7 @@ export function createGovernor(o: HostedOptions) {
   const grpOf = (s: Session): string => s.hx?.grp ?? s.p.grp
 
   /** A session from `ip` (address group `grp`) with this page origin is about to open. */
-  function admitSession(i: { ip: string; grp: string; origin: string; end: (why: Why) => void; tell?: Tell; claimed?: boolean }): { ok: true; adm: Admission } | ({ ok: false } & Why) {
+  function admitSession(i: { ip: string; grp: string; origin: string; publish?: boolean; end: (why: Why) => void; tell?: Tell; claimed?: boolean }): { ok: true; adm: Admission } | ({ ok: false } & Why) {
     const no = (w: Why) => ({ ok: false as const, ...w })
     const denied = deniedClient(i.ip, i.origin)
     if (denied) return no(denied)
@@ -467,7 +469,7 @@ export function createGovernor(o: HostedOptions) {
     live.add(handle)
     let done = false
     const adm: Admission = {
-      grp: i.grp, net: coarseKey(i.grp), handle, fair: c.linkBytesPerSec ? new Bucket(Infinity) : null, sent: 0, waiting: false,
+      grp: i.grp, publish: !!i.publish, net: coarseKey(i.grp), handle, fair: c.linkBytesPerSec ? new Bucket(Infinity) : null, sent: 0, waiting: false,
       release() {
         if (done) return
         done = true
@@ -488,7 +490,7 @@ export function createGovernor(o: HostedOptions) {
   function admitConnect(s: Session, host: string, _port: number): Why | { waitMs: number } | null {
     const closed = notAccepting()
     if (closed) return closed
-    const crawl = sites(grpOf(s), host)
+    const crawl = sites(grpOf(s), host, !!s.hx?.publish)
     if (crawl) return crawl
     return dialGate(s)
   }
@@ -691,7 +693,7 @@ export function createGovernor(o: HostedOptions) {
     blockedAddr,
     liveStreams: () => used,
     limitsLine: () =>
-      `only listed sites; ${c.sitesPerAddrPerHour} different sites an hour per address, ${c.sitesPerNetPerHour} per network; failed dials slow the sender` +
+      `only listed pages may use this relay; ${c.sitesPerAddrPerHour} different sites an hour per address, ${c.sitesPerNetPerHour} per network; failed dials slow the sender` +
       (c.monthBytes ? `; ${Math.round(c.monthBytes / 1e9)} GB a month, heavy users slowed first when the month runs ahead` : ''),
     stop() {
       clearInterval(ticker)

package/relay/main.ts

@@ -53,7 +53,7 @@ Flags:
   --allow-loopback-ports <list>  guests may reach this computer's loopback on these ports
   --block-ports <list>           destination ports refused (none)
   --allow-hosts <list>           only these names (name[:port|port], a leading . for subdomains)
-  --hosted                       the public hosted profile (docs/relay-hosted.md): only listed sites, no crawling,
+  --hosted                       the public hosted profile (docs/relay-hosted.md): only listed pages may use this relay, no crawling,
                                  failed dials slowed, the month's data cap, host limits, deny lists, a control
                                  socket, /healthz, /readyz
     --abuse-contact <text>       required: an address or URL, shown in the MOTD, /healthz and every refusal
@@ -326,7 +326,7 @@ export function relayOptions(c: Conf, env: Record<string, string | undefined> =
     if (c.behindTls && !isPrivateBind(host)) throw new UsageError(`--hosted --behind-tls binds only to loopback or a private address (the proxy\'s side), not ${host}: a public bind would let anyone skip the proxy and its TLS`)
     if (!tls && !c.behindTls && !isPrivateBind(host)) throw new UsageError(`--hosted refuses plain ws:// on the public address ${host}: give --tls-cert and --tls-key, or put a TLS proxy in front with --behind-tls (bound to loopback)`)
     if (c.localTls) throw new UsageError('--local-tls is for a person\'s own computer, not --hosted')
-    if (!c.sitesFile) throw new UsageError('--hosted needs --sites-file <file>: the page origins it serves (JSON {"sites": [...]}); a hosted relay serves only listed sites')
+    if (!c.sitesFile) throw new UsageError('--hosted needs --sites-file <file>: the page origins it serves (JSON {"sites": [...]}); only listed pages may use this relay')
   }
   const w = c.wisp ?? {}
   const secret = env.WMD_RELAY_SECRET || randomBytes(32).toString('base64url') // tokens (wmn's issuer) only when the secret is shared

package/relay/relay.ts

@@ -132,6 +132,7 @@ export function createRelay(o: RelayOptions) {
         const r = health(url.pathname)
         return req.method === 'HEAD' ? new Response(null, { status: r.status, headers: r.headers }) : r // same status and headers, no body
       }
+      if (gov && wisp && url.pathname === '/publish-token') return wisp.publishToken(req, srv, url)
       if (url.pathname === '/relay') {
         const protos = req.headers.get('sec-websocket-protocol') || ''
         if (wisp && protos.includes('wisp-v2') && !protos.includes(PROTOCOL)) return new Response(`this is the wmd-relay.v1 endpoint; Wisp is at ${wisp.path}\n`, { status: 400 })

package/relay/token.ts

@@ -1,5 +1,5 @@
 // P-B10 relay: session tokens. A token is "v1.<payload>.<mac>", base64url without padding,
-// where the payload is JSON {sid, exp (ms since the epoch), uid?, grp?} and the mac is
+// where the payload is JSON {sid, exp (ms since the epoch), uid?, grp?, kind?} and the mac is
 // HMAC-SHA256 of "v1.<payload>" with the relay's secret. Whoever serves the page
 // (test/serve.ts, or wmn's server) mints them with the same secret (TokenIssuer below); the
 // browser presents one as a WebSocket subprotocol "t.<token>" (not in the URL, so it stays
@@ -9,6 +9,8 @@
 //   uid  the guest identity the relay keeps caps for, across reconnects and fresh tokens
 //        (one guest machine of one page); defaults to sid
 //   grp  the client the issuer saw (an address, a user): the relay caps sessions per grp
+//   kind "publish": a publish identity (TokenIssuer.issuePublish), which a hosted relay gives a higher sites-an-hour
+//        count (hosted.ts). Only the signature makes it one: the relay never takes a client's word for it
 //
 // Parsing is strict: a token is exactly one encoding of its payload and MAC, so the same
 // token never verifies under two spellings.
@@ -19,19 +21,20 @@ const b64u = (b: Buffer) => b.toString('base64url')
 const B64U = /^[A-Za-z0-9_-]+$/
 const ID = /^[A-Za-z0-9_.:\/-]{1,128}$/
 
-export type TokenFields = { sid?: string; uid?: string; grp?: string; ttlMs?: number; now?: number }
+export type TokenFields = { sid?: string; uid?: string; grp?: string; kind?: string; ttlMs?: number; now?: number }
 
-export function mintToken(secret: string, { sid = b64u(randomBytes(12)), uid, grp, ttlMs = 60 * 60_000, now = Date.now() }: TokenFields = {}): string {
+export function mintToken(secret: string, { sid = b64u(randomBytes(12)), uid, grp, kind, ttlMs = 60 * 60_000, now = Date.now() }: TokenFields = {}): string {
   if (!secret || secret.length < 16) throw new Error('relay secret: at least 16 characters')
   const body: Record<string, unknown> = { sid, exp: now + ttlMs }
   if (uid) body.uid = uid
   if (grp) body.grp = grp
+  if (kind) body.kind = kind
   const payload = b64u(Buffer.from(JSON.stringify(body)))
   const mac = createHmac('sha256', secret).update(`v1.${payload}`).digest()
   return `v1.${payload}.${b64u(mac)}`
 }
 
-export type TokenInfo = { sid: string; exp: number; uid?: string; grp?: string }
+export type TokenInfo = { sid: string; exp: number; uid?: string; grp?: string; kind?: 'publish' }
 
 /** The token's session, or an error string. */
 export function verifyToken(secret: string, token: string, now = Date.now()): TokenInfo | string {
@@ -52,8 +55,9 @@ export function verifyToken(secret: string, token: string, now = Date.now()): To
   }
   if (typeof info !== 'object' || !info || typeof info.sid !== 'string' || !ID.test(info.sid) || typeof info.exp !== 'number') return 'bad token'
   for (const k of ['uid', 'grp'] as const) if (info[k] !== undefined && (typeof info[k] !== 'string' || !ID.test(info[k]!))) return 'bad token'
+  if (info.kind !== undefined && info.kind !== 'publish') return 'bad token'
   if (info.exp <= now) return 'token expired'
-  return { sid: info.sid, exp: info.exp, uid: info.uid, grp: info.grp }
+  return { sid: info.sid, exp: info.exp, uid: info.uid, grp: info.grp, ...(info.kind ? { kind: info.kind } : {}) }
 }
 
 /** The key a client's limits are kept under: IPv6 addresses by their /64 (one subscriber holds 2^64 of them). */
@@ -79,6 +83,7 @@ export function clientKey(client: string): string {
  * an hour. At most `maxClients` clients are remembered (the least recently used go first).
  */
 export const MIN_TTL_MS = 60_000
+export const PUBLISH_TTL_MS = 60 * 60_000
 export class TokenIssuer {
   perMinute: number
   instancesPerHour: number
@@ -86,6 +91,7 @@ export class TokenIssuer {
   maxClients: number
   minTtlMs: number
   clients = new Map<string, { times: number[]; instances: Map<string, number> }>() // least recently used first
+  publishing = new Map<string, { instance: string; token: string; until: number }>() // per client: its live publish identity, in the order they end
   constructor(public secret: string, { perMinute = 60, instancesPerHour = 20, maxTtlMs = 60 * 60_000, maxClients = 10_000, minTtlMs = MIN_TTL_MS } = {}) {
     this.perMinute = perMinute
     this.instancesPerHour = instancesPerHour
@@ -113,4 +119,27 @@ export class TokenIssuer {
     const ttl = Math.min(Math.max(ttlMs, this.minTtlMs), this.maxTtlMs)
     return { token: mintToken(this.secret, { uid: `${key}/${instance}`, grp: key, ttlMs: ttl, now }) }
   }
+  /** A publish identity for this client, for one hour; at most one per client at a time. The page that holds it gets the
+      same token again (a reconnect); another page of the client is refused until its hour is over. When `maxClients`
+      identities are live, a new one is refused (503) until the first ends: a live one is never dropped. */
+  issuePublish(client: string, instance: string, now = Date.now()): { token: string; until: number } | { error: string; status: number; until?: number } {
+    if (!ID.test(client)) return { error: 'bad client', status: 400 }
+    if (!/^[A-Za-z0-9_-]{8,64}$/.test(instance || '')) return { error: 'instance: 8-64 characters [A-Za-z0-9_-]', status: 400 }
+    const key = clientKey(client)
+    for (const [k, p] of this.publishing) {
+      if (p.until > now) break
+      this.publishing.delete(k)
+    }
+    const live = this.publishing.get(key)
+    if (live && live.instance === instance) return { token: live.token, until: live.until }
+    if (live) return { error: `this address already has a publish identity; another can be had in ${Math.ceil((live.until - now) / 60_000)} min`, status: 429, until: live.until }
+    if (this.publishing.size >= this.maxClients) {
+      const first = this.publishing.values().next().value!
+      return { error: `the relay is giving out as many publish identities as it can hold; try again in ${Math.ceil((first.until - now) / 60_000)} min`, status: 503, until: first.until }
+    }
+    const until = now + PUBLISH_TTL_MS
+    const token = mintToken(this.secret, { uid: `${key}/publish.${instance}`, grp: key, kind: 'publish', ttlMs: PUBLISH_TTL_MS, now })
+    this.publishing.set(key, { instance, token, until })
+    return { token, until }
+  }
 }

package/relay/wisp.ts

@@ -1,7 +1,8 @@
 // W59: the Wisp v2 adapter (src/net/wisp.js, docs/wisp-wmd-extensions.md) over the relay engine.
 //
 // Who may use it (plan §5):
-//   - token: the HMAC `t.<token>` subprotocol, as wmd-relay.v1 (wmn's bundled relay)
+//   - token: the HMAC `t.<token>` subprotocol, as wmd-relay.v1 (wmn's bundled relay); on a hosted relay, the publish
+//     identity it issues itself (relay.ts /publish-token), used only from the address it was given to
 //   - open: no key (the standalone relay's default); identity = the client's instance id (0xE7)
 //     plus its address group, bandwidth shared per address
 //   - keys (optional): Wisp password auth, scrypt hashes; a relay with keys refuses keyless clients
@@ -23,7 +24,7 @@ import { type AllowEntry, type Engine, type Principal, type Session, type Shared
 import { type Governor, type Why, resolveClient } from './hosted.ts'
 import { type OwnerKey, verifySecret } from './keys.ts'
 import type { Cidr } from './policy.ts'
-import { clientKey, verifyToken } from './token.ts'
+import { TokenIssuer, clientKey, verifyToken } from './token.ts'
 
 export type WispOptions = {
   enabled?: boolean
@@ -184,6 +185,32 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     return reply(body, status)
   }
 
+  const issuer = w.hosted && o.secret ? new TokenIssuer(o.secret) : null
+
+  /** Hosted: GET /publish-token?instance=ID from a listed page gives {token, until}, a publish identity for this address
+      (token.ts issuePublish), or {error, until?} in words. The page's origin may read the answer. */
+  function publishToken(req: Request, srv: any, url: URL): Response {
+    const origin = req.headers.get('origin') || ''
+    if (!issuer || origin.length > 256 || !w.hosted!.siteAllowed(origin)) {
+      log({ ev: 'publish-token-refused', why: 'not-allowed-site' })
+      return reply('this page is not one of the sites this relay serves\n', 403)
+    }
+    const headers = { 'access-control-allow-origin': origin, vary: 'Origin', 'cache-control': 'no-store' }
+    if (req.method !== 'GET') return new Response(`GET only\n${w.hosted!.contactLine}`, { status: 405, headers })
+    const ip = clientIp(req, srv)
+    if (typeof ip !== 'string') {
+      for (const [k, v] of Object.entries(headers)) ip.headers.set(k, v)
+      return ip
+    }
+    const r = issuer.issuePublish(ip, url.searchParams.get('instance') || '')
+    if ('token' in r) {
+      log({ ev: 'publish-token', addr: clientKey(ip), origin })
+      return Response.json(r, { headers })
+    }
+    log({ ev: 'publish-token-refused', why: r.error, addr: clientKey(ip) })
+    return Response.json({ error: r.error, ...(r.until ? { until: r.until } : {}), abuseContact: w.hosted!.contact }, { status: r.status, headers })
+  }
+
   async function fetch(req: Request, srv: any, url: URL): Promise<Response | undefined> {
     const ip = clientIp(req, srv)
     if (typeof ip !== 'string') return ip
@@ -210,11 +237,13 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     let uid = ''
     let grp = ''
     let exp = Infinity
+    let publish = false
     let p: Principal | null = null
     const tok = protos.find(x => x.startsWith('t.'))
     if (tok && o.secret) {
       const info = verifyToken(o.secret, tok.slice(2))
       if (typeof info === 'string') return refuse(401, `${info}\n`, info, { origin })
+      if (w.hosted && info.grp !== addr) return refuse(401, 'this token was given to another address\n', 'token of another address', { addr })
       if (engine.hasSession(info.sid)) {
         stats.refusedSessions++
         return reply('session already connected\n', 409)
@@ -224,7 +253,9 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
       uid = info.uid || info.sid
       grp = info.grp || uid
       exp = info.exp
-      const a = engine.admit(uid, grp, exp, { proto: 'wisp' })
+      publish = info.kind === 'publish'
+      // a hosted publish session shares its address's bandwidth with the address's open sessions
+      const a = engine.admit(uid, grp, exp, { proto: 'wisp' }, w.hosted && publish ? sharedFor(grp) : undefined)
       if (!a.ok) return reply(a.body, a.status)
       p = a.p
     } else if (rest) {
@@ -271,7 +302,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     const full = w.hosted?.claimHandshake() ?? null
     const headers: Record<string, string> = {}
     if (v2 && protos.length) headers['Sec-WebSocket-Protocol'] = protos.includes(SUBPROTOCOL) ? SUBPROTOCOL : protos[0]
-    const data = { kind: 'wisp', v2, mode, addr, ip, sid, uid, grp, exp, p, key, origin, hs: null as any, hsTimer: null as any, pre, claimed: !!w.hosted && !full, refuse: full as Why | null, at: performance.now() }
+    const data = { kind: 'wisp', v2, mode, addr, ip, sid, uid, grp, exp, publish, p, key, origin, hs: null as any, hsTimer: null as any, pre, claimed: !!w.hosted && !full, refuse: full as Why | null, at: performance.now() }
     const ok = srv.upgrade(req, Object.keys(headers).length ? { data, headers } : { data }) // Bun refuses an empty headers object
     if (!ok) {
       release(data)
@@ -417,7 +448,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
         tell(REASON_KIND.REFUSED, why.rule, why.until ?? 0, why.words)
         ws.close(4011, why.words.slice(0, 120))
       }
-      adm = w.hosted.admitSession({ ip: d.ip, grp: d.grp, origin: d.origin, end, tell, claimed: d.claimed })
+      adm = w.hosted.admitSession({ ip: d.ip, grp: d.grp, origin: d.origin, publish: d.publish, end, tell, claimed: d.claimed })
       if (!adm.ok) return refuseHs(ws, client, adm)
       d.claimed = false // the session holds the socket's count now
       d.rel = adm.adm.release
@@ -664,6 +695,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
       ws.close(1013, 'the relay is full')
     },
     fetch,
+    publishToken,
     open: open_,
     message,
     close,

package/src/boot.js

@@ -51,9 +51,11 @@ import {
 } from './host/ckpt.js'
 import { captureJit, jitSnapStats, modulesOf, ownersNeedingHub, restoreStore } from './host/jit-snap.js'
 export { CheckpointError, memoryStore, opfsStore } from './host/ckpt.js'
+export { DiskError, deleteDisk, openDisk, sweepDisks } from './host/disk.js'
 export { NetSwitch } from './net/switch.js'
 export { ImageFiles, openImage, parseGroup, parsePasswd, resolveUser } from './image-files.js'
 export { P9_CACHE_DEFAULT, p9InitCmdline, p9MountOptions } from './host/p9mount.js'
+export { loadImageTar } from './registry/load-tar.js'
 
 const PAGE = 65536
 /** A trap with the memory this close to its maximum is reported as out of memory. */
@@ -414,12 +416,30 @@ function toShared(bytes) {
   return sab
 }
 
+/** vm.wasm's arguments for a machine with `disk`: --swap <MiB> and wmd.disk=data appended to --append. */
+export function diskArgs(args, disk) {
+  if (args.includes('--swap')) throw new TypeError('--swap and a disk: give one')
+  const mib = disk.bytes / 2 ** 20
+  if (!Number.isInteger(mib) || mib < 1) throw new TypeError(`disk size ${disk.bytes} is not whole MiB`)
+  const out = [...args, '--swap', String(mib)]
+  const i = out.indexOf('--append')
+  if (i < 0 || i + 1 >= out.length) throw new TypeError('a disk needs --append (the kernel line gets wmd.disk=data)')
+  // the disk holds data: a wmd.disk= of any other value would let the guest init write a swap header over it
+  const given = out[i + 1].match(/(?:^| )wmd\.disk(?:=(\S*))?(?= |$)/g)
+  if (given && given.some(g => g.trim() !== 'wmd.disk=data')) throw new TypeError(`a disk needs wmd.disk=data on the kernel line, not ${given.map(g => g.trim()).join(' ')}`)
+  if (!given) out[i + 1] = `${out[i + 1]} wmd.disk=data`
+  return out
+}
+
 /**
  * @param {object} o
  * @param {string|Uint8Array|ArrayBuffer} o.vmUrl  URL of vm.wasm, or its bytes
  * @param {Record<string,string|Uint8Array>} o.files  preopen contents: path → URL or bytes
  * @param {string[]} o.args                     vm.wasm arguments (after argv[0])
  * @param {string[]} [o.env]                    "K=V" (the runner passes only WMD_*)
+ * @param {object} [o.disk]                     a disk from openDisk (host/disk.js): the guest gets it as virtio-blk
+ *        /dev/vda (vm.wasm's --swap MiB sizes the device) with wmd.disk=data on the kernel line, so guest init leaves
+ *        it alone. A machine with a disk takes no checkpoints: guest memory and the disk would not match
  * @param {boolean} [o.heapReserve]             W21 (default true, false in mobile WebKit: heapReserveDefault, D-516): vm.wasm grows its memory to the maximum
  *        before any thread starts (WMD_HEAP_RESERVE=1), so memory never grows under running
  *        vCPUs. Chromium checks memory.fill/copy against a per-Worker size that lags another
@@ -520,6 +540,10 @@ export async function bootWmd(o) {
     o.onStderr?.(text)
   }
   const preopen = o.preopen || '/g'
+  if (o.disk) {
+    if (o.checkpoint || o.restore || o.restoreLazy) throw new TypeError('a machine with a disk takes no checkpoints')
+    o = { ...o, args: diskArgs(o.args, o.disk) }
+  }
   const argv = ['vm.wasm', ...o.args]
   const env = vmEnv(o.env, o.heapReserve ?? heapReserveDefault())
   // the default Worker is started from a literal new URL(..., import.meta.url) so bundlers (Vite, webpack)
@@ -1550,6 +1574,7 @@ export async function bootWmd(o) {
       tableMin,
       memory,
       ram: ramMem,
+      blk: o.disk?.mailbox || null,
       ctlSab,
       files,
       preopen,