@threadedstack/wmd-web 0.1.18 → 0.1.20

package/README.md

@@ -159,10 +159,11 @@ The interface this host implements is in [docs/host-interface.md](docs/host-inte
 
 `createBuilder` runs BuildKit (moby/buildkit v0.33.0) in a privileged builder machine in the page and builds a
 Dockerfile there. The result is `{index, packs}`, the shape `pullImage` returns, so `bootWmd` can run it at once.
-Chromium, Firefox and WebKit all pass the same 18 build steps (plan `docs/plans/web-buildkit-impl.md`, section 10).
+Chromium, Firefox and WebKit all pass the same 18 build steps, and Node and Go projects for arm64 and amd64 (plan
+`docs/plans/web-buildkit-impl.md`, section 10).
 
 ```js
-import { createBuilder } from 'wmd-web/build'
+import { buildImage, createBuilder } from 'wmd-web/build'
 
 const builder = await createBuilder({
   vmUrl, files,                       // as bootWmd: the engine and the builder arch's guest files
@@ -176,6 +177,9 @@ const builder = await createBuilder({
 const r = await builder.build({ context: { Dockerfile: 'FROM alpine:3.20\nRUN echo hi > /x\n' } })
 // r: {index, packs, log, path: 'fast'|'buildkit', ms, cached}
 await builder.close()
+
+// or, for one build: createBuilder + build + close in one call
+const r2 = await buildImage({ ...builderOptions, context, timeoutMs: 30 * 60_000 })
 ```
 
 - **The cache stays.** The disk outlives the builder and the visit. After a reload, or after a tab was killed
@@ -190,7 +194,24 @@ await builder.close()
   for layer the same as BuildKit's (40 compared cases): under 0.4 s once its base image is pulled, where a warm
   BuildKit takes 5 to 14 s.
 - **Errors:** `BuildError {step, tail, reason}`: `reason` is `'no-space'`, `'quota'`, `'disk'`, `'timeout'`, `'pin'`
-  or `'secret'`. Cancelling with `signal` stops the step and leaves nothing behind.
+  or `'secret'`. Cancelling with `signal` stops the step and leaves nothing behind. A failed build also carries `jit`
+  (the engine's last JIT messages), `net` (the guest network's counters and events), and, when the machine itself
+  stopped, `machine` (why, and its last output).
+- **RAM and vCPUs.** The default is 1536 MiB and 2 vCPUs (`ram`, `cpus`). Small builds and the proxy project in the
+  plan fit. A build that compiles a native addon or bundles a large app needed 3072 MiB and 4 vCPUs (plan 10.5); a
+  single `RUN` step must finish within `timeoutMs`, since BuildKit resumes from its cache only between steps.
+- **WebKit** builds Go and Node images and the real projects of the plan, but slower: its code space fills and new
+  code runs in the interpreter for a while (a `{t:'slow', reason:'webkit-code-space'}` event says so).
+  - In WebKit, node in `RUN` steps runs its JavaScript in the interpreter (D-553, `nodeInterpreted`, an
+    `{t:'node-interpreted'}` event): node's own JIT writes code faster than WebKit's code pool takes it, and a
+    1,600-package `pnpm` install did not end in 27 minutes. A preload bound at `/dev/.wmd-node.js` and named in
+    `NODE_OPTIONS` (`--require`) turns off V8's compilers; WebAssembly still works. The image and the cache keys do
+    not change. `nodeInterpreted: false` turns it off; `true` turns it on elsewhere.
+  - npm closes a TLS handshake that is not done 15 s after the connect. The guest network lets 4 handshakes go at
+    once, sends a ClientHello that waited on a fresh connection, and sends an unanswered one again once (D-552); it
+    dials a stream again when the remote closes it before any byte crossed (D-551).
+  - WebKit cannot read a Blob of 128 MiB or more whole (`NotReadableError`). Read the `tar` Blob a build returns
+    with `blobBytes(blob)` from `wmd-web/build`, which reads it in slices (D-554).
 - **The builder images** are built with `make web-builder` (wmd builds them; Docker Desktop is not used) into
   `web/dist/builder/builder-<arch>.tar`, and their sha256 is pinned in `src/build/pin.js`. `make web-release` puts
   them next to the release tgz.

package/RELEASE.json

@@ -1,66 +1,50 @@
 {
   "name": "wmd-web",
-  "version": "0.4.0",
-  "commit": "cb2d3874a3d65236337c76c3c59147fe152a010c",
-  "previous": "wmd-web-v0.1.0",
+  "version": "0.5.1",
+  "commit": "f0089e9f12e73d951b5a3a47f962764edc1c39a3",
+  "previous": "wmd-web-v0.5.0",
   "decisions": [
-    "D-486",
-    "D-488",
-    "D-494",
-    "D-498",
-    "D-505",
-    "D-506",
-    "D-507",
-    "D-509",
-    "D-511",
-    "D-516",
-    "D-518",
-    "D-531",
-    "D-533",
-    "D-539",
-    "D-542",
-    "D-544",
-    "D-545",
-    "D-547",
-    "D-548"
+    "D-552",
+    "D-554"
   ],
-  "vmSources": "7a707b82c43d3f1eb999476cad68e7ce79cc9cb5",
+  "vmSources": "543f2ee1a363719c2a973fff350d54a54d325286",
   "guest": {
     "arch": "arm64",
     "files": {
       "vm.wasm": {
-        "sha256": "40299e37e788189e6d3bea9d9cdaefa241a5acf27931e6aae14117c100b036ec",
-        "bytes": 1561499
+        "sha256": "73bee71d7453a130f7991c7cead3bc12787050116217f635f9f523f584d2577d",
+        "bytes": 1562868
       },
       "Image": {
-        "sha256": "1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b",
+        "sha256": "6952a8d4f1bafec9c3e3b17e2c4af20ab5de8a7086956d087979804be2a01353",
         "bytes": 39049224
       },
       "Image-4k": {
-        "sha256": "6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b",
+        "sha256": "47a1dcf06bcd1cc31fb4f02adbd1e035bc9e9a243fe57f3efd71b4a287581de2",
         "bytes": 38828040
       },
       "initramfs.cpio.gz": {
-        "sha256": "f008aa9e0f6b4e913242f9982d3b194dba7dc39042a24bc89be64630672125af",
-        "bytes": 4685204
+        "sha256": "efa6eb2338cb01d87eeebce8035ed7a26b00edcfc3b9db963b38272715d2ee9d",
+        "bytes": 4683846
       }
     }
   },
   "files": {
-    "README.md": "3c12ca29aab5160b955ba3f50035f19be1dbf0278c4fb006e253a37ac9efbe6a",
+    "README.md": "6962e410444bcf44ba6dd8498673e2de17ee4f8b96e1d29044a55b8f0dbcd8de",
     "export-page/coi-sw.js": "0300e670b413cb2e0aae35843fc583546bbf6687df971ab6df41cfa3b0d494d7",
     "export-page/index.html": "f5450c89e63601bbbbffef421f03491b231d1f143070012c1c5315d4b0059f43",
     "export-page/run.css": "b57adf1e76553549c1f89ff157b7973409dedc21889cd3932948cd05b51adc6f",
     "export-page/run.js": "717636a632a40469106ddf1383e198eaac5d83651b5866d0699ec04b82462b7f",
-    "guest/arm64/Image": "1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b",
-    "guest/arm64/Image-4k": "6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b",
-    "guest/arm64/initramfs.cpio.gz": "f008aa9e0f6b4e913242f9982d3b194dba7dc39042a24bc89be64630672125af",
-    "guest/arm64/vm.wasm": "40299e37e788189e6d3bea9d9cdaefa241a5acf27931e6aae14117c100b036ec",
-    "package.json": "778ef2457ab01f302387c811d796b7c61f97feb095725da285655629965f2b5e",
+    "guest/arm64/Image": "6952a8d4f1bafec9c3e3b17e2c4af20ab5de8a7086956d087979804be2a01353",
+    "guest/arm64/Image-4k": "47a1dcf06bcd1cc31fb4f02adbd1e035bc9e9a243fe57f3efd71b4a287581de2",
+    "guest/arm64/initramfs.cpio.gz": "efa6eb2338cb01d87eeebce8035ed7a26b00edcfc3b9db963b38272715d2ee9d",
+    "guest/arm64/vm.wasm": "73bee71d7453a130f7991c7cead3bc12787050116217f635f9f523f584d2577d",
+    "package.json": "7d43c05047d0a3ea56b4a7709f48a60de32e15c3bdf328d55435c46637981b40",
     "src/base.js": "982f8e46dbb727ec542a494ebdb884ccd19a31ff4e091425b2d1fc01b0d4754a",
-    "src/boot.js": "62f5cf901d797d53bef584b3c6c0c7fb1a4cc8e2450e9570d6289c72036ed025",
-    "src/build/builder.js": "751ea022e1f6f0c6ab79bd4efd6cfe0d705268ff7f699af87dcc64d1871ccc52",
-    "src/build/context.js": "4baf1884d366455f3f76f397575d515671585ebf5deb27bc35bc2b1dd9c6bc11",
+    "src/blob.js": "bb2a2f0d87ce236280a7be2f246c926010dfe07f649c27da5e70d8eb5eaf7061",
+    "src/boot.js": "b1819d2b8abd46fd42f10a986719a311bedca705fc368a4cd19cc121b04b57d2",
+    "src/build/builder.js": "0d5355c8570b0ecedce25602ff7f60c49b7daaed669426328f05f52e26048213",
+    "src/build/context.js": "d38097c61dbdadc4d41290c010425a4422f499aba473fa2b83f2a5c41e543571",
     "src/build/dockerfile.js": "5957fe3d4ac8ff8b312f980ee04bd1d98ad4373a61f9a33e02f1ff1108b6ea53",
     "src/build/dockerignore.js": "8fc03d09da020f43ea483979b1183891b71d76f012379ab7813838ea0e53c546",
     "src/build/fast.js": "bf7373a43cce995b21751c317a58d18438f9fd2f670266b314b1151ef3cdb9e8",
@@ -93,8 +77,8 @@
     "src/host/clock.js": "aba6af3a8b1a985ae340892222e4b482d33afe20f5c8b3a4c7403ce87d1e247c",
     "src/host/ctl.js": "ca2ebbb940c9668cb0573c959cd2ff8982f96d58dfdb8dce39d304b89e033dd7",
     "src/host/disk-map.js": "3d7e0d7d488ce25fc6a0e4dc0a9b1c8fc2af96bac1c03491a1d30be392af268b",
-    "src/host/disk-worker.js": "9d470cdc72d9e1fbf955e466ec2562abe780f2c6d89174ca3e4fbb1828f8dd76",
-    "src/host/disk.js": "373e23539e3916674b7e8708d2e3ae22578214129e56936765db5c8f465cb343",
+    "src/host/disk-worker.js": "6e82bfeb4aa3f8b02bd7cb2de7427def26114e016f6b05196c8485d7fc8315ee",
+    "src/host/disk.js": "e941e62cc2d9a83582d53c80c09c5f0f682508f77390237bedff4051c489b2cd",
     "src/host/errno.js": "26021d85962084be2c43c98fb3accafc2a8f11fbf3c8a5b8d484bd6a088ae4db",
     "src/host/fs.js": "d6753f74db35b7b01baa8709fc7fdcdfc8d638dfe42f454b0a833b22111e8f4a",
     "src/host/gcnudge.js": "4a6569cb73ae24a11fc4ac8c467380bd9efba0bb5fa9867b797bc4f153ab15ab",
@@ -103,7 +87,7 @@
     "src/host/jit-hub-core.js": "f8d39dd0e4cbd42568bf56a8a7105f321d33bcf2d5faf8756d45a1ed692bd344",
     "src/host/jit-hub.js": "cf3fc9f9d4e3fb50c41d6de5e2523f380d3c6c1feaf61c15d90cde89aec2f4c3",
     "src/host/jit-snap.js": "3932f36047c9edeb7f59282d1c37fef19c8f5a9fe4bd0192fb93aa64030fe51d",
-    "src/host/jit.js": "cb39f90df63752a7d4077f6972731cf4b1d8026cb4ca594e32f9203be6fe1edb",
+    "src/host/jit.js": "72ef7334506fde2b3724cee93b52a987fbd6d45473b8de93d914db8a80ae9e17",
     "src/host/layout.js": "d5bc5aa9292630b204438df5d6278fda6daa2a65acc061e1c57e0272b32fecc8",
     "src/host/monitor.js": "ae1741ba35668bcc6192642246494433b175668933921634c699af5a0a69acc0",
     "src/host/p9mount.js": "43246601fc1e27d5294742e4482ed166a50c665cb5f526c3646a47dd27e06150",
@@ -113,14 +97,14 @@
     "src/host/wasi.js": "fbafa239b8fb4083f7e8c910b1adab38901e8e7cffd247284a9abb2847eaf4bd",
     "src/host/wmd-imports.js": "3a00562f7c9c7787ab3841b14dde5d903a78f43ed19c43a2f5e31fd8a14c09d2",
     "src/host/yamux.js": "f4044f14f7e836224bdb4b1e53e59c565ee478b83b2e428fffb9f7a048768f79",
-    "src/image-files.js": "7bac16bd278d17ba291dfe4771564627b5efbdda927b429018a75e8b906dd4cd",
+    "src/image-files.js": "19f90a7d3c8accae5093e53bb85fe7e158ac1dd18866283722171eccb1d7e37a",
     "src/net/dhcp-dns.js": "fdfa7c7d5cbbb4a8774b7dee195bf2df22a0c3fe8f20ac9694b24485fa8f25d5",
     "src/net/net-worker.js": "0a2ce998f69807e386ded7ff1bee53687a1d59ed76d7ef46e2eb6f958b14ad38",
     "src/net/packet.js": "d658f26fa84324a88d7dbc26af9c4a9c10e977c3060f16f7db431496eacd1127",
     "src/net/peer.js": "d83cf85639f6197fe26c877eb79f8c9afc017bf2362dc2b22a8b6bd0f45f3f19",
     "src/net/proto.js": "abfc8cd6f4331cbc98fcb1b0bba0795ff3042cc7db10476673ead1723249eee0",
     "src/net/relay-client.js": "4101de76dc0251db46839dc786cce677e4020795adf022bdce14bcf7edf8f864",
-    "src/net/stack.js": "cd451cce7130f08e59cc19c961d2a5fca8416e577a40df1419ab6d5046018891",
+    "src/net/stack.js": "b721e4e116b0611abfe8ec1c6f2e212efd89a11f745fef2222ca5de474d8c0a9",
     "src/net/switch.js": "bb047fb1ceeb10a84b5475916ead916362accf0e6f42a86561653dfad0c6c50d",
     "src/net/wisp-client.js": "653961c4b735f2f74e5ea89e75c387f15ec8b8d9ed4c45d06c7d6b2a04968d42",
     "src/net/wisp-transport.js": "2b71201f3c686078c867e8995bf5c1cd6d47a0d415f476f3a33fbd5729a55f74",
@@ -128,7 +112,7 @@
     "src/registry/client.js": "fd38550e9afd82c5cb498513d245c33357a71697b3b2e6dd78538934bcc9d879",
     "src/registry/converter.js": "1fb8294fd5275f6772c27130cc9ba70bd46fafaf440884d42448036618229f37",
     "src/registry/hub.js": "cd672323bd1f96bd1d7e9eed80d4f36891e919e456fc0deb378fa0c7e3b6f249",
-    "src/registry/load-tar.js": "9dd2b5d8884c64247339b141ab5a57b8ab1a2e007c32aa45b1103b37405b0ed7",
+    "src/registry/load-tar.js": "219d6701f5c938e27e3ca36ade8fcea8114a5f2b1511496f0dbd60137290867f",
     "src/registry/oci.js": "6a51ea16b34a65c0e732730f3e07d7ad16c50f6943a486ec05ca6aa3e98676a1",
     "src/registry/pack-worker.js": "fd6798851cb7815dacfcb67fc78c41654209c5504d8fd343aae524e99de1c741",
     "src/registry/pull.js": "24f2e01a76c9f7f27896af69b989f8c97f0ca31712b3e8110aaa5b5e3148b04b",

package/guest/arm64/Image

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: package/guest/arm64/Image
 size: 39049224 bytes
-sha256: 1d296c86b1fee0a10d0c2571b7e19854fe85ca0250cc9fadfe75eb268e48be7b
+sha256: 6952a8d4f1bafec9c3e3b17e2c4af20ab5de8a7086956d087979804be2a01353

package/guest/arm64/Image-4k

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: package/guest/arm64/Image-4k
 size: 38828040 bytes
-sha256: 6296e3d9fc7bb06a26340bb4453b5f0970d6b55452903546ba2c762820e2df5b
+sha256: 47a1dcf06bcd1cc31fb4f02adbd1e035bc9e9a243fe57f3efd71b4a287581de2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@threadedstack/wmd-web",
-  "version": "0.1.18",
+  "version": "0.1.20",
   "license": "MIT",
   "homepage": "https://github.com/threadedstack/wmd-releases",
   "repository": {
@@ -9,9 +9,9 @@
   },
   "description": "Browser host for wmd's guest vm.wasm (Web Workers + shared memory + WASI preview1)",
   "wmd": {
-    "tag": "v0.1.18",
-    "commit": "cb2d3874a3d65236337c76c3c59147fe152a010c",
-    "web": "0.4.0"
+    "tag": "v0.1.20",
+    "commit": "f0089e9f12e73d951b5a3a47f962764edc1c39a3",
+    "web": "0.5.1"
   },
   "type": "module",
   "main": "./src/boot.js",

package/relay/hosted.ts

@@ -18,6 +18,7 @@ import { Bucket, type Session, type Stream } from './engine.ts'
 import { type Addr, type Cidr, addrString, cidrContains, hostedBlockedReason, parseCidr, parseIP } from './policy.ts'
 
 const HOUR = 3_600_000
+const MINUTE = 60_000
 
 export type HostedOptions = {
   abuseContact: string
@@ -26,11 +27,12 @@ export type HostedOptions = {
   now?: () => number // ms since the epoch (tests)
   warn?: (line: string) => void // one-line operator notices (stderr)
   sites?: string[] // A: page origin patterns served (required: none listed serves no page)
-  sitesPerAddrPerHour?: number // B: different sites per address group per hour (200: plan §16); a publish identity three times that
+  sitesPerAddrPerHour?: number // B: different sites per address group per hour (800)
   sitesPerNetPerHour?: number // B: per network (800)
   monthBytes?: number // C: bytes relayed per UTC month (the operator's figure from the provider's free outbound data); 0 = no cap
   usageFile?: string // C: the month's counter survives a restart here
   dialLimitMs?: number // D, G: the engine's dial limit (connectTimeoutMs, 10 s)
+  failedDialsPerMin?: number // D: failed connections per address group within a minute before any is slowed (20)
   fdLimit?: number // E: the open-file limit (read from the host when absent)
   fdOpen?: number // E: descriptors open at start (read from the host when absent)
   dnsServers?: number // E: resolvers the relay asks (each may hold a UDP and a TCP socket)
@@ -57,7 +59,6 @@ export type Handle = { ip: string; grp: string; origin: string; end: (why: Why)
 /** What a session holds from the governor until it ends. */
 export type Admission = {
   grp: string
-  publish: boolean
   net: string
   handle: Handle
   fair: Bucket | null
@@ -257,10 +258,11 @@ export function createGovernor(o: HostedOptions) {
   const warn = o.warn ?? ((m: string) => void process.stderr.write(`${m}\n`))
   const host = o.fdLimit === undefined || o.fdOpen === undefined ? hostLimits() : null
   const c = {
-    sitesPerAddrPerHour: o.sitesPerAddrPerHour ?? 200,
+    sitesPerAddrPerHour: o.sitesPerAddrPerHour ?? 800,
     sitesPerNetPerHour: o.sitesPerNetPerHour ?? 800,
     monthBytes: o.monthBytes ?? 0,
     dialLimitMs: o.dialLimitMs ?? 10_000,
+    failedDialsPerMin: o.failedDialsPerMin ?? 20,
     fdLimit: o.fdLimit ?? host!.fdLimit,
     fdOpen: o.fdOpen ?? host!.fdOpen,
     dnsServers: o.dnsServers ?? getServers().length,
@@ -404,7 +406,7 @@ export function createGovernor(o: HostedOptions) {
     return m
   }
   const oldest = (m: Map<string, number>) => Math.min(...m.values()) + HOUR
-  function sites(grp: string, host: string, publish: boolean): Why | null {
+  function sites(grp: string, host: string): Why | null {
     const t = now()
     const site = host.toLowerCase().replace(/\.$/, '')
     const net = coarseKey(grp)
@@ -415,9 +417,8 @@ export function createGovernor(o: HostedOptions) {
     fresh(a, t)
     fresh(n, t)
     if (!a.has(site)) {
-      const max = publish ? 3 * c.sitesPerAddrPerHour : c.sitesPerAddrPerHour
-      if (a.size >= max) {
-        return { rule: 'too-many-sites', words: `${max} different sites in an hour from this address${publish ? ' while publishing' : ''}: a new site waits until one ages out; sites already reached keep working`, until: oldest(a) }
+      if (a.size >= c.sitesPerAddrPerHour) {
+        return { rule: 'too-many-sites', words: `${c.sitesPerAddrPerHour} different sites in an hour from this address: a new site waits until one ages out; sites already reached keep working`, until: oldest(a) }
       }
       if (!n.has(site) && n.size >= c.sitesPerNetPerHour) {
         return { rule: 'too-many-sites', words: `${c.sitesPerNetPerHour} different sites in an hour from this network: a new site waits until one ages out; sites already reached keep working`, until: oldest(n) }
@@ -429,12 +430,16 @@ export function createGovernor(o: HostedOptions) {
   }
 
   // ---- D
-  const dials = new Map<string, { w: number; gate: number; told: boolean }>()
+  const dials = new Map<string, { w: number; gate: number; told: boolean; fails: number[] }>()
   const dialOf = (grp: string) => {
     let d = dials.get(grp)
-    if (!d) dials.set(grp, (d = { w: 0, gate: 0, told: false }))
+    if (!d) dials.set(grp, (d = { w: 0, gate: 0, told: false, fails: [] }))
     return d
   }
+  const recentFails = (d: { fails: number[] }, t: number) => {
+    while (d.fails.length && t - d.fails[0] >= MINUTE) d.fails.shift()
+    return d.fails.length
+  }
   function dialGate(s: Session): Why | { waitMs: number } | null {
     const d = dials.get(grpOf(s))
     if (!d || !d.w) return null
@@ -454,7 +459,7 @@ export function createGovernor(o: HostedOptions) {
   const grpOf = (s: Session): string => s.hx?.grp ?? s.p.grp
 
   /** A session from `ip` (address group `grp`) with this page origin is about to open. */
-  function admitSession(i: { ip: string; grp: string; origin: string; publish?: boolean; end: (why: Why) => void; tell?: Tell; claimed?: boolean }): { ok: true; adm: Admission } | ({ ok: false } & Why) {
+  function admitSession(i: { ip: string; grp: string; origin: string; end: (why: Why) => void; tell?: Tell; claimed?: boolean }): { ok: true; adm: Admission } | ({ ok: false } & Why) {
     const no = (w: Why) => ({ ok: false as const, ...w })
     const denied = deniedClient(i.ip, i.origin)
     if (denied) return no(denied)
@@ -469,7 +474,7 @@ export function createGovernor(o: HostedOptions) {
     live.add(handle)
     let done = false
     const adm: Admission = {
-      grp: i.grp, publish: !!i.publish, net: coarseKey(i.grp), handle, fair: c.linkBytesPerSec ? new Bucket(Infinity) : null, sent: 0, waiting: false,
+      grp: i.grp, net: coarseKey(i.grp), handle, fair: c.linkBytesPerSec ? new Bucket(Infinity) : null, sent: 0, waiting: false,
       release() {
         if (done) return
         done = true
@@ -490,7 +495,7 @@ export function createGovernor(o: HostedOptions) {
   function admitConnect(s: Session, host: string, _port: number): Why | { waitMs: number } | null {
     const closed = notAccepting()
     if (closed) return closed
-    const crawl = sites(grpOf(s), host, !!s.hx?.publish)
+    const crawl = sites(grpOf(s), host)
     if (crawl) return crawl
     return dialGate(s)
   }
@@ -506,15 +511,22 @@ export function createGovernor(o: HostedOptions) {
   /** D: a dial was refused, found no route, timed out or named a host that does not exist, after `ms`. */
   function dialFailed(s: Session, ms: number) {
     const d = dialOf(grpOf(s))
+    const t = now()
+    d.fails.push(t)
+    if (d.fails.length > c.failedDialsPerMin) d.fails.shift()
+    if (!d.w && recentFails(d, t) < c.failedDialsPerMin) return
     d.w = Math.min(c.dialLimitMs, Math.max(1, Math.round(ms), d.w * 2))
   }
   function dialOk(s: Session) {
     const d = dials.get(grpOf(s))
     if (!d) return
+    if (!d.w) return
     d.w = Math.floor(d.w / 2)
     if (!d.w) {
-      dials.delete(grpOf(s))
+      d.gate = 0
       if (d.told) s.hx?.handle.tell?.(3, 'failed-dials', 0, '')
+      d.told = false
+      if (!recentFails(d, now())) dials.delete(grpOf(s))
     }
   }
   const dialWait = (s: Session) => dials.get(grpOf(s))?.w ?? 0
@@ -618,12 +630,13 @@ export function createGovernor(o: HostedOptions) {
   ticker.unref?.()
   const saver = o.usageFile ? setInterval(saveUsage, 15_000) : null
   saver?.unref?.()
-  const sweep = setInterval(() => {
+  function sweep() {
     const t = now()
     for (const m of [addrSites, netSites]) for (const [k, x] of m) if (!fresh(x, t).size) m.delete(k)
-    for (const [k, d] of dials) if (d.gate + HOUR < t) dials.delete(k)
-  }, 60_000)
-  sweep.unref?.()
+    for (const [k, d] of dials) if (d.gate + HOUR < t && !recentFails(d, t)) dials.delete(k)
+  }
+  const sweeper = setInterval(sweep, 60_000)
+  sweeper.unref?.()
 
   const state = () => {
     roll()
@@ -686,6 +699,7 @@ export function createGovernor(o: HostedOptions) {
     bytes,
     limitWhy,
     tick,
+    sweep,
     slowed,
     take,
     waitMs,
@@ -697,7 +711,7 @@ export function createGovernor(o: HostedOptions) {
       (c.monthBytes ? `; ${Math.round(c.monthBytes / 1e9)} GB a month, heavy users slowed first when the month runs ahead` : ''),
     stop() {
       clearInterval(ticker)
-      clearInterval(sweep)
+      clearInterval(sweeper)
       if (saver) clearInterval(saver)
       saveUsage()
     },

package/relay/main.ts

@@ -79,7 +79,7 @@ const FLAG_KEYS = ['host', 'port', 'origins', 'keys', 'tlsCert', 'tlsKey', 'loca
 const CAP_KEYS = ['maxConns', 'maxSessions', 'maxSessionsPerGroup', 'opensPerMin', 'connectsPerMin', 'dnsPerMin', 'rateDown', 'rateUp', 'maxBytes', 'selfAddrs', 'dnsServers', 'connectTimeoutMs', 'dnsTimeoutMs'] as const
 const WISP_KEYS = ['path', 'buffer', 'maxPayload', 'maxHeldBytes', 'maxSessionHeldBytes', 'maxHeldTotal', 'maxGroupHeldBytes', 'instancesPerHour', 'maxPrincipals', 'preAuthPerAddr', 'handshakeMs', 'streamIdleMs', 'prefixKeys'] as const
 // the numbers of the hosted rules, under "hosted" in a config file (hosted.ts HostedOptions; all whole numbers)
-const HOSTED_KEYS = ['sitesPerAddrPerHour', 'sitesPerNetPerHour'] as const
+const HOSTED_KEYS = ['sitesPerAddrPerHour', 'sitesPerNetPerHour', 'failedDialsPerMin'] as const
 // keys of the limits plan §16 removed: a config that still has them starts, with one line saying they do nothing
 const LEGACY_KEYS = new Set(['maxBytesPerDay', 'maxBytesPerMonth'])
 const LEGACY_HOSTED_KEYS = new Set(['groupStreams', 'groupConnectsPerMin', 'groupBytesPerHour', 'groupDnsPerMin', 'coarseSessions', 'coarseInstancesPerHour', 'coarseRateDown', 'coarseRateUp', 'originTable', 'originSessions', 'originShare', 'otherOriginSessions', 'noOriginSessions', 'originsPerAddrPerHour', 'originRateDown', 'originRateUp', 'maxStreams', 'maxRateDown', 'maxRateUp', 'destConnectsPerMin', 'destStreams', 'distinctDests', 'failedConnects', 'failedWindowMs', 'failedCoolMs'])

package/relay/relay.ts

@@ -132,7 +132,6 @@ export function createRelay(o: RelayOptions) {
         const r = health(url.pathname)
         return req.method === 'HEAD' ? new Response(null, { status: r.status, headers: r.headers }) : r // same status and headers, no body
       }
-      if (gov && wisp && url.pathname === '/publish-token') return wisp.publishToken(req, srv, url)
       if (url.pathname === '/relay') {
         const protos = req.headers.get('sec-websocket-protocol') || ''
         if (wisp && protos.includes('wisp-v2') && !protos.includes(PROTOCOL)) return new Response(`this is the wmd-relay.v1 endpoint; Wisp is at ${wisp.path}\n`, { status: 400 })

package/relay/token.ts

@@ -1,5 +1,5 @@
 // P-B10 relay: session tokens. A token is "v1.<payload>.<mac>", base64url without padding,
-// where the payload is JSON {sid, exp (ms since the epoch), uid?, grp?, kind?} and the mac is
+// where the payload is JSON {sid, exp (ms since the epoch), uid?, grp?} and the mac is
 // HMAC-SHA256 of "v1.<payload>" with the relay's secret. Whoever serves the page
 // (test/serve.ts, or wmn's server) mints them with the same secret (TokenIssuer below); the
 // browser presents one as a WebSocket subprotocol "t.<token>" (not in the URL, so it stays
@@ -9,8 +9,6 @@
 //   uid  the guest identity the relay keeps caps for, across reconnects and fresh tokens
 //        (one guest machine of one page); defaults to sid
 //   grp  the client the issuer saw (an address, a user): the relay caps sessions per grp
-//   kind "publish": a publish identity (TokenIssuer.issuePublish), which a hosted relay gives a higher sites-an-hour
-//        count (hosted.ts). Only the signature makes it one: the relay never takes a client's word for it
 //
 // Parsing is strict: a token is exactly one encoding of its payload and MAC, so the same
 // token never verifies under two spellings.
@@ -21,20 +19,19 @@ const b64u = (b: Buffer) => b.toString('base64url')
 const B64U = /^[A-Za-z0-9_-]+$/
 const ID = /^[A-Za-z0-9_.:\/-]{1,128}$/
 
-export type TokenFields = { sid?: string; uid?: string; grp?: string; kind?: string; ttlMs?: number; now?: number }
+export type TokenFields = { sid?: string; uid?: string; grp?: string; ttlMs?: number; now?: number }
 
-export function mintToken(secret: string, { sid = b64u(randomBytes(12)), uid, grp, kind, ttlMs = 60 * 60_000, now = Date.now() }: TokenFields = {}): string {
+export function mintToken(secret: string, { sid = b64u(randomBytes(12)), uid, grp, ttlMs = 60 * 60_000, now = Date.now() }: TokenFields = {}): string {
   if (!secret || secret.length < 16) throw new Error('relay secret: at least 16 characters')
   const body: Record<string, unknown> = { sid, exp: now + ttlMs }
   if (uid) body.uid = uid
   if (grp) body.grp = grp
-  if (kind) body.kind = kind
   const payload = b64u(Buffer.from(JSON.stringify(body)))
   const mac = createHmac('sha256', secret).update(`v1.${payload}`).digest()
   return `v1.${payload}.${b64u(mac)}`
 }
 
-export type TokenInfo = { sid: string; exp: number; uid?: string; grp?: string; kind?: 'publish' }
+export type TokenInfo = { sid: string; exp: number; uid?: string; grp?: string }
 
 /** The token's session, or an error string. */
 export function verifyToken(secret: string, token: string, now = Date.now()): TokenInfo | string {
@@ -55,9 +52,8 @@ export function verifyToken(secret: string, token: string, now = Date.now()): To
   }
   if (typeof info !== 'object' || !info || typeof info.sid !== 'string' || !ID.test(info.sid) || typeof info.exp !== 'number') return 'bad token'
   for (const k of ['uid', 'grp'] as const) if (info[k] !== undefined && (typeof info[k] !== 'string' || !ID.test(info[k]!))) return 'bad token'
-  if (info.kind !== undefined && info.kind !== 'publish') return 'bad token'
   if (info.exp <= now) return 'token expired'
-  return { sid: info.sid, exp: info.exp, uid: info.uid, grp: info.grp, ...(info.kind ? { kind: info.kind } : {}) }
+  return { sid: info.sid, exp: info.exp, uid: info.uid, grp: info.grp }
 }
 
 /** The key a client's limits are kept under: IPv6 addresses by their /64 (one subscriber holds 2^64 of them). */
@@ -83,7 +79,6 @@ export function clientKey(client: string): string {
  * an hour. At most `maxClients` clients are remembered (the least recently used go first).
  */
 export const MIN_TTL_MS = 60_000
-export const PUBLISH_TTL_MS = 60 * 60_000
 export class TokenIssuer {
   perMinute: number
   instancesPerHour: number
@@ -91,7 +86,6 @@ export class TokenIssuer {
   maxClients: number
   minTtlMs: number
   clients = new Map<string, { times: number[]; instances: Map<string, number> }>() // least recently used first
-  publishing = new Map<string, { instance: string; token: string; until: number }>() // per client: its live publish identity, in the order they end
   constructor(public secret: string, { perMinute = 60, instancesPerHour = 20, maxTtlMs = 60 * 60_000, maxClients = 10_000, minTtlMs = MIN_TTL_MS } = {}) {
     this.perMinute = perMinute
     this.instancesPerHour = instancesPerHour
@@ -119,27 +113,4 @@ export class TokenIssuer {
     const ttl = Math.min(Math.max(ttlMs, this.minTtlMs), this.maxTtlMs)
     return { token: mintToken(this.secret, { uid: `${key}/${instance}`, grp: key, ttlMs: ttl, now }) }
   }
-  /** A publish identity for this client, for one hour; at most one per client at a time. The page that holds it gets the
-      same token again (a reconnect); another page of the client is refused until its hour is over. When `maxClients`
-      identities are live, a new one is refused (503) until the first ends: a live one is never dropped. */
-  issuePublish(client: string, instance: string, now = Date.now()): { token: string; until: number } | { error: string; status: number; until?: number } {
-    if (!ID.test(client)) return { error: 'bad client', status: 400 }
-    if (!/^[A-Za-z0-9_-]{8,64}$/.test(instance || '')) return { error: 'instance: 8-64 characters [A-Za-z0-9_-]', status: 400 }
-    const key = clientKey(client)
-    for (const [k, p] of this.publishing) {
-      if (p.until > now) break
-      this.publishing.delete(k)
-    }
-    const live = this.publishing.get(key)
-    if (live && live.instance === instance) return { token: live.token, until: live.until }
-    if (live) return { error: `this address already has a publish identity; another can be had in ${Math.ceil((live.until - now) / 60_000)} min`, status: 429, until: live.until }
-    if (this.publishing.size >= this.maxClients) {
-      const first = this.publishing.values().next().value!
-      return { error: `the relay is giving out as many publish identities as it can hold; try again in ${Math.ceil((first.until - now) / 60_000)} min`, status: 503, until: first.until }
-    }
-    const until = now + PUBLISH_TTL_MS
-    const token = mintToken(this.secret, { uid: `${key}/publish.${instance}`, grp: key, kind: 'publish', ttlMs: PUBLISH_TTL_MS, now })
-    this.publishing.set(key, { instance, token, until })
-    return { token, until }
-  }
 }

package/relay/wisp.ts

@@ -1,8 +1,7 @@
 // W59: the Wisp v2 adapter (src/net/wisp.js, docs/wisp-wmd-extensions.md) over the relay engine.
 //
 // Who may use it (plan §5):
-//   - token: the HMAC `t.<token>` subprotocol, as wmd-relay.v1 (wmn's bundled relay); on a hosted relay, the publish
-//     identity it issues itself (relay.ts /publish-token), used only from the address it was given to
+//   - token: the HMAC `t.<token>` subprotocol, as wmd-relay.v1 (wmn's bundled relay)
 //   - open: no key (the standalone relay's default); identity = the client's instance id (0xE7)
 //     plus its address group, bandwidth shared per address
 //   - keys (optional): Wisp password auth, scrypt hashes; a relay with keys refuses keyless clients
@@ -24,7 +23,7 @@ import { type AllowEntry, type Engine, type Principal, type Session, type Shared
 import { type Governor, type Why, resolveClient } from './hosted.ts'
 import { type OwnerKey, verifySecret } from './keys.ts'
 import type { Cidr } from './policy.ts'
-import { TokenIssuer, clientKey, verifyToken } from './token.ts'
+import { clientKey, verifyToken } from './token.ts'
 
 export type WispOptions = {
   enabled?: boolean
@@ -185,32 +184,6 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     return reply(body, status)
   }
 
-  const issuer = w.hosted && o.secret ? new TokenIssuer(o.secret) : null
-
-  /** Hosted: GET /publish-token?instance=ID from a listed page gives {token, until}, a publish identity for this address
-      (token.ts issuePublish), or {error, until?} in words. The page's origin may read the answer. */
-  function publishToken(req: Request, srv: any, url: URL): Response {
-    const origin = req.headers.get('origin') || ''
-    if (!issuer || origin.length > 256 || !w.hosted!.siteAllowed(origin)) {
-      log({ ev: 'publish-token-refused', why: 'not-allowed-site' })
-      return reply('this page is not one of the sites this relay serves\n', 403)
-    }
-    const headers = { 'access-control-allow-origin': origin, vary: 'Origin', 'cache-control': 'no-store' }
-    if (req.method !== 'GET') return new Response(`GET only\n${w.hosted!.contactLine}`, { status: 405, headers })
-    const ip = clientIp(req, srv)
-    if (typeof ip !== 'string') {
-      for (const [k, v] of Object.entries(headers)) ip.headers.set(k, v)
-      return ip
-    }
-    const r = issuer.issuePublish(ip, url.searchParams.get('instance') || '')
-    if ('token' in r) {
-      log({ ev: 'publish-token', addr: clientKey(ip), origin })
-      return Response.json(r, { headers })
-    }
-    log({ ev: 'publish-token-refused', why: r.error, addr: clientKey(ip) })
-    return Response.json({ error: r.error, ...(r.until ? { until: r.until } : {}), abuseContact: w.hosted!.contact }, { status: r.status, headers })
-  }
-
   async function fetch(req: Request, srv: any, url: URL): Promise<Response | undefined> {
     const ip = clientIp(req, srv)
     if (typeof ip !== 'string') return ip
@@ -237,13 +210,11 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     let uid = ''
     let grp = ''
     let exp = Infinity
-    let publish = false
     let p: Principal | null = null
     const tok = protos.find(x => x.startsWith('t.'))
     if (tok && o.secret) {
       const info = verifyToken(o.secret, tok.slice(2))
       if (typeof info === 'string') return refuse(401, `${info}\n`, info, { origin })
-      if (w.hosted && info.grp !== addr) return refuse(401, 'this token was given to another address\n', 'token of another address', { addr })
       if (engine.hasSession(info.sid)) {
         stats.refusedSessions++
         return reply('session already connected\n', 409)
@@ -253,9 +224,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
       uid = info.uid || info.sid
       grp = info.grp || uid
       exp = info.exp
-      publish = info.kind === 'publish'
-      // a hosted publish session shares its address's bandwidth with the address's open sessions
-      const a = engine.admit(uid, grp, exp, { proto: 'wisp' }, w.hosted && publish ? sharedFor(grp) : undefined)
+      const a = engine.admit(uid, grp, exp, { proto: 'wisp' })
       if (!a.ok) return reply(a.body, a.status)
       p = a.p
     } else if (rest) {
@@ -302,7 +271,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
     const full = w.hosted?.claimHandshake() ?? null
     const headers: Record<string, string> = {}
     if (v2 && protos.length) headers['Sec-WebSocket-Protocol'] = protos.includes(SUBPROTOCOL) ? SUBPROTOCOL : protos[0]
-    const data = { kind: 'wisp', v2, mode, addr, ip, sid, uid, grp, exp, publish, p, key, origin, hs: null as any, hsTimer: null as any, pre, claimed: !!w.hosted && !full, refuse: full as Why | null, at: performance.now() }
+    const data = { kind: 'wisp', v2, mode, addr, ip, sid, uid, grp, exp, p, key, origin, hs: null as any, hsTimer: null as any, pre, claimed: !!w.hosted && !full, refuse: full as Why | null, at: performance.now() }
     const ok = srv.upgrade(req, Object.keys(headers).length ? { data, headers } : { data }) // Bun refuses an empty headers object
     if (!ok) {
       release(data)
@@ -448,7 +417,7 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
         tell(REASON_KIND.REFUSED, why.rule, why.until ?? 0, why.words)
         ws.close(4011, why.words.slice(0, 120))
       }
-      adm = w.hosted.admitSession({ ip: d.ip, grp: d.grp, origin: d.origin, publish: d.publish, end, tell, claimed: d.claimed })
+      adm = w.hosted.admitSession({ ip: d.ip, grp: d.grp, origin: d.origin, end, tell, claimed: d.claimed })
       if (!adm.ok) return refuseHs(ws, client, adm)
       d.claimed = false // the session holds the socket's count now
       d.rel = adm.adm.release
@@ -695,7 +664,6 @@ export function createWisp(engine: Engine, o: { secret?: string; port: () => num
       ws.close(1013, 'the relay is full')
     },
     fetch,
-    publishToken,
     open: open_,
     message,
     close,

package/src/blob.js

@@ -0,0 +1,12 @@
+// D-554: WebKit (Playwright's WebKit 26 here) cannot read a Blob of 128 MiB or more whole: arrayBuffer() and stream()
+// fail with NotReadableError ("The I/O read operation failed."). Slices of the same Blob read. Measured with 128, 256
+// and 400 MiB Blobs; 64 MiB read whole. Readers of large slices (a tar entry, a layer) go through here too.
+const SLICE = 16 << 20
+
+/** A Blob's bytes, read in 16 MiB slices. */
+export async function blobBytes(b) {
+  if (b.size <= SLICE) return new Uint8Array(await b.arrayBuffer())
+  const out = new Uint8Array(b.size)
+  for (let o = 0; o < b.size; o += SLICE) out.set(new Uint8Array(await b.slice(o, o + SLICE).arrayBuffer()), o)
+  return out
+}