@threadedstack/wmd-web 0.0.0-stage → 0.1.13

package/page/sw.js

@@ -0,0 +1,381 @@
+// P-B7: the service worker of the pack host (served at /sw.js, scope /).
+//
+// - /blobs/sha256/<hex> (packs): cache first, forever. A pack is put in the cache only after
+//   its bytes hash to the name in its URL; a response that does not is refused (the page
+//   gets a 502 with the reason) and nothing is cached.
+// - image indexes: network first, cached on every 200, the cache when the network fails.
+// - the page, its modules, the Workers' scripts, vm.wasm, kernel, initramfs: VERSIONED. A version is
+//   a cache (wmd-app-<n>). A navigation that finds the host's page different from the cached one
+//   (or a background refresh that found changed files) starts version n+1 (the changed files, the
+//   rest from the network on first use); the page load is pinned to the version that served its
+//   navigation, and so are its Workers (a Worker's script request names its creator, its imports
+//   name the Worker). Everything a page and its Workers load comes from one version; a version
+//   stays until no client uses it. Offline: the cache of the version the page is pinned to.
+// - POST /pull (the proxy) and other non-GET requests pass through, never cached.
+// The page's own document and modules load before this worker controls the page, so the
+// page sends their URLs once it is controlled ({t:'warm'}) and they are cached then.
+
+const BLOBS = 'wmd-blobs-v1'
+const APP_PREFIX = 'wmd-app-' // a version of the page and its files: wmd-app-<n>
+const NEXT = 'wmd-app-next' // files the background refresh found changed, moved into the next version
+const META = 'wmd-meta' // {latest, clients:{clientId: version}} in one cache entry (this worker is stopped when idle)
+
+self.addEventListener('install', () => self.skipWaiting())
+
+self.addEventListener('activate', e => {
+  e.waitUntil(
+    (async () => {
+      // the old single-cache layout (wmd-app-v1) is dropped: it has no version number
+      for (const k of await caches.keys()) if (k === 'wmd-app-v1' || k === 'wmd-app-next-v1') await caches.delete(k)
+      await self.clients.claim()
+    })(),
+  )
+})
+
+let metaP = null
+const loadMeta = () =>
+  (metaP ??= (async () => {
+    try {
+      const r = await (await caches.open(META)).match('/__meta')
+      const m = r ? await r.json() : null
+      if (m && typeof m.latest === 'number') {
+        for (const [id, v] of Object.entries(m.clients || {})) if (typeof v === 'number') m.clients[id] = { v, t: Date.now(), miss: 0 }
+        m.clients ||= {}
+        return m
+      }
+    } catch {}
+    return { latest: 0, clients: {} }
+  })())
+async function saveMeta() {
+  const m = await loadMeta()
+  const ids = Object.keys(m.clients)
+  for (const k of ids.slice(0, Math.max(0, ids.length - 300))) delete m.clients[k] // bounded
+  try {
+    await (await caches.open(META)).put('/__meta', new Response(JSON.stringify(m)))
+  } catch {}
+}
+const PIN_MIN_MS = 30_000 // a pin younger than this is never dropped: a new page is not listed by clients.matchAll for a while
+const pin = (m, id, v) => (m.clients[id] = { v, t: Date.now(), miss: 0 })
+const bucketName = n => `${APP_PREFIX}${n}`
+
+// Drops versions no client uses (and no longer the latest). Where clients cannot be listed, drops nothing.
+async function sweep() {
+  const m = await loadMeta()
+  let live
+  try {
+    live = new Set((await self.clients.matchAll({ includeUncontrolled: true, type: 'all' })).map(c => c.id))
+  } catch {
+    return
+  }
+  // a pin goes when its client was absent from two sweeps in a row and the pin is not young
+  for (const [id, p] of Object.entries(m.clients)) {
+    if (live.has(id)) p.miss = 0
+    else if (Date.now() - p.t > PIN_MIN_MS && ++p.miss >= 2) delete m.clients[id]
+  }
+  const used = new Set([m.latest, ...Object.values(m.clients).map(p => p.v)])
+  for (const k of await caches.keys()) {
+    const n = k.startsWith(APP_PREFIX) && k !== NEXT ? Number(k.slice(APP_PREFIX.length)) : NaN
+    if (Number.isInteger(n) && !used.has(n)) await caches.delete(k)
+  }
+  await saveMeta()
+}
+
+/** The version a request is served from: its client's (a page, or a Worker resolved to its page), else the latest. */
+async function versionOf(e) {
+  const m = await loadMeta()
+  const id = e.clientId
+  if (id && m.clients[id]) return m.clients[id].v
+  return m.latest
+}
+
+const hex = buf => [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
+const fail = (status, msg) => new Response(`${msg}\n`, { status, headers: { 'Content-Type': 'text/plain', 'X-Wmd-Sw': 'refused' } })
+// marks where a response came from (the page records it per pack)
+const tag = (r, src) => {
+  const h = new Headers(r.headers)
+  h.set('X-Wmd-Source', src)
+  return new Response(r.body, { status: r.status, statusText: r.statusText, headers: h })
+}
+// A full cache (quota) must not break a visit: the response is served when it cannot be kept.
+async function keep(cache, url, res) {
+  try {
+    await cache.put(url, res)
+    return true
+  } catch {
+    stats.putFailed++
+    return false
+  }
+}
+const stats = { versions: 0, putFailed: 0, blobHits: 0, blobFetches: 0, blobGzip: 0, blobRefused: 0, appNet: 0, appCache: 0, appMiss: 0 }
+
+async function blob(req, want) {
+  const cache = await caches.open(BLOBS)
+  const hit = await cache.match(req.url)
+  if (hit) {
+    stats.blobHits++
+    return tag(hit, 'cache')
+  }
+  stats.blobFetches++
+  let res
+  try {
+    res = await fetch(req.url, { cache: 'no-store' })
+  } catch (e) {
+    return fail(504, `pack ${want.slice(0, 12)}: not cached and the host is unreachable (${e?.message || e})`)
+  }
+  if (!res.ok) return res
+  if (res.headers.get('content-encoding') === 'gzip') stats.blobGzip++
+  let buf
+  try {
+    buf = await res.arrayBuffer()
+  } catch (e) {
+    return fail(502, `pack ${want.slice(0, 12)}: the transfer broke (${e?.message || e}); nothing cached`)
+  }
+  const got = hex(await crypto.subtle.digest('SHA-256', buf))
+  if (got !== want) {
+    stats.blobRefused++
+    return fail(502, `pack ${want.slice(0, 12)}: the host sent bytes that hash to ${got.slice(0, 12)}; refused, nothing cached`)
+  }
+  const headers = new Headers(res.headers)
+  headers.set('X-Wmd-Verified', `sha256:${got}`)
+  headers.set('Content-Length', String(buf.byteLength))
+  headers.delete('Content-Encoding') // the body is decoded already
+  const ok = new Response(buf, { status: 200, headers })
+  await keep(cache, req.url, ok.clone())
+  return tag(ok, 'network')
+}
+
+const sameBytes = async (a, b) => {
+  if (a.byteLength !== b.byteLength) return false
+  return hex(await crypto.subtle.digest('SHA-256', a)) === hex(await crypto.subtle.digest('SHA-256', b))
+}
+
+// Background refresh of a file served from version n: a changed file goes to NEXT (the version a later
+// navigation starts), an unchanged one is dropped.
+async function refresh(req, hit) {
+  try {
+    const res = await fetch(req, { cache: 'no-store' })
+    if (res.status !== 200 || res.type !== 'basic') return
+    stats.appNet++
+    const [a, b] = await Promise.all([hit.clone().arrayBuffer(), res.clone().arrayBuffer()])
+    if (!(await sameBytes(a, b))) await keep(await caches.open(NEXT), req.url, res.clone())
+  } catch {}
+}
+
+// A file of the page's version: its cache, else the network (kept in that version).
+async function versioned(e) {
+  const req = e.request
+  const n = await versionOf(e)
+  const m = await loadMeta()
+  const cache = await caches.open(bucketName(n || (m.latest = 1, await saveMeta(), 1)))
+  // a Worker's script: its own client (the resulting one) is pinned to its creator's version
+  if (e.resultingClientId && (req.destination === 'worker' || req.destination === 'sharedworker')) {
+    pin(m, e.resultingClientId, n || 1)
+    await saveMeta()
+  }
+  const hit = await cache.match(req.url)
+  if (hit) {
+    stats.appCache++
+    if (n === m.latest) e.waitUntil(refresh(req, hit))
+    return hit
+  }
+  try {
+    const res = await fetch(req, { cache: 'no-store' })
+    if (res.status === 200 && res.type === 'basic') {
+      stats.appNet++
+      await keep(cache, req.url, res.clone())
+    }
+    return res
+  } catch {
+    stats.appMiss++
+    return fail(504, `offline and not cached: ${new URL(req.url).pathname}`)
+  }
+}
+
+// Image indexes: network first (current whenever the host is up), the latest version's cache offline.
+async function index(req) {
+  const m = await loadMeta()
+  const cache = await caches.open(bucketName(m.latest || 1))
+  try {
+    const res = await fetch(req, { cache: 'no-store' })
+    if (res.status === 200 && res.type === 'basic') {
+      stats.appNet++
+      await keep(cache, req.url, res.clone())
+    }
+    return res
+  } catch {
+    // an index is data, not code: the newest version that has it
+    const names = (await caches.keys()).filter(k => k.startsWith(APP_PREFIX) && k !== NEXT).sort((a, b) => Number(b.slice(APP_PREFIX.length)) - Number(a.slice(APP_PREFIX.length)))
+    for (const name of names) {
+      const hit = await (await caches.open(name)).match(req.url)
+      if (hit) {
+        stats.appCache++
+        return hit
+      }
+    }
+    stats.appMiss++
+    return fail(504, `offline and not cached: ${new URL(req.url).pathname}`)
+  }
+}
+
+// A navigation. The host's page is compared with the latest version's: if it differs, or a refresh
+// found changed files, a new version starts (the changed files; the rest come from the network on
+// first use), so the page never loads next to modules of another version. The new page load is
+// pinned to the version that served it.
+// Builds version n from the one before: every file of the previous version is revalidated (a
+// conditional request when it has an ETag or Last-Modified, so unchanged files cost no download),
+// kept as it was when the host does not answer, dropped when the host says 404. The page goes in
+// last. Only a complete version is switched to (m.latest): if anything cannot be stored the new
+// version is deleted and the old one keeps serving.
+async function startVersion(m, req, res) {
+  const old = m.latest || 0
+  const n = old + 1
+  const cache = await caches.open(bucketName(n))
+  try {
+    if (old) {
+      const oldCache = await caches.open(bucketName(old))
+      const keys = (await oldCache.keys()).filter(k => new URL(k.url).pathname !== new URL(req.url).pathname)
+      let i = 0
+      const work = async () => {
+        for (let k = keys[i++]; k; k = keys[i++]) {
+          const was = await oldCache.match(k.url)
+          if (!was || (was.headers.get('content-type') || '').startsWith('text/html')) continue
+          const h = new Headers()
+          if (was.headers.get('etag')) h.set('If-None-Match', was.headers.get('etag'))
+          else if (was.headers.get('last-modified')) h.set('If-Modified-Since', was.headers.get('last-modified'))
+          let r = null
+          try {
+            r = await fetch(k.url, { cache: 'no-store', headers: h })
+          } catch {}
+          if (r && r.status === 404) continue
+          if (r && r.status === 200 && r.type === 'basic') {
+            stats.appNet++
+            if (!(await keep(cache, k.url, r))) throw new Error('cache full')
+          } else if (!(await keep(cache, k.url, was))) throw new Error('cache full') // 304, or the host did not answer
+        }
+      }
+      await Promise.all([work(), work(), work(), work()])
+    }
+    if (!(await keep(cache, req.url, res.clone()))) throw new Error('cache full')
+  } catch {
+    await caches.delete(bucketName(n))
+    return 0
+  }
+  m.latest = n
+  stats.versions++
+  const next = await caches.open(NEXT)
+  for (const k of await next.keys()) await next.delete(k) // what the refresh found is in the version now
+  return n
+}
+
+// A navigation. The host's page is compared with the latest version's: if it differs, or a refresh
+// found changed files, a new version is built (startVersion) before the page is handed out, so the page
+// never loads next to modules of another version. The page load is pinned to the version that served it.
+async function navigate(e) {
+  const req = e.request
+  const m = await loadMeta()
+  let res = null
+  try {
+    res = await fetch(req)
+  } catch {}
+  let n = m.latest || 0
+  if (res && res.status === 200 && res.type === 'basic') {
+    stats.appNet++
+    const have = n ? await (await caches.open(bucketName(n))).match(req.url, { ignoreSearch: true }) : null // the same page with another query is the same version
+    const nextKeys = await (await caches.open(NEXT)).keys()
+    const changed = !have || !(await sameBytes(await have.clone().arrayBuffer(), await res.clone().arrayBuffer()))
+    if (!n || changed || nextKeys.length) {
+      const made = await startVersion(m, req, res)
+      if (made) n = made
+      else if (have) {
+        // the update could not be stored: the old version keeps serving, page included
+        stats.appCache++
+        if (e.resultingClientId) pin(m, e.resultingClientId, n)
+        await saveMeta()
+        return have
+      }
+    } else await keep(await caches.open(bucketName(n)), req.url, res.clone())
+  } else if (!res) {
+    n = n || 1
+    const cache = await caches.open(bucketName(n))
+    const hit = (await cache.match(req.url)) || (await cache.match(req.url, { ignoreSearch: true }))
+    if (hit) {
+      stats.appCache++
+      if (e.resultingClientId) (pin(m, e.resultingClientId, n), await saveMeta())
+      return hit
+    }
+    stats.appMiss++
+    return fail(504, `offline and not cached: ${new URL(req.url).pathname}`)
+  }
+  if (e.resultingClientId) pin(m, e.resultingClientId, n)
+  await saveMeta()
+  e.waitUntil(sweep())
+  return res
+}
+
+self.addEventListener('fetch', e => {
+  const req = e.request
+  const url = new URL(req.url)
+  if (url.origin !== self.location.origin || req.method !== 'GET') return
+  const m = url.pathname.match(/^\/blobs\/sha256\/([0-9a-f]{64})$/)
+  if (m) return e.respondWith(blob(req, m[1]))
+  if (url.pathname === '/sw.js' || url.pathname === '/_store') return
+  if (req.mode === 'navigate') return e.respondWith(navigate(e))
+  if (url.pathname.startsWith('/idx/')) return e.respondWith(index(req))
+  e.respondWith(versioned(e))
+})
+
+// The page asks for its module graph to be cached, for the stats, and to empty the caches.
+self.addEventListener('message', e => {
+  const { t, urls, id } = e.data || {}
+  const reply = x => e.source?.postMessage({ id, ...x })
+  if (t === 'warm') {
+    e.waitUntil(
+      (async () => {
+        const m = await loadMeta()
+        const ver = (e.source && m.clients[e.source.id]?.v) || m.latest || 1
+        if (!m.latest) {
+          m.latest = 1
+          await saveMeta()
+        }
+        const cache = await caches.open(bucketName(ver))
+        let n = 0
+        for (const u of urls || []) {
+          const url = new URL(u, self.location.origin)
+          if (url.origin !== self.location.origin || url.pathname.startsWith('/blobs/')) continue
+          try {
+            const r = await fetch(url, { cache: 'no-store' })
+            if (r.status === 200) {
+              // a file the page's version already has is not replaced under it
+              if (!(await cache.match(url.href)) && (await keep(cache, url.href, r))) n++
+            }
+          } catch {}
+        }
+        reply({ t: 'warmed', n })
+      })(),
+    )
+  } else if (t === 'stats') {
+    e.waitUntil(
+      (async () => {
+        const out = { ...stats, caches: {} }
+        for (const name of [BLOBS, bucketName((await loadMeta()).latest || 1)]) {
+          const c = await caches.open(name)
+          const keys = await c.keys()
+          let bytes = 0
+          for (const k of keys) {
+            const r = await c.match(k)
+            bytes += Number(r?.headers.get('content-length') || 0) || (await r.arrayBuffer()).byteLength
+          }
+          out.caches[name] = { entries: keys.length, bytes }
+        }
+        reply({ t: 'stats', stats: out })
+      })(),
+    )
+  } else if (t === 'clear') {
+    e.waitUntil(
+      (async () => {
+        for (const k of await caches.keys()) await caches.delete(k)
+        reply({ t: 'cleared' })
+      })(),
+    )
+  }
+})

package/relay/control.ts

@@ -0,0 +1,139 @@
+// W59 S7: the control socket of a hosted relay (plan §15, H8): `wmd-relay ctl stop-new|resume|drain|status`.
+//
+// A Unix socket, never TCP, in a directory only the relay's user can enter (0700, owned by it) with the socket at 0600;
+// those permissions are the whole guarantee. The directory is never under /tmp (or any shared temp directory): another
+// user could have planted it. Peer uid: Bun's socket object exposes no peer credentials (its fields are fd, localAddress,
+// remoteAddress, ...; no SO_PEERCRED / getpeereid), so there is no uid check beyond the permissions (H12, recorded
+// in docs/relay-hosted.md). One request per connection: a line with the command, a line of JSON back.
+
+import { chmodSync, existsSync, lstatSync, mkdirSync, realpathSync, rmSync, statSync } from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+
+export const COMMANDS = ['stop-new', 'resume', 'drain', 'status'] as const
+export type Command = (typeof COMMANDS)[number]
+const SOCKET = 'ctl.sock'
+const TMP_ROOTS = ['/tmp', '/private/tmp', '/var/tmp', '/private/var/tmp']
+
+export class ControlError extends Error {}
+
+/** The directory is refused (in words) when it could be tampered with; made 0700 when it does not exist. */
+export function checkControlDir(dir: string): string {
+  dir = resolve(dir)
+  // the real location of the nearest existing ancestor, so a symlink into /tmp is not a way round
+  let anc = dir
+  while (!existsSync(anc) && dirname(anc) !== anc) anc = dirname(anc)
+  if (existsSync(anc) && !statSync(anc).isDirectory()) {
+    throw new ControlError(`the control directory ${dir} cannot be made: ${anc} is a file, not a directory; give a path that is a directory (for example /run/wmd-relay)`)
+  }
+  const real = join(realpathSync(anc), dir.slice(anc.length))
+  for (const t of TMP_ROOTS) {
+    if (real === t || real.startsWith(`${t}/`) || dir === t || dir.startsWith(`${t}/`)) {
+      throw new ControlError(`the control directory ${dir} is under ${t}, where other users can plant files; use a directory of the relay's user (for example /run/wmd-relay)`)
+    }
+  }
+  mkdirSync(dir, { recursive: true, mode: 0o700 })
+  const st = statSync(dir)
+  if (typeof process.getuid === 'function' && st.uid !== process.getuid()) {
+    throw new ControlError(`the control directory ${dir} belongs to another user (uid ${st.uid}); it must belong to the relay's user`)
+  }
+  if (st.mode & 0o077) {
+    throw new ControlError(`the control directory ${dir} is open to other users (mode ${(st.mode & 0o777).toString(8)}); run: chmod 700 ${dir}`)
+  }
+  return join(dir, SOCKET)
+}
+
+export type ControlHandlers = Record<Command, () => unknown | Promise<unknown>>
+
+/** Listen on the control socket. Resolves to its path and a stop function. */
+export async function startControl(dir: string, handlers: ControlHandlers): Promise<{ path: string; stop: () => void }> {
+  const path = checkControlDir(dir)
+  if (path.length > 100) throw new ControlError(`the control socket path is too long for a Unix socket (${path.length} characters); pick a shorter --control-dir`)
+  if (existsSync(path)) {
+    if (!lstatSync(path).isSocket()) throw new ControlError(`${path} exists and is not a socket`)
+    const alive = await ctl(path, 'status').then(() => true, () => false)
+    if (alive) throw new ControlError(`another wmd-relay is using ${path}`)
+    rmSync(path) // left by a relay that was killed
+  }
+  const um = process.umask(0o177)
+  let listener: any
+  try {
+    listener = Bun.listen({
+      unix: path,
+      socket: {
+        open(sock: any) {
+          sock.data = ''
+        },
+        async data(sock: any, chunk: Uint8Array) {
+          sock.data += Buffer.from(chunk).toString()
+          if (sock.data.length > 256) return sock.end()
+          const nl = sock.data.indexOf('\n')
+          if (nl < 0) return
+          const cmd = sock.data.slice(0, nl).trim()
+          sock.data = ''
+          let out: unknown
+          try {
+            if (!(COMMANDS as readonly string[]).includes(cmd)) out = { ok: false, error: `unknown command '${cmd}' (${COMMANDS.join(', ')})` }
+            else out = { ok: true, ...((await handlers[cmd as Command]()) as object) }
+          } catch (e: any) {
+            out = { ok: false, error: String(e?.message || e) }
+          }
+          sock.write(JSON.stringify(out) + '\n')
+          sock.end()
+        },
+        error() {},
+      },
+    })
+  } finally {
+    process.umask(um)
+  }
+  chmodSync(path, 0o600)
+  return {
+    path,
+    stop() {
+      listener.stop(true)
+      rmSync(path, { force: true })
+    },
+  }
+}
+
+/** One command to the socket at `path`; the reply object. Rejects when nothing listens or the socket cannot be opened. */
+export function ctl(path: string, cmd: Command | string): Promise<any> {
+  return new Promise((res, rej) => {
+    let buf = ''
+    let done = false
+    const fin = (f: () => void) => {
+      if (!done) {
+        done = true
+        clearTimeout(t)
+        f()
+      }
+    }
+    const t = setTimeout(() => fin(() => rej(new ControlError('no answer from the relay in 10 s'))), 10_000)
+    Bun.connect({
+      unix: path,
+      socket: {
+        open(s: any) {
+          s.write(`${cmd}\n`)
+        },
+        data(_s: any, chunk: Uint8Array) {
+          buf += Buffer.from(chunk).toString()
+        },
+        close() {
+          fin(() => {
+            try {
+              res(JSON.parse(buf))
+            } catch {
+              rej(new ControlError('the relay sent no answer'))
+            }
+          })
+        },
+        error(_s: any, e: Error) {
+          fin(() => rej(e))
+        },
+        connectError(_s: any, e: Error) {
+          fin(() => rej(e))
+        },
+      },
+    }).catch(e => fin(() => rej(e)))
+  })
+}