@threadedstack/wmd-web 0.0.0-stage → 0.1.12

package/src/registry/converter.js

@@ -0,0 +1,68 @@
+// W71: the page's side of pack-worker.js: one Worker that converts layers to packs with wmdpack.wasm.
+//
+//   const conv = await createConverter(wmdpackWasm)          // a URL, bytes or a WebAssembly.Module
+//   const p = await conv.convert(layerBytes, belowShas)       // {sha, size, bytes, ms, fileBytes, wasmBytes}
+//   await conv.add(sha, bytes)                                // a pack the page already has, for later BELOWs
+//   conv.close()
+
+let seq = 0
+
+async function toModule(src, fetchFn) {
+  if (src instanceof WebAssembly.Module) return src
+  if (src instanceof Uint8Array || src instanceof ArrayBuffer) return WebAssembly.compile(src)
+  if (typeof src === 'string' || src instanceof URL) {
+    const r = await fetchFn(src)
+    if (!r.ok) throw new Error(`wmdpack.wasm: ${src}: HTTP ${r.status}`)
+    return WebAssembly.compile(await r.arrayBuffer())
+  }
+  throw new TypeError('createConverter: wmdpack.wasm as a URL, bytes or a WebAssembly.Module')
+}
+
+/**
+ * @param {string|URL|Uint8Array|ArrayBuffer|WebAssembly.Module} wasm wmdpack.wasm (`make wmdpack-wasm`)
+ * @param {{maxFileBytes?: number, fetch?: typeof fetch}} [o] maxFileBytes: the largest pack (default 4 GiB)
+ */
+export async function createConverter(wasm, { maxFileBytes, fetch: fetchFn = globalThis.fetch } = {}) {
+  const module = await toModule(wasm, fetchFn)
+  const worker = new Worker(new URL('./pack-worker.js', import.meta.url), { type: 'module', name: 'wmdpack' })
+  const waiting = new Map()
+  let broken = null
+  const failAll = err => {
+    broken = err
+    for (const w of waiting.values()) w.reject(err)
+    waiting.clear()
+  }
+  worker.onmessage = ({ data: m }) => {
+    const w = waiting.get(m.id)
+    if (!w) return
+    waiting.delete(m.id)
+    if (m.t === 'error') w.reject(new Error(m.message))
+    else w.resolve(m)
+  }
+  worker.onerror = e => failAll(new Error(`the converter Worker failed: ${e.message || 'error'}`))
+  worker.postMessage({ t: 'init', module, maxFileBytes })
+  const call = (msg, transfer = []) => {
+    if (broken) return Promise.reject(broken)
+    const id = ++seq
+    return new Promise((resolve, reject) => {
+      waiting.set(id, { resolve, reject })
+      worker.postMessage({ ...msg, id }, transfer)
+    })
+  }
+  return {
+    /** Converts one layer (tar or gzip; its bytes are moved to the Worker) over the packs below it, top first. */
+    async convert(layer, below = []) {
+      const buf = layer instanceof ArrayBuffer ? layer : layer.byteOffset === 0 && layer.byteLength === layer.buffer.byteLength ? layer.buffer : layer.slice().buffer
+      const m = await call({ t: 'convert', layer: buf, below }, [buf])
+      return { sha: m.sha, size: m.size, bytes: m.bytes, ms: m.ms, fileBytes: m.fileBytes, wasmBytes: m.wasmBytes }
+    },
+    /** A pack the page already has (from its store), so a later layer can convert over it. */
+    async add(sha, bytes) {
+      await call({ t: 'add', sha, bytes })
+    },
+    close() {
+      failAll(new Error('the converter is closed'))
+      worker.terminate()
+    },
+  }
+}

package/src/registry/hub.js

@@ -0,0 +1,221 @@
+// W71: Docker Hub from the page, over a fetch the caller injects (in wmn's hosted editor,
+// createRelayFetch: TLS in the page, the bytes through a wisp relay). The same rules as the
+// registry proxy (web/registry/upstream.ts):
+//
+//   - only Docker Hub's registry and auth hosts, and its blob CDN hosts for a blob GET reached by
+//     a redirect, are contacted (plus `local` host:port names, for a test registry);
+//   - redirects are followed by hand, and the token is never sent to another host;
+//   - anonymous bearer tokens only (a private image is refused);
+//   - every manifest and blob is checked against its digest and size, with size caps.
+//
+// The fetch must answer `redirect: 'manual'` with the 3xx response itself (createRelayFetch and
+// Bun do; a browser's own fetch gives an opaque redirect, which is refused here).
+
+import { ACCEPT_MANIFEST, hostAllowed, MAX_MANIFEST_BYTES, parseChallenge } from './oci.js'
+import { DOCKER_HUB, isDigest } from './ref.js'
+
+export const MAX_BLOB_BYTES = 2 * 2 ** 30 // one layer in the page's memory
+
+export class RegistryError extends Error {
+  constructor(message, status = 502) {
+    super(message)
+    this.name = 'RegistryError'
+    this.status = status
+  }
+}
+
+const hex = buf => [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
+export const sha256 = async bytes => `sha256:${hex(await crypto.subtle.digest('SHA-256', bytes))}`
+
+const timeoutSignal = (signal, ms) => {
+  const sigs = [signal, ms ? AbortSignal.timeout(ms) : null].filter(Boolean)
+  return sigs.length > 1 ? AbortSignal.any(sigs) : sigs[0]
+}
+
+/**
+ * The body, read into one buffer of at most `max` bytes (`exact`: it must be exactly that many).
+ * A lying or missing Content-Length cannot make it bigger.
+ */
+async function readBody(res, max, what, { exact = false, onBytes, idleMs = 0 } = {}) {
+  const out = new Uint8Array(exact ? max : Math.min(max, Number(res.headers.get('content-length')) || 64 << 10))
+  let buf = out
+  let n = 0
+  const reader = res.body?.getReader()
+  if (!reader) {
+    const b = new Uint8Array(await res.arrayBuffer())
+    if (b.length > max) throw new RegistryError(`${what}: more than ${max} bytes, refused`)
+    return b
+  }
+  let timer = null
+  const arm = () => {
+    if (!idleMs) return
+    clearTimeout(timer)
+    timer = setTimeout(() => reader.cancel(new Error(`no bytes for ${idleMs} ms`)).catch(() => {}), idleMs)
+  }
+  try {
+    arm()
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      arm()
+      if (n + value.length > max) {
+        reader.cancel().catch(() => {})
+        throw new RegistryError(`${what}: more than ${max} bytes, refused`)
+      }
+      if (n + value.length > buf.length) {
+        const g = new Uint8Array(Math.min(max, Math.max(buf.length * 2, n + value.length)))
+        g.set(buf.subarray(0, n))
+        buf = g
+      }
+      buf.set(value, n)
+      n += value.length
+      onBytes?.(value.length)
+    }
+  } finally {
+    clearTimeout(timer)
+  }
+  return n === buf.length ? buf : buf.slice(0, n)
+}
+
+/** One repository on Docker Hub (or a `local` test registry), with its anonymous pull token. */
+export class HubRepo {
+  /**
+   * @param {{registry: string, repo: string}} ref a parsed reference (ref.js parseRef)
+   * @param {{fetch: typeof fetch, local?: string[]}} o
+   */
+  constructor(ref, { fetch: fetchFn, local = [] }) {
+    if (typeof fetchFn !== 'function') throw new TypeError('pullImage: pass a fetch (createRelayFetch in the hosted editor)')
+    this.fetch = fetchFn
+    this.local = local
+    this.repo = ref.repo
+    if (ref.registry === DOCKER_HUB) this.base = 'https://registry-1.docker.io'
+    else if (local.includes(ref.registry)) this.base = `http://${ref.registry}`
+    else throw new RegistryError(`registry ${ref.registry} is not allowed: the page pulls from Docker Hub (docker.io) only`, 403)
+    this.token = null
+    this.stats = { requests: 0, tokenFetches: 0, redirects: 0 }
+  }
+
+  async #fetch(url, accept, auth, o, blobCdn = false) {
+    if (!hostAllowed(url, this.local, blobCdn)) throw new RegistryError(`refusing to connect to ${url.host}: not a Docker Hub host`, 403)
+    const headers = {}
+    if (accept) headers.Accept = accept
+    if (auth && this.token) headers.Authorization = `Bearer ${this.token}`
+    this.stats.requests++
+    let res
+    try {
+      res = await this.fetch(url.href, { headers, redirect: 'manual', signal: o.signal })
+    } catch (e) {
+      if (o.signal?.aborted) throw o.signal.reason ?? e
+      throw new RegistryError(`${url.host}: ${e?.name === 'TimeoutError' ? 'timed out' : e?.message || e}`)
+    }
+    if (res.type === 'opaqueredirect') throw new RegistryError(`${url.host}: the fetch hid a redirect (it must answer redirect: 'manual' with the 3xx response)`)
+    return res
+  }
+
+  /** GET with the token, the challenge answered once, redirects followed by hand (no auth header to another host). */
+  async get(path, accept, o = {}) {
+    const url = new URL(`${this.base}/v2/${this.repo}/${path}`)
+    for (let attempt = 0; attempt < 2; attempt++) {
+      let res = await this.#fetch(url, accept, true, o)
+      let cur = url
+      let hops = 0
+      while (res.status >= 300 && res.status < 400 && res.headers.get('location')) {
+        if (++hops > 5) throw new RegistryError(`${url.pathname}: too many redirects`)
+        res.body?.cancel().catch(() => {})
+        const next = new URL(res.headers.get('location'), cur)
+        this.stats.redirects++
+        res = await this.#fetch(next, accept, next.host === url.host, o, path.startsWith('blobs/'))
+        cur = next
+      }
+      if (res.status === 401 && attempt === 0 && cur === url) {
+        const ch = parseChallenge(res.headers.get('www-authenticate') || '')
+        res.body?.cancel().catch(() => {})
+        if (ch.scheme !== 'bearer') throw new RegistryError(`${this.repo}: the registry wants ${ch.scheme || 'unknown'} auth; only anonymous bearer tokens are supported`, 401)
+        await this.#fetchToken(ch.params, o)
+        continue
+      }
+      return res
+    }
+    throw new RegistryError(`${this.repo}: unauthorized (anonymous pulls only; a private image is not served)`, 401)
+  }
+
+  async #fetchToken(p, o) {
+    if (!p.realm) throw new RegistryError('auth challenge without a realm')
+    let u
+    try {
+      u = new URL(p.realm)
+    } catch {
+      throw new RegistryError(`bad token realm ${JSON.stringify(p.realm)}`)
+    }
+    if (!hostAllowed(u, this.local)) throw new RegistryError(`refusing token realm ${u.host}: not a Docker Hub host`, 403)
+    if (p.service) u.searchParams.set('service', p.service)
+    u.searchParams.set('scope', p.scope || `repository:${this.repo}:pull`)
+    this.stats.tokenFetches++
+    const res = await this.#fetch(u, 'application/json', false, { signal: timeoutSignal(o.signal, 30_000) })
+    if (!res.ok) throw new RegistryError(`token from ${u.host}: HTTP ${res.status}`, res.status === 401 ? 401 : 502)
+    let j = {}
+    try {
+      j = JSON.parse(new TextDecoder().decode(await readBody(res, MAX_MANIFEST_BYTES, `token from ${u.host}`)))
+    } catch (e) {
+      if (e instanceof RegistryError) throw e
+    }
+    const t = j.token || j.access_token
+    if (typeof t !== 'string' || !t) throw new RegistryError(`token from ${u.host}: no token in the reply`)
+    this.token = t
+  }
+
+  /** A manifest or index: {body, doc, mediaType, digest}. With `want` (a digest), bytes that hash otherwise are refused. */
+  async manifest(reference, want = null, o = {}) {
+    const res = await this.get(`manifests/${reference}`, ACCEPT_MANIFEST, { signal: timeoutSignal(o.signal, 60_000) })
+    if (res.status === 404) {
+      res.body?.cancel().catch(() => {})
+      throw new RegistryError(`${this.repo}:${reference}: not found (or private)`, 404)
+    }
+    if (!res.ok) {
+      res.body?.cancel().catch(() => {})
+      throw new RegistryError(`${this.repo} manifest ${reference}: HTTP ${res.status}`, res.status === 429 ? 429 : 502)
+    }
+    const what = `${this.repo} manifest ${reference}`
+    const body = await readBody(res, MAX_MANIFEST_BYTES, what).catch(e => {
+      throw e instanceof RegistryError ? e : new RegistryError(`${what}: the transfer broke: ${e?.message || e}`)
+    })
+    const digest = await sha256(body)
+    const expect = want || (isDigest(reference) ? reference : null)
+    if (expect && digest !== expect) throw new RegistryError(`${this.repo} manifest: the registry sent bytes with digest ${digest}, not ${expect}; refused`)
+    let doc
+    try {
+      doc = JSON.parse(new TextDecoder().decode(body))
+    } catch {
+      throw new RegistryError(`${what}: not JSON`)
+    }
+    if (!doc || typeof doc !== 'object') throw new RegistryError(`${what}: not an object`)
+    const mediaType = doc.mediaType || (res.headers.get('content-type') || '').split(';')[0].trim()
+    return { body, doc, mediaType, digest }
+  }
+
+  /**
+   * A blob into memory, checked against its digest and size.
+   * @param {{digest: string, size: number}} desc
+   * @param {{signal?: AbortSignal, onBytes?: (n: number) => void, max?: number, idleMs?: number}} [o]
+   */
+  async blob(desc, o = {}) {
+    const max = o.max ?? MAX_BLOB_BYTES
+    const short = desc.digest?.slice?.(0, 19)
+    if (!isDigest(desc.digest)) throw new RegistryError(`bad digest ${desc.digest}`)
+    if (!(Number.isSafeInteger(desc.size) && desc.size >= 0 && desc.size <= max)) throw new RegistryError(`blob ${short}: ${desc.size} bytes refused (at most ${max})`)
+    const res = await this.get(`blobs/${desc.digest}`, null, { signal: o.signal })
+    if (!res.ok) {
+      res.body?.cancel().catch(() => {})
+      throw new RegistryError(`${this.repo} blob ${short}: HTTP ${res.status}`)
+    }
+    const idleMs = o.idleMs ?? 60_000
+    const body = await readBody(res, desc.size, `blob ${short}`, { exact: true, onBytes: o.onBytes, idleMs }).catch(e => {
+      if (o.signal?.aborted) throw o.signal.reason ?? e
+      throw e instanceof RegistryError ? e : new RegistryError(`blob ${short}: download failed: ${e?.message || e}`)
+    })
+    if (body.length !== desc.size) throw new RegistryError(`blob ${short}: the connection ended after ${body.length} of ${desc.size} bytes`)
+    const d = await sha256(body)
+    if (d !== desc.digest) throw new RegistryError(`blob ${desc.digest}: the bytes received hash to ${d}; refused`)
+    return body
+  }
+}

package/src/registry/oci.js

@@ -0,0 +1,76 @@
+// W71: the OCI and Docker distribution pieces the registry proxy (web/registry/upstream.ts,
+// pull.ts) and the page's own pull (pull.js) share. Plain JS that imports only ref.js.
+
+import { isDigest, platformMatches, platformString } from './ref.js'
+
+export const MT = {
+  ociIndex: 'application/vnd.oci.image.index.v1+json',
+  ociManifest: 'application/vnd.oci.image.manifest.v1+json',
+  dockerList: 'application/vnd.docker.distribution.manifest.list.v2+json',
+  dockerManifest: 'application/vnd.docker.distribution.manifest.v2+json',
+}
+export const ACCEPT_MANIFEST = [MT.ociIndex, MT.ociManifest, MT.dockerList, MT.dockerManifest].join(', ')
+export const isIndexType = mt => mt === MT.ociIndex || mt === MT.dockerList
+export const isManifestType = mt => mt === MT.ociManifest || mt === MT.dockerManifest
+
+const TAR = ['application/vnd.oci.image.layer.v1.tar', 'application/vnd.docker.image.rootfs.diff.tar']
+const GZIP = ['application/vnd.oci.image.layer.v1.tar+gzip', 'application/vnd.docker.image.rootfs.diff.tar.gzip']
+const ZSTD = ['application/vnd.oci.image.layer.v1.tar+zstd']
+
+/** Layer media types a converter takes. zstd: the proxy's converter reads it, the page's (wmdpack.wasm) does not. */
+export function isLayerType(mt, { zstd = true } = {}) {
+  return TAR.includes(mt) || GZIP.includes(mt) || (zstd && ZSTD.includes(mt))
+}
+export const isZstdLayer = mt => ZSTD.includes(mt)
+
+export const MAX_MANIFEST_BYTES = 4 << 20
+export const MAX_LAYERS = 256
+
+// Exact names only (W35): a suffix match on .docker.io / .docker.com would let a token realm
+// or a redirect send a pull to any Docker subdomain.
+export const HUB_HOSTS = ['registry-1.docker.io', 'auth.docker.io']
+// Blob CDN hosts. Observed 2026-10-09 with curl against the Hub API: blob GETs for alpine,
+// python, node, bash, ubuntu, debian and busybox answer 307 to production.cloudfront.docker.com;
+// production.cloudflare.docker.com is the older Hub blob CDN. These are allowed only for a blob
+// GET reached by a redirect from an allowed registry host, never as a realm or a first hop.
+// No R2 host was seen, so none is listed; add an exact name here if the Hub starts using one.
+export const BLOB_CDN_HOSTS = ['production.cloudfront.docker.com', 'production.cloudflare.docker.com']
+
+/** Whether a pull may connect to this URL. `local`: extra host:port names (plain http too); `blobCdn`: a blob GET redirected from an allowed host. */
+export function hostAllowed(u, local = [], blobCdn = false) {
+  if (local.includes(u.host)) return u.protocol === 'http:' || u.protocol === 'https:'
+  if (u.protocol !== 'https:') return false
+  if (u.port) return false
+  return HUB_HOSTS.includes(u.hostname) || (blobCdn && BLOB_CDN_HOSTS.includes(u.hostname))
+}
+
+/** `Bearer realm="...",service="...",scope="..."` -> {scheme, params}. */
+export function parseChallenge(h) {
+  const m = (h || '').match(/^\s*(\w+)\s+(.*)$/)
+  const params = {}
+  if (!m) return { scheme: '', params }
+  for (const x of m[2].matchAll(/([a-zA-Z_]+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^,\s]*))\s*,?/g)) {
+    params[x[1].toLowerCase()] = x[2] !== undefined ? x[2].replace(/\\(.)/g, '$1') : x[3]
+  }
+  return { scheme: m[1].toLowerCase(), params }
+}
+
+export const isAttestation = m =>
+  (m.platform?.os === 'unknown' && m.platform?.architecture === 'unknown') || m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest'
+
+/**
+ * The index entry for the first of `wants` (parsed platforms, in order of preference) the index
+ * has, as {desc, platform}; a nested index without a platform when none matches; else null
+ * and the platforms it has.
+ */
+export function pickPlatform(index, wants) {
+  const ms = Array.isArray(index?.manifests) ? index.manifests : []
+  const usable = ms.filter(m => m && isDigest(m.digest) && !isAttestation(m))
+  for (const want of wants) {
+    const hit = usable.find(m => m.platform && platformMatches(want, m.platform))
+    if (hit) return { desc: hit, platform: want }
+  }
+  const nested = usable.find(m => isIndexType(m.mediaType || '') && !m.platform)
+  if (nested) return { desc: nested, platform: null }
+  return { desc: null, platform: null, have: [...new Set(usable.filter(m => m.platform).map(m => platformString(m.platform)))] }
+}

package/src/registry/pack-worker.js

@@ -0,0 +1,70 @@
+// W71: the Worker that turns image layers into packs with wmdpack.wasm (wmd's own converter,
+// union.PackLayer, built for wasip1: `make wmdpack-wasm`). converter.js drives it.
+//
+// Messages in:  {t: 'init', module: WebAssembly.Module, maxFileBytes}
+//               {t: 'add', id, sha, bytes}              a pack the page already has (a cached lower layer)
+//               {t: 'convert', id, layer: ArrayBuffer, below: [sha...]}   below: the lower layers' packs, top first
+// Messages out: {t: 'done', id, sha, size, bytes, ms, fileBytes, wasmBytes}  bytes: the pack (shared when the
+//               page is cross-origin isolated, so the Worker and the page hold one copy); wasmBytes: the
+//               converter's wasm memory at its end
+//               {t: 'error', id, message}
+//
+// The Worker keeps every pack it made or was given, by sha256: a later layer's BELOW reads them in place.
+
+import { MemFs, runWasi } from './wasi-mem.js'
+
+let module = null
+let maxFileBytes = 4 * 2 ** 30
+const packs = new Map() // 'sha256:<hex>' -> Uint8Array
+
+const hex = buf => [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
+const shared = typeof SharedArrayBuffer !== 'undefined' && globalThis.crossOriginIsolated === true
+
+function convert({ id, layer, below }) {
+  const fs = new MemFs({ maxFileBytes })
+  const args = ['wmdpack', '/out.pack']
+  below.forEach((sha, i) => {
+    const b = packs.get(sha)
+    if (!b) throw new Error(`the pack ${sha.slice(0, 19)} below this layer is not in the converter`)
+    fs.addFile(`below/${i}.pack`, b)
+    args.push(`/below/${i}.pack`)
+  })
+  const t0 = performance.now()
+  const r = runWasi(module, { args, stdin: new Uint8Array(layer), fs })
+  const ms = Math.round(performance.now() - t0)
+  if (r.code !== 0) throw new Error(`converting the layer: ${r.stderr.trim() || `exit ${r.code}`}`)
+  const out = fs.take('out.pack')
+  if (!out) throw new Error('the converter wrote no pack')
+  return { out, ms, fileBytes: Number(r.stdout.trim()) || 0, wasmBytes: r.memBytes }
+}
+
+self.onmessage = async ({ data: m }) => {
+  try {
+    if (m.t === 'init') {
+      module = m.module
+      if (m.maxFileBytes) maxFileBytes = m.maxFileBytes
+      return
+    }
+    if (m.t === 'add') {
+      packs.set(m.sha, m.bytes)
+      self.postMessage({ t: 'added', id: m.id })
+      return
+    }
+    if (m.t === 'convert') {
+      const { out, ms, fileBytes, wasmBytes } = convert(m)
+      // WebCrypto refuses views of shared memory: hash before the shared copy
+      const sha = `sha256:${hex(await crypto.subtle.digest('SHA-256', out))}`
+      let bytes = out
+      if (shared) {
+        bytes = new Uint8Array(new SharedArrayBuffer(out.length))
+        bytes.set(out)
+      }
+      packs.set(sha, bytes)
+      self.postMessage({ t: 'done', id: m.id, sha, size: bytes.length, bytes: shared ? bytes : bytes.slice(), ms, fileBytes, wasmBytes })
+      return
+    }
+    throw new Error(`unknown message ${m.t}`)
+  } catch (e) {
+    self.postMessage({ t: 'error', id: m.id, message: e?.message || String(e) })
+  }
+}