@threadedstack/wmd-web 0.0.0-stage → 0.1.12

package/src/registry/pull.js

@@ -0,0 +1,258 @@
+// W71: pull an image in the page, with no pack host and no proxy.
+//
+//   const { index, packs } = await pullImage('python:3.12-slim', { fetch, converter: wmdpackWasmUrl })
+//   bootWmd({ ..., channels: { services: [{ ..., layers: packs }] } })
+//
+// It does what the registry proxy does (web/registry/pull.ts): resolve the reference (an index:
+// linux/arm64 first, else linux/amd64), check the config, download each layer (digest and size
+// checked) and convert it with wmd's own converter (wmdpack.wasm: union.PackLayer for wasip1) over
+// the packs below it, in a Worker. The result is the proxy's: an IMAGE_SCHEMA index (layers top
+// first, each pack named by the sha256 of its bytes) and the packs in that order — the same bytes
+// `wmd pull` and the proxy make for the same layers.
+//
+// Cache (optional): `store` with get(key) and put(key, bytes) (either may be async). Keys:
+//   'sha256:<hex>'   a pack's bytes (checked against its name when read back)
+//   'chain:<hex>'    a layer chain's record (JSON bytes {pack, size, layer}); the key is the
+//                    proxy's chain key: the converter and every layer digest from the bottom up
+// With a store, a layer whose chain is recorded and whose pack is there is not downloaded again.
+// When the page is cross-origin isolated the packs are views of SharedArrayBuffers (the converter
+// Worker and bootWmd use them in place, with no copy): a store that keeps them (IndexedDB, OPFS)
+// copies them first (bytes.slice()).
+
+import { createConverter } from './converter.js'
+import { HubRepo, RegistryError, sha256 } from './hub.js'
+import { isIndexType, isLayerType, isManifestType, isZstdLayer, MAX_LAYERS, pickPlatform } from './oci.js'
+import { IMAGE_SCHEMA, isDigest, parsePlatform, parseRef, platformMatches, platformString } from './ref.js'
+
+export { RegistryError }
+export const CONVERTER = 'wmdpack/1 union.PackLayer'
+export const DEFAULT_PLATFORMS = ['linux/arm64', 'linux/amd64']
+
+const enc = new TextEncoder()
+// WebCrypto does not hash views of shared memory
+const unshared = b => (typeof SharedArrayBuffer !== 'undefined' && b.buffer instanceof SharedArrayBuffer ? b.slice() : b)
+const hexOf = async s => (await sha256(enc.encode(s))).slice(7)
+
+/** The proxy's chain keys (web/registry/store.ts chainKeys): one per layer, bottom first. */
+export async function chainKeys(layerDigests) {
+  const keys = []
+  let prev = await hexOf(CONVERTER)
+  for (const d of layerDigests) {
+    prev = await hexOf(`${prev} ${d}`)
+    keys.push(prev)
+  }
+  return keys
+}
+
+async function fromStore(store, key) {
+  if (!store) return null
+  try {
+    const v = await store.get(key)
+    if (v == null) return null
+    return v instanceof Uint8Array ? v : new Uint8Array(v)
+  } catch {
+    return null
+  }
+}
+
+/**
+ * @param {string} refStr an image reference (docker.io only)
+ * @param {object} o
+ * @param {typeof fetch} o.fetch the fetch to reach Docker Hub with (it must return 3xx for redirect: 'manual')
+ * @param {string|URL|Uint8Array|WebAssembly.Module|{convert: Function, add: Function}} o.converter wmdpack.wasm, or a converter from createConverter
+ * @param {string|string[]} [o.platform] platforms in order of preference (default linux/arm64, then linux/amd64)
+ * @param {(e: object) => void} [o.onProgress] the proxy's pull events: resolved, layer, progress
+ * @param {AbortSignal} [o.signal]
+ * @param {{get: (key: string) => any, put: (key: string, bytes: Uint8Array) => any}} [o.store]
+ * @param {number} [o.concurrency] downloads at once (default 3)
+ * @param {string[]} [o.local] test registry host:port names allowed besides Docker Hub
+ * @returns {Promise<{index: object, packs: Uint8Array[]}>} packs top first, as index.layers
+ */
+export async function pullImage(refStr, o = {}) {
+  const t0 = performance.now()
+  const ref = parseRef(refStr)
+  const wants = (Array.isArray(o.platform) ? o.platform : o.platform ? [o.platform] : DEFAULT_PLATFORMS).map(p => parsePlatform(p))
+  const ev = o.onProgress || (() => {})
+  const sig = o.signal
+  const hub = new HubRepo(ref, { fetch: o.fetch, local: o.local })
+  if (!o.converter) throw new TypeError('pullImage: pass converter: wmdpack.wasm (a URL, its bytes or a WebAssembly.Module)')
+
+  // 1. resolve
+  let man = await hub.manifest(ref.digest || ref.tag, ref.digest, { signal: sig })
+  let indexDigest = null
+  let want = null
+  for (let depth = 0; ; depth++) {
+    const looksIndex = isIndexType(man.mediaType || '') || (!man.mediaType && Array.isArray(man.doc.manifests) && !man.doc.layers)
+    if (!looksIndex) break
+    if (depth >= 2) throw new RegistryError(`${ref.ref}: indexes nested more than one level are not supported`)
+    indexDigest ??= man.digest
+    const pick = pickPlatform(man.doc, want ? [want] : wants)
+    if (!pick.desc) {
+      const have = pick.have.length ? pick.have.join(', ') : 'no platforms'
+      throw new RegistryError(`${ref.ref}: no ${wants.map(platformString).join(' or ')} image in its index (it has: ${have})`, 404)
+    }
+    want = pick.platform ?? want
+    man = await hub.manifest(pick.desc.digest, pick.desc.digest, { signal: sig })
+  }
+  if (man.doc.schemaVersion === 1 || !(isManifestType(man.mediaType || '') || (!man.mediaType && Array.isArray(man.doc.layers)))) {
+    throw new RegistryError(`${ref.ref}: unsupported manifest type ${JSON.stringify(man.mediaType || `schema ${man.doc.schemaVersion}`)}`)
+  }
+  const layers = man.doc.layers
+  if (!Array.isArray(layers) || layers.length === 0) throw new RegistryError(`${ref.ref}: the manifest has no layers`)
+  if (layers.length > MAX_LAYERS) throw new RegistryError(`${ref.ref}: ${layers.length} layers (at most ${MAX_LAYERS})`)
+  for (const l of layers) {
+    if (!isDigest(l?.digest) || !Number.isSafeInteger(l.size) || l.size < 0) throw new RegistryError(`${ref.ref}: bad layer descriptor ${JSON.stringify(l).slice(0, 200)}`)
+    if (isZstdLayer(l.mediaType)) throw new RegistryError(`${ref.ref}: layer ${l.digest.slice(0, 19)} is zstd; the page's converter takes tar and gzip layers only`)
+    if (!isLayerType(l.mediaType || '', { zstd: false })) throw new RegistryError(`${ref.ref}: layer ${l.digest.slice(0, 19)} has media type ${l.mediaType} (foreign or not a filesystem tar)`)
+  }
+
+  // 2. config (the platform is checked against it too: an index entry can lie)
+  const cfgDesc = man.doc.config
+  if (!cfgDesc || !isDigest(cfgDesc.digest)) throw new RegistryError(`${ref.ref}: the manifest has no config`)
+  const cfgBytes = await hub.blob(cfgDesc, { signal: sig, max: 4 << 20 })
+  let cfg
+  try {
+    cfg = JSON.parse(new TextDecoder().decode(cfgBytes))
+  } catch {
+    throw new RegistryError(`${ref.ref}: the config is not JSON`)
+  }
+  const cfgPlat = cfg.os && cfg.architecture ? { os: cfg.os, architecture: cfg.architecture, variant: cfg.variant || '' } : null
+  if (!want) want = cfgPlat ? wants.find(w => platformMatches(w, cfgPlat)) : wants[0]
+  if (!want || (cfgPlat && !platformMatches(want, cfgPlat))) {
+    throw new RegistryError(`${ref.ref}: the image is ${cfgPlat ? platformString(cfgPlat) : 'of no platform'}, not ${wants.map(platformString).join(' or ')}`, 404)
+  }
+  const total = layers.reduce((a, l) => a + l.size, 0)
+  ev({ t: 'resolved', ref: ref.ref, platform: platformString(want), manifest: man.digest, index: indexDigest, layers: layers.length, bytes: total })
+
+  // 3. layers: cached ones from the store, the rest downloaded (a few at once, in order) and converted in order
+  const keys = await chainKeys(layers.map(l => l.digest))
+  const cached = []
+  for (let i = 0; i < layers.length; i++) {
+    let rec = null
+    const raw = await fromStore(o.store, `chain:${keys[i]}`)
+    if (raw) {
+      try {
+        const r = JSON.parse(new TextDecoder().decode(raw))
+        if (isDigest(r.pack) && r.layer === layers[i].digest) {
+          const bytes = await fromStore(o.store, r.pack)
+          if (bytes && bytes.length === r.size && (await sha256(unshared(bytes))) === r.pack) rec = { pack: r.pack, size: r.size, bytes }
+        }
+      } catch {}
+    }
+    cached.push(rec)
+  }
+
+  const abort = new AbortController()
+  const inner = sig ? AbortSignal.any([sig, abort.signal]) : abort.signal
+  const todo = layers.map((_, i) => i).filter(i => !cached[i])
+  const ownConverter = !(typeof o.converter?.convert === 'function')
+  let conv = null
+  let got = 0
+  let downloadBytes = 0
+  const tDl0 = performance.now()
+  let tDlEnd = tDl0
+  const downloads = layers.map(() => null)
+  const limit = Math.max(1, o.concurrency ?? 3)
+  let next = 0
+  let running = 0
+  const startNext = () => {
+    while (running < limit && next < todo.length) {
+      const i = todo[next++]
+      running++
+      const l = layers[i]
+      const t = performance.now()
+      ev({ t: 'layer', i, state: 'downloading', digest: l.digest, bytes: l.size })
+      let mine = 0
+      const p = hub
+        .blob(l, {
+          signal: inner,
+          onBytes: n => {
+            got += n
+            mine += n
+            ev({ t: 'progress', bytes: got, total, layer: i, layerBytes: mine, layerTotal: l.size })
+          },
+        })
+        .then(b => {
+          downloadBytes += b.length
+          tDlEnd = performance.now()
+          ev({ t: 'layer', i, state: 'downloaded', digest: l.digest, bytes: b.length, ms: Math.round(performance.now() - t) })
+          return b
+        })
+        .finally(() => {
+          running--
+          startNext()
+        })
+      p.catch(() => {})
+      downloads[i] = p
+    }
+  }
+
+  const out = [] // bottom first: {pack, size, layer, bytes, cached}
+  let convertMs = 0
+  let fileBytes = 0
+  let wasmBytes = 0
+  try {
+    startNext()
+    if (todo.length) conv = ownConverter ? await createConverter(o.converter) : o.converter
+    for (let i = 0; i < layers.length; i++) {
+      inner.throwIfAborted()
+      const l = layers[i]
+      if (cached[i]) {
+        const c = cached[i]
+        out.push({ pack: c.pack, size: c.size, layer: l.digest, bytes: c.bytes, cached: true })
+        // a later layer converts over it
+        if (todo.some(j => j > i)) await conv.add(c.pack, c.bytes)
+        ev({ t: 'layer', i, state: 'cached', digest: l.digest, pack: c.pack, packBytes: c.size })
+        continue
+      }
+      const blob = await downloads[i]
+      downloads[i] = null
+      const below = out.slice(0, i).map(x => x.pack).reverse()
+      const r = await conv.convert(blob, below)
+      convertMs += r.ms
+      fileBytes += r.fileBytes
+      wasmBytes = Math.max(wasmBytes, r.wasmBytes || 0)
+      out.push({ pack: r.sha, size: r.size, layer: l.digest, bytes: r.bytes, cached: false })
+      if (o.store) {
+        await o.store.put(r.sha, r.bytes)
+        await o.store.put(`chain:${keys[i]}`, enc.encode(JSON.stringify({ pack: r.sha, size: r.size, layer: l.digest })))
+      }
+      ev({ t: 'layer', i, state: 'converted', digest: l.digest, pack: r.sha, packBytes: r.size, ms: r.ms })
+    }
+  } catch (e) {
+    abort.abort(e)
+    await Promise.allSettled(downloads.filter(Boolean))
+    throw e
+  } finally {
+    if (conv && ownConverter) conv.close()
+  }
+
+  // 4. the index (top first, as the Overlay takes the packs): the proxy's document
+  const c = cfg.config || {}
+  const top = out.slice().reverse()
+  const index = {
+    schema: IMAGE_SCHEMA,
+    ref: ref.ref,
+    platform: platformString(want),
+    index: indexDigest,
+    manifest: man.digest,
+    os: cfg.os || null,
+    architecture: cfg.architecture || null,
+    config: { Env: c.Env || null, Cmd: c.Cmd || null, Entrypoint: c.Entrypoint || null, WorkingDir: c.WorkingDir || '', User: c.User || '', ExposedPorts: c.ExposedPorts || null },
+    layers: top.map(x => ({ pack: x.pack, size: x.size, layer: x.layer })),
+    converter: CONVERTER,
+    pulled: new Date().toISOString(),
+    stats: {
+      totalMs: Math.round(performance.now() - t0),
+      layerBytes: total,
+      downloadBytes,
+      downloadMs: todo.length ? Math.round(tDlEnd - tDl0) : 0,
+      convertMs,
+      fileBytes,
+      wasmBytes, // the converter's largest wasm memory, one layer's
+      packBytes: out.reduce((a, x) => a + x.size, 0),
+      cachedLayers: out.filter(x => x.cached).length,
+    },
+  }
+  return { index, packs: top.map(x => x.bytes) }
+}

package/src/registry/ref.js

@@ -0,0 +1,110 @@
+// P-B7: image references, platforms and the static pack host's layout. Plain JS with no
+// imports, so the page, the service worker's page and the Bun proxy share it.
+//
+// The static host layout (content addressed, immutable URLs, one small index per
+// image@platform):
+//
+//   /blobs/sha256/<hex>                    a pack (wmd's pack format), named by the sha256 of
+//                                          its own bytes; never changes (Cache-Control immutable)
+//   /idx/<registry>/<repo>/<tag>/<os>-<arch>[-<variant>].json
+//                                          the image's index for one platform (IMAGE_SCHEMA):
+//                                          what a tag points at now; revalidated on every visit
+//
+// Any static file server can serve a folder in this layout (with the page's COOP/COEP and
+// same-origin CORP); the registry proxy (web/registry/) writes the same layout as its cache.
+
+export const IMAGE_SCHEMA = 'wmd-web-image/1'
+export const DOCKER_HUB = 'docker.io'
+export const DEFAULT_PLATFORM = 'linux/arm64'
+
+const DIGEST_RE = /^sha256:[0-9a-f]{64}$/
+const TAG_RE = /^[\w][\w.-]{0,127}$/
+const COMPONENT_RE = /^[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*$/
+const HOST_RE = /^[a-zA-Z0-9](?:[a-zA-Z0-9.-]{0,253}[a-zA-Z0-9])?(?::\d{1,5})?$/
+
+export const isDigest = d => typeof d === 'string' && DIGEST_RE.test(d)
+
+/**
+ * Parses an image reference the way docker does: `busybox` is docker.io/library/busybox:latest,
+ * a first component with a dot, a colon or "localhost" is a registry host.
+ * @returns {{registry:string, repo:string, tag:string|null, digest:string|null, name:string, ref:string}}
+ */
+export function parseRef(s) {
+  if (typeof s !== 'string' || !s || s.length > 512) throw new Error(`bad image reference ${JSON.stringify(s)}`)
+  let rest = s
+  let digest = null
+  const at = rest.indexOf('@')
+  if (at >= 0) {
+    digest = rest.slice(at + 1)
+    rest = rest.slice(0, at)
+    if (!isDigest(digest)) throw new Error(`bad digest in ${JSON.stringify(s)} (only sha256:<64 hex> is accepted)`)
+  }
+  let tag = null
+  const slash = rest.lastIndexOf('/')
+  const colon = rest.lastIndexOf(':')
+  if (colon > slash) {
+    tag = rest.slice(colon + 1)
+    rest = rest.slice(0, colon)
+    if (!TAG_RE.test(tag)) throw new Error(`bad tag in ${JSON.stringify(s)}`)
+  }
+  const parts = rest.split('/')
+  let registry = DOCKER_HUB
+  if (parts.length > 1 && (parts[0].includes('.') || parts[0].includes(':') || parts[0] === 'localhost')) {
+    registry = parts.shift()
+    if (!HOST_RE.test(registry)) throw new Error(`bad registry host in ${JSON.stringify(s)}`)
+  }
+  if (registry === 'index.docker.io' || registry === 'registry-1.docker.io') registry = DOCKER_HUB
+  if (registry === DOCKER_HUB && parts.length === 1) parts.unshift('library')
+  if (!parts.length || !parts.every(p => COMPONENT_RE.test(p))) throw new Error(`bad repository name in ${JSON.stringify(s)}`)
+  const repo = parts.join('/')
+  if (!tag && !digest) tag = 'latest'
+  const name = `${registry}/${repo}`
+  return { registry, repo, tag, digest, name, ref: `${name}${tag ? `:${tag}` : ''}${digest ? `@${digest}` : ''}` }
+}
+
+/** "linux/arm64/v8" -> {os, architecture, variant}. */
+export function parsePlatform(p = DEFAULT_PLATFORM) {
+  const [os, architecture, variant = '', extra] = String(p).split('/')
+  if (!os || !architecture || extra !== undefined || ![os, architecture, variant].every(x => x === '' || /^[a-z0-9_]+$/.test(x))) {
+    throw new Error(`bad platform ${JSON.stringify(p)} (want os/arch[/variant])`)
+  }
+  return { os, architecture, variant: normVariant(architecture, variant) }
+}
+
+/** arm64's only variant is v8: "linux/arm64" and "linux/arm64/v8" are the same platform. */
+function normVariant(arch, v) {
+  if (arch === 'arm64' && (v === '' || v === 'v8')) return ''
+  return v || ''
+}
+
+export const platformString = p => `${p.os}/${p.architecture}${normVariant(p.architecture, p.variant || '') ? `/${p.variant}` : ''}`
+
+/** Whether an index entry's platform runs a request for `want` (exact os and arch; variant equal once normalized). */
+export function platformMatches(want, have) {
+  if (!have) return false
+  return have.os === want.os && have.architecture === want.architecture && normVariant(have.architecture, have.variant || '') === want.variant
+}
+
+/** The index path for an image@platform (the tag, or "@sha256-<hex>" for a digest reference). */
+export function indexPath(ref, platform = DEFAULT_PLATFORM) {
+  const r = typeof ref === 'string' ? parseRef(ref) : ref
+  const p = typeof platform === 'string' ? parsePlatform(platform) : platform
+  const at = r.digest ? `@${r.digest.replace(':', '-')}` : r.tag
+  return `idx/${r.registry.replace(':', '_')}/${r.repo}/${at}/${p.os}-${p.architecture}${p.variant ? `-${p.variant}` : ''}.json`
+}
+
+export const blobPath = digest => {
+  if (!isDigest(digest)) throw new Error(`bad digest ${JSON.stringify(digest)}`)
+  return `blobs/sha256/${digest.slice(7)}`
+}
+
+/** Checks an image index document (from the host or the proxy) before the page trusts it. */
+export function checkImageIndex(x) {
+  if (!x || x.schema !== IMAGE_SCHEMA) throw new Error(`not a ${IMAGE_SCHEMA} index`)
+  if (!Array.isArray(x.layers) || x.layers.length === 0 || x.layers.length > 256) throw new Error('index: bad layer list')
+  for (const l of x.layers) {
+    if (!isDigest(l.pack) || !Number.isSafeInteger(l.size) || l.size < 16) throw new Error(`index: bad layer entry ${JSON.stringify(l)}`)
+  }
+  if (x.config !== null && typeof x.config !== 'object') throw new Error('index: bad config')
+  return x
+}