@threadedstack/wmd-web 0.1.17 → 0.1.18

package/src/host/wmd-imports.js

@@ -3,7 +3,7 @@
 // P-B1 runs the interpreter with no channels, so only ram_map is ever called. The
 // rest are stubs with the values the guest treats as "not available":
 //   ram_map → 1 (RAM left as plain wasm memory; return value ignored, main_wasm.c:562)
-//   blk_rw → -38 (no swap disk: the browser never passes --swap, so vm.wasm makes no device, D-497)
+//   blk_rw → -38 without a disk (bootWmd's o.disk passes --swap and the disk's mailbox: host/disk.js)
 //   win_init → 1 (address windows off, win.c:226-253); other win_* → 0 (never called)
 //   snapshot → -1 ("continuing without it")
 //   ctl_attach → -1 (no runner control socket: the guest starts no ctl thread, W3.3)
@@ -26,7 +26,7 @@ const EBADF = -9
 const ECONNREFUSED = -111
 const ENOSYS = -38
 
-export function createWmdImports({ jit = {}, chan = {}, counters, ckpt = null, ram = null } = {}) {
+export function createWmdImports({ jit = {}, chan = {}, counters, ckpt = null, ram = null, blk = null } = {}) {
   const note = counters ? name => noteHostCall(counters, name) : () => {}
   return {
     // channels (P-B3)
@@ -54,7 +54,7 @@ export function createWmdImports({ jit = {}, chan = {}, counters, ckpt = null, r
     ram_map2: ram?.ram_map2 || ((_off, _lowLen, _total) => -1),
     ram_rw: ram?.ram_rw || ((_off, _ptr, _len, _write) => -1),
     ram_cas: ram?.ram_cas || ((_off, _size, _expect, _desired) => 0n),
-    blk_rw: (_op, _off, _ptr, _len) => ENOSYS,
+    blk_rw: blk?.blk_rw || ((_op, _off, _ptr, _len) => ENOSYS), // a disk in OPFS (host/disk.js), when bootWmd has one
 
     // W3.3 checkpoints: no control socket and no checkpoint in the browser
     // P-B9: with `ckpt` (host/ckpt-point.js) the guest starts its ctl thread and takes checkpoints

package/src/image-files.js

@@ -53,7 +53,8 @@ export class ImageFiles {
   /** A file's text (UTF-8), or null when it does not exist. */
   readText(path) {
     const b = this.readFile(path)
-    return b && new TextDecoder().decode(b)
+    // a view on shared memory (wasm) is copied first: browsers' TextDecoder refuses it
+    return b && new TextDecoder().decode(typeof SharedArrayBuffer !== 'undefined' && b.buffer instanceof SharedArrayBuffer ? b.slice() : b)
   }
 
   /** {type ('file', 'dir', 'symlink', ...), mode (permission bits), uid, gid, size, mtime (BigInt ns), target?}, or null. */

package/src/registry/hub.js

@@ -24,6 +24,20 @@ export class RegistryError extends Error {
   }
 }
 
+/** A mirror URL 'http(s)://host[:port]' (nothing after it) -> URL; anything else is refused. */
+export function parseMirror(m) {
+  let u
+  try {
+    u = new URL(m)
+  } catch {
+    throw new TypeError(`bad mirror ${JSON.stringify(m)}: http(s)://host[:port]`)
+  }
+  if ((u.protocol !== 'http:' && u.protocol !== 'https:') || u.username || u.password || u.search || u.hash || (u.pathname !== '/' && u.pathname !== '')) {
+    throw new TypeError(`bad mirror ${JSON.stringify(m)}: http(s)://host[:port]`)
+  }
+  return u
+}
+
 const hex = buf => [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
 export const sha256 = async bytes => `sha256:${hex(await crypto.subtle.digest('SHA-256', bytes))}`
 
@@ -81,14 +95,26 @@ async function readBody(res, max, what, { exact = false, onBytes, idleMs = 0 } =
 export class HubRepo {
   /**
    * @param {{registry: string, repo: string}} ref a parsed reference (ref.js parseRef)
-   * @param {{fetch: typeof fetch, local?: string[]}} o
+   * @param {{fetch: typeof fetch, local?: string[], mirror?: string}} o  mirror: 'http(s)://host[:port]', a Docker Hub mirror
+   *   tried first for Docker Hub references (BuildKit's mirror semantics: the upstream answers whatever the mirror cannot);
+   *   its host is allowed to be contacted because the caller named it. mirrorWaitMs: how long the mirror may take to answer
+   *   before the upstream is asked instead.
    */
-  constructor(ref, { fetch: fetchFn, local = [] }) {
+  constructor(ref, { fetch: fetchFn, local = [], mirror = null, mirrorWaitMs = 15_000 }) {
     if (typeof fetchFn !== 'function') throw new TypeError('pullImage: pass a fetch (createRelayFetch in the hosted editor)')
     this.fetch = fetchFn
     this.local = local
+    this.mirrorBase = null
+    this.mirrorWaitMs = mirrorWaitMs
     this.repo = ref.repo
-    if (ref.registry === DOCKER_HUB) this.base = 'https://registry-1.docker.io'
+    if (ref.registry === DOCKER_HUB) {
+      this.base = 'https://registry-1.docker.io'
+      if (mirror) {
+        const u = parseMirror(mirror)
+        this.mirrorBase = u.origin
+        this.local = [...local, u.host]
+      }
+    }
     else if (local.includes(ref.registry)) this.base = `http://${ref.registry}`
     else throw new RegistryError(`registry ${ref.registry} is not allowed: the page pulls from Docker Hub (docker.io) only`, 403)
     this.token = null
@@ -112,11 +138,52 @@ export class HubRepo {
     return res
   }
 
-  /** GET with the token, the challenge answered once, redirects followed by hand (no auth header to another host). */
-  async get(path, accept, o = {}) {
-    const url = new URL(`${this.base}/v2/${this.repo}/${path}`)
+  /**
+   * GET from the mirror first, then from the upstream, and `read` the answer. Any mirror failure goes on to the upstream,
+   * as BuildKit does: no connection, no answer within mirrorWaitMs, 404, 401, 5xx, or bytes `read` refuses (a wrong
+   * digest, a short body). The mirror never gets a token. A caller abort ends the request with no fallback.
+   * @param {(res: Response) => Promise<any>} read  takes the upstream answer whatever its status; a mirror's only when ok
+   * @param {AbortSignal} [signal] the caller's
+   * @param {number} [ms] a time limit on each source's request and read
+   */
+  async #fromEither(path, accept, signal, ms, read) {
+    if (signal?.aborted) throw signal.reason
+    if (this.mirrorBase && !this.mirrorDown) {
+      const ac = new AbortController()
+      const onAbort = () => ac.abort(signal.reason)
+      signal?.addEventListener('abort', onAbort, { once: true })
+      const wait = setTimeout(() => ac.abort(new RegistryError(`mirror ${this.mirrorBase}: no answer in ${this.mirrorWaitMs} ms`)), this.mirrorWaitMs)
+      try {
+        const res = await this.#getFrom(this.mirrorBase, path, accept, { signal: timeoutSignal(ac.signal, ms) }, false)
+        clearTimeout(wait)
+        if (res.ok) {
+          const out = await read(res)
+          this.stats.mirrorHits = (this.stats.mirrorHits || 0) + 1
+          return out
+        }
+        res.body?.cancel().catch(() => {})
+      } catch (e) {
+        if (signal?.aborted) throw signal.reason ?? e
+        // a mirror that cannot be reached, hangs or sends bad bytes is not asked again for this repository
+        this.mirrorDown = true
+      } finally {
+        clearTimeout(wait)
+        signal?.removeEventListener('abort', onAbort)
+      }
+      this.stats.mirrorMisses = (this.stats.mirrorMisses || 0) + 1
+    }
+    const o = { signal: timeoutSignal(signal, ms) }
+    return read(await this.#getFrom(this.base, path, accept, o, true))
+  }
+
+  /**
+   * GET with the token (`tokens`; never for a mirror), the challenge answered once, redirects followed by hand (no auth
+   * header to another host).
+   */
+  async #getFrom(base, path, accept, o, tokens) {
+    const url = new URL(`${base}/v2/${this.repo}/${path}`)
     for (let attempt = 0; attempt < 2; attempt++) {
-      let res = await this.#fetch(url, accept, true, o)
+      let res = await this.#fetch(url, accept, tokens, o)
       let cur = url
       let hops = 0
       while (res.status >= 300 && res.status < 400 && res.headers.get('location')) {
@@ -124,10 +191,10 @@ export class HubRepo {
         res.body?.cancel().catch(() => {})
         const next = new URL(res.headers.get('location'), cur)
         this.stats.redirects++
-        res = await this.#fetch(next, accept, next.host === url.host, o, path.startsWith('blobs/'))
+        res = await this.#fetch(next, accept, tokens && next.host === url.host, o, path.startsWith('blobs/'))
         cur = next
       }
-      if (res.status === 401 && attempt === 0 && cur === url) {
+      if (tokens && res.status === 401 && attempt === 0 && cur === url) {
         const ch = parseChallenge(res.headers.get('www-authenticate') || '')
         res.body?.cancel().catch(() => {})
         if (ch.scheme !== 'bearer') throw new RegistryError(`${this.repo}: the registry wants ${ch.scheme || 'unknown'} auth; only anonymous bearer tokens are supported`, 401)
@@ -166,31 +233,32 @@ export class HubRepo {
 
   /** A manifest or index: {body, doc, mediaType, digest}. With `want` (a digest), bytes that hash otherwise are refused. */
   async manifest(reference, want = null, o = {}) {
-    const res = await this.get(`manifests/${reference}`, ACCEPT_MANIFEST, { signal: timeoutSignal(o.signal, 60_000) })
-    if (res.status === 404) {
-      res.body?.cancel().catch(() => {})
-      throw new RegistryError(`${this.repo}:${reference}: not found (or private)`, 404)
-    }
-    if (!res.ok) {
-      res.body?.cancel().catch(() => {})
-      throw new RegistryError(`${this.repo} manifest ${reference}: HTTP ${res.status}`, res.status === 429 ? 429 : 502)
-    }
     const what = `${this.repo} manifest ${reference}`
-    const body = await readBody(res, MAX_MANIFEST_BYTES, what).catch(e => {
-      throw e instanceof RegistryError ? e : new RegistryError(`${what}: the transfer broke: ${e?.message || e}`)
+    return this.#fromEither(`manifests/${reference}`, ACCEPT_MANIFEST, o.signal, 60_000, async res => {
+      if (res.status === 404) {
+        res.body?.cancel().catch(() => {})
+        throw new RegistryError(`${this.repo}:${reference}: not found (or private)`, 404)
+      }
+      if (!res.ok) {
+        res.body?.cancel().catch(() => {})
+        throw new RegistryError(`${this.repo} manifest ${reference}: HTTP ${res.status}`, res.status === 429 ? 429 : 502)
+      }
+      const body = await readBody(res, MAX_MANIFEST_BYTES, what).catch(e => {
+        throw e instanceof RegistryError ? e : new RegistryError(`${what}: the transfer broke: ${e?.message || e}`)
+      })
+      const digest = await sha256(body)
+      const expect = want || (isDigest(reference) ? reference : null)
+      if (expect && digest !== expect) throw new RegistryError(`${this.repo} manifest: the registry sent bytes with digest ${digest}, not ${expect}; refused`)
+      let doc
+      try {
+        doc = JSON.parse(new TextDecoder().decode(body))
+      } catch {
+        throw new RegistryError(`${what}: not JSON`)
+      }
+      if (!doc || typeof doc !== 'object') throw new RegistryError(`${what}: not an object`)
+      const mediaType = doc.mediaType || (res.headers.get('content-type') || '').split(';')[0].trim()
+      return { body, doc, mediaType, digest }
     })
-    const digest = await sha256(body)
-    const expect = want || (isDigest(reference) ? reference : null)
-    if (expect && digest !== expect) throw new RegistryError(`${this.repo} manifest: the registry sent bytes with digest ${digest}, not ${expect}; refused`)
-    let doc
-    try {
-      doc = JSON.parse(new TextDecoder().decode(body))
-    } catch {
-      throw new RegistryError(`${what}: not JSON`)
-    }
-    if (!doc || typeof doc !== 'object') throw new RegistryError(`${what}: not an object`)
-    const mediaType = doc.mediaType || (res.headers.get('content-type') || '').split(';')[0].trim()
-    return { body, doc, mediaType, digest }
   }
 
   /**
@@ -203,19 +271,20 @@ export class HubRepo {
     const short = desc.digest?.slice?.(0, 19)
     if (!isDigest(desc.digest)) throw new RegistryError(`bad digest ${desc.digest}`)
     if (!(Number.isSafeInteger(desc.size) && desc.size >= 0 && desc.size <= max)) throw new RegistryError(`blob ${short}: ${desc.size} bytes refused (at most ${max})`)
-    const res = await this.get(`blobs/${desc.digest}`, null, { signal: o.signal })
-    if (!res.ok) {
-      res.body?.cancel().catch(() => {})
-      throw new RegistryError(`${this.repo} blob ${short}: HTTP ${res.status}`)
-    }
     const idleMs = o.idleMs ?? 60_000
-    const body = await readBody(res, desc.size, `blob ${short}`, { exact: true, onBytes: o.onBytes, idleMs }).catch(e => {
-      if (o.signal?.aborted) throw o.signal.reason ?? e
-      throw e instanceof RegistryError ? e : new RegistryError(`blob ${short}: download failed: ${e?.message || e}`)
+    return this.#fromEither(`blobs/${desc.digest}`, null, o.signal, 0, async res => {
+      if (!res.ok) {
+        res.body?.cancel().catch(() => {})
+        throw new RegistryError(`${this.repo} blob ${short}: HTTP ${res.status}`)
+      }
+      const body = await readBody(res, desc.size, `blob ${short}`, { exact: true, onBytes: o.onBytes, idleMs }).catch(e => {
+        if (o.signal?.aborted) throw o.signal.reason ?? e
+        throw e instanceof RegistryError ? e : new RegistryError(`blob ${short}: download failed: ${e?.message || e}`)
+      })
+      if (body.length !== desc.size) throw new RegistryError(`blob ${short}: the connection ended after ${body.length} of ${desc.size} bytes`)
+      const d = await sha256(body)
+      if (d !== desc.digest) throw new RegistryError(`blob ${desc.digest}: the bytes received hash to ${d}; refused`)
+      return body
     })
-    if (body.length !== desc.size) throw new RegistryError(`blob ${short}: the connection ended after ${body.length} of ${desc.size} bytes`)
-    const d = await sha256(body)
-    if (d !== desc.digest) throw new RegistryError(`blob ${desc.digest}: the bytes received hash to ${d}; refused`)
-    return body
   }
 }