@threadedstack/wmd-web 0.1.17 → 0.1.18

package/src/build/inspect.js

@@ -0,0 +1,78 @@
+// Reading built layers back, for the fast-path parity gate (docs/plans/web-buildkit-impl.md 5.5): the entries of a
+// layer tar (path, type, mode, owner, size, sha256, link target, xattrs, mtime) and the layers of a Docker tar
+// (`docker save` layout or an OCI layout, layers plain or gzip).
+import { scanTarBytes } from './tar.js'
+
+const dec = new TextDecoder()
+const TYPES = { 0: 'file', 7: 'file', 1: 'hardlink', 2: 'symlink', 3: 'char', 4: 'block', 5: 'dir', 6: 'fifo' }
+
+const hex = (buf) => [...new Uint8Array(buf)].map((x) => x.toString(16).padStart(2, '0')).join('')
+const sha256Hex = async (u8) => hex(await crypto.subtle.digest('SHA-256', u8))
+
+/** gunzip when the bytes start with the gzip magic; else the bytes themselves. */
+export async function maybeGunzip(u8) {
+  if (!(u8.length > 2 && u8[0] === 0x1f && u8[1] === 0x8b)) return u8
+  const s = new Blob([u8]).stream().pipeThrough(new DecompressionStream('gzip'))
+  return new Uint8Array(await new Response(s).arrayBuffer())
+}
+
+/** './a/', '/a' and 'a' are the same path: no leading './' or '/', no trailing '/'. */
+export function cleanEntryPath(p) {
+  return p.replace(/^(\.\/|\/)+/, '').replace(/\/+$/, '') || '.'
+}
+
+/**
+ * Every entry of a layer tar (plain or gzip) as {path, type, mode, uid, gid, size, sha256, linkname, xattrs, mtime}
+ * in archive order. `sha256` is the file's content for regular files, else null; `mode` the permission bits (with
+ * setuid/setgid/sticky).
+ */
+export async function layerEntries(bytes) {
+  const u8 = await maybeGunzip(bytes)
+  const out = []
+  for await (const e of scanTarBytes(u8)) {
+    const type = TYPES[e.type] || `type-${e.type}`
+    out.push({
+      path: cleanEntryPath(e.name),
+      type,
+      mode: e.mode & 0o7777,
+      uid: e.uid,
+      gid: e.gid,
+      size: type === 'file' ? e.size : 0,
+      sha256: type === 'file' ? await sha256Hex(u8.subarray(e.offset, e.offset + e.size)) : null,
+      linkname: e.linkname,
+      xattrs: e.xattrs || {},
+      mtime: e.mtime,
+    })
+  }
+  return out
+}
+
+/**
+ * A Docker tar (what BuildKit's type=docker output writes: a `docker save` layout with manifest.json, or an OCI
+ * layout with index.json) as {config, layers}: `config` the image config JSON, `layers` the layer tars' bytes
+ * bottom first, as stored (plain or gzip).
+ */
+export async function dockerTarLayers(bytes) {
+  const files = new Map()
+  for await (const e of scanTarBytes(bytes)) {
+    if (e.type === '0' || e.type === '7') files.set(cleanEntryPath(e.name), bytes.subarray(e.offset, e.offset + e.size))
+  }
+  const get = (p) => {
+    const f = files.get(cleanEntryPath(p))
+    if (!f) throw new Error(`the Docker tar has no ${p}`)
+    return f
+  }
+  const json = (p) => JSON.parse(dec.decode(get(p)))
+  const blob = (digest) => get(`blobs/${digest.replace(':', '/')}`)
+  if (files.has('manifest.json')) {
+    const m = json('manifest.json')[0]
+    return { config: json(m.Config), layers: m.Layers.map((l) => get(l)) }
+  }
+  if (files.has('index.json')) {
+    let m = json('index.json').manifests[0]
+    let man = JSON.parse(dec.decode(blob(m.digest)))
+    if (man.manifests) man = JSON.parse(dec.decode(blob(man.manifests[0].digest)))
+    return { config: JSON.parse(dec.decode(blob(man.config.digest))), layers: man.layers.map((l) => blob(l.digest)) }
+  }
+  throw new Error('the tar has neither manifest.json nor index.json')
+}

package/src/build/pin.js

@@ -0,0 +1,6 @@
+// The wmd-web builder images (scripts/web-builder.sh writes this file): the sha256 and size of each
+// `docker save` tar in web/dist/builder/. createBuilder refuses a tar that does not match.
+export const BUILDER_PINS = {
+  arm64: { file: 'builder-arm64.tar', sha256: '905d0735a2fb7aa12f00bb21f50e82651d1f65db0aa030aa38ab4033047072ee', bytes: 107119104 },
+  amd64: { file: 'builder-amd64.tar', sha256: 'be2199d5eb05d1bcc2814999749de17d4e4ce206726f31fe8a495cf602fd3ccd', bytes: 113018368 },
+}

package/src/build/tar.js

@@ -0,0 +1,304 @@
+// A small deterministic tar writer and header scanner (docs/plans/web-buildkit-impl.md 5.6).
+// Output is ustar; a pax extended header (typeflag 'x') carries what ustar cannot hold: names over the
+// name+prefix limits, non-ASCII names, link targets over 100 bytes, sizes of 8 GiB or more, and uid/gid/mtime
+// that overflow their octal fields. Nothing reads the host clock: mtime is what the caller gives, else 0.
+
+const enc = new TextEncoder()
+const dec = new TextDecoder()
+const BLOCK = 512
+
+/** TextDecoder.decode that also takes a view on a SharedArrayBuffer (wasm memory), which browsers' decoders refuse. */
+export function decodeText(u8, d = dec) {
+  return typeof SharedArrayBuffer !== 'undefined' && u8.buffer instanceof SharedArrayBuffer ? d.decode(u8.slice()) : d.decode(u8)
+}
+
+/** Throws on a path that must not enter an archive: absolute, '..', NUL, empty components. */
+export function checkTarPath(path) {
+  if (path === './') return // the archive's root folder entry
+  if (typeof path !== 'string' || path === '') throw new Error('tar: empty path')
+  if (path.includes('\0')) throw new Error(`tar: NUL in path ${JSON.stringify(path)}`)
+  if (path.startsWith('/')) throw new Error(`tar: absolute path ${path}`)
+  const parts = path.split('/')
+  if (parts[parts.length - 1] === '') parts.pop() // one trailing '/' is a directory spelling
+  for (const p of parts) {
+    if (p === '' || p === '.' || p === '..') throw new Error(`tar: unsafe path ${path}`)
+  }
+}
+
+function isAscii(s) {
+  for (let i = 0; i < s.length; i++) if (s.charCodeAt(i) > 0x7f) return false
+  return true
+}
+
+function toSeconds(t) {
+  if (t === undefined || t === null) return 0
+  if (t instanceof Date) return Math.floor(t.getTime() / 1000)
+  return Math.max(0, Math.floor(Number(t) || 0))
+}
+
+function octal(buf, off, len, value) {
+  const s = value.toString(8).padStart(len - 1, '0')
+  for (let i = 0; i < len - 1; i++) buf[off + i] = s.charCodeAt(i)
+  buf[off + len - 1] = 0
+}
+const octMax = (len) => 8 ** (len - 1) - 1
+
+function putStr(buf, off, str) {
+  for (let i = 0; i < str.length; i++) buf[off + i] = str.charCodeAt(i)
+}
+
+/** One pax record: "<len> <key>=<value>\n" where len counts itself. */
+function paxRecord(key, value) {
+  const body = enc.encode(` ${key}=${value}\n`)
+  let n = body.length + 1
+  while (String(n).length + body.length !== n) n = String(n).length + body.length
+  const head = enc.encode(String(n))
+  const out = new Uint8Array(n)
+  out.set(head, 0)
+  out.set(body, head.length)
+  return out
+}
+
+function concat(parts, total) {
+  const out = new Uint8Array(total ?? parts.reduce((a, p) => a + p.length, 0))
+  let o = 0
+  for (const p of parts) {
+    out.set(p, o)
+    o += p.length
+  }
+  return out
+}
+
+function rawHeader(name, prefix, typeflag, size, mode, mtime, uid, gid, linkname) {
+  const h = new Uint8Array(BLOCK)
+  putStr(h, 0, name)
+  octal(h, 100, 8, mode)
+  octal(h, 108, 8, uid)
+  octal(h, 116, 8, gid)
+  octal(h, 124, 12, size)
+  octal(h, 136, 12, mtime)
+  h.fill(0x20, 148, 156)
+  h[156] = typeflag.charCodeAt(0)
+  putStr(h, 157, linkname)
+  putStr(h, 257, 'ustar')
+  h[262] = 0
+  h[263] = 0x30
+  h[264] = 0x30
+  putStr(h, 345, prefix)
+  let sum = 0
+  for (let i = 0; i < BLOCK; i++) sum += h[i]
+  const cs = sum.toString(8).padStart(6, '0')
+  putStr(h, 148, cs)
+  h[154] = 0
+  h[155] = 0x20
+  return h
+}
+
+/**
+ * The header block(s) of one entry: a pax header first when needed, then the ustar header.
+ * Exported so the over-8-GiB size path can be tested without 8 GiB of data.
+ * @returns {Uint8Array}
+ */
+export function tarEntryHeader(path, typeflag, size, { mode, mtime, uid = 0, gid = 0, linkname = '' } = {}) {
+  checkTarPath(path)
+  const isDir = typeflag === '5'
+  const full = isDir && !path.endsWith('/') ? `${path}/` : path
+  const pax = []
+  let name = full
+  let prefix = ''
+  const nameBytes = enc.encode(full)
+  if (isAscii(full) && nameBytes.length <= 100) {
+    // plain
+  } else if (isAscii(full) && nameBytes.length <= 255 + 1 + 100) {
+    // ustar prefix/name split at a '/'
+    let at = -1
+    for (let i = full.length - 2; i > 0; i--) {
+      if (full[i] !== '/') continue
+      if (full.length - i - 1 <= 100 && i <= 155) {
+        at = i
+        break
+      }
+    }
+    if (at > 0) {
+      prefix = full.slice(0, at)
+      name = full.slice(at + 1)
+    } else pax.push(['path', full])
+  } else pax.push(['path', full])
+  if (pax.length) {
+    // ustar fallback name: ASCII-only tail, never empty
+    name = full.replace(/[^\x20-\x7e]/g, '_').slice(-100)
+    prefix = ''
+  }
+  let link = linkname
+  if (link && (!isAscii(link) || enc.encode(link).length > 100)) {
+    pax.push(['linkpath', link])
+    link = link.replace(/[^\x20-\x7e]/g, '_').slice(0, 100)
+  }
+  const m = (mode ?? (isDir ? 0o755 : 0o644)) & 0o7777
+  let t = toSeconds(mtime)
+  if (t > octMax(12)) {
+    pax.push(['mtime', String(t)])
+    t = octMax(12)
+  }
+  let u = uid
+  if (u > octMax(8)) {
+    pax.push(['uid', String(u)])
+    u = octMax(8)
+  }
+  let g = gid
+  if (g > octMax(8)) {
+    pax.push(['gid', String(g)])
+    g = octMax(8)
+  }
+  let sz = size
+  if (sz > octMax(12)) {
+    pax.push(['size', String(sz)])
+    sz = octMax(12)
+  }
+  const main = rawHeader(name, prefix, typeflag, sz, m, t, u, g, link)
+  if (!pax.length) return main
+  const recs = pax.map(([k, v]) => paxRecord(k, v))
+  const dataLen = recs.reduce((a, r) => a + r.length, 0)
+  const padded = Math.ceil(dataLen / BLOCK) * BLOCK
+  const tail = (full.replace(/\/$/, '').split('/').pop() || 'x').replace(/[^\x20-\x7e]/g, '_').slice(-60)
+  const ph = rawHeader(`PaxHeaders.0/${tail}`, '', 'x', dataLen, 0o644, 0, 0, 0, '')
+  const body = new Uint8Array(padded)
+  body.set(concat(recs, dataLen), 0)
+  return concat([ph, body, main])
+}
+
+/** A tar writer. Paths are relative, '/'-separated, without '..' or a leading '/'. */
+export function tarWriter() {
+  const chunks = []
+  let total = 0
+  let done = false
+  const push = (c) => {
+    if (done) throw new Error('tar: writer already finished')
+    chunks.push(c)
+    total += c.length
+  }
+  const pad = (n) => (BLOCK - (n % BLOCK)) % BLOCK
+  return {
+    file(path, bytes, { mode, mtime, uid, gid } = {}) {
+      const data = typeof bytes === 'string' ? enc.encode(bytes) : bytes
+      push(tarEntryHeader(path, '0', data.length, { mode, mtime, uid, gid }))
+      if (data.length) {
+        push(data)
+        const p = pad(data.length)
+        if (p) push(new Uint8Array(p))
+      }
+    },
+    dir(path, { mode, mtime, uid, gid } = {}) {
+      push(tarEntryHeader(path, '5', 0, { mode, mtime, uid, gid }))
+    },
+    symlink(path, target, { mtime, uid, gid } = {}) {
+      if (!target) throw new Error('tar: empty symlink target')
+      push(tarEntryHeader(path, '2', 0, { mode: 0o777, mtime, uid, gid, linkname: target }))
+    },
+    /** The archive so far, ended with two zero blocks. */
+    finish() {
+      if (done) throw new Error('tar: writer already finished')
+      push(new Uint8Array(BLOCK * 2))
+      done = true
+      return concat(chunks, total)
+    },
+  }
+}
+
+function parseNum(buf, off, len) {
+  if (buf[off] & 0x80) {
+    // GNU base-256
+    let v = buf[off] & 0x7f
+    for (let i = 1; i < len; i++) v = v * 256 + buf[off + i]
+    return v
+  }
+  let s = ''
+  for (let i = 0; i < len && buf[off + i] !== 0 && buf[off + i] !== 0x20; i++) s += String.fromCharCode(buf[off + i])
+  return s ? parseInt(s, 8) : 0
+}
+
+function cstr(buf, off, len) {
+  let e = off
+  while (e < off + len && buf[e] !== 0) e++
+  return decodeText(buf.subarray(off, e))
+}
+
+function parsePax(data) {
+  const out = {}
+  let p = 0
+  while (p < data.length) {
+    let sp = p
+    while (sp < data.length && data[sp] !== 0x20) sp++
+    const n = parseInt(decodeText(data.subarray(p, sp)), 10)
+    if (!(n > 0) || p + n > data.length) break
+    const rec = decodeText(data.subarray(sp + 1, p + n - 1))
+    const eq = rec.indexOf('=')
+    if (eq > 0) out[rec.slice(0, eq)] = rec.slice(eq + 1)
+    p += n
+  }
+  return out
+}
+
+/**
+ * Walks a tar's headers without reading file data. `read(offset, length)` returns a Uint8Array (sync or
+ * promise) so a Blob can be walked by slices. Yields real entries only (pax 'x'/'g' and GNU 'L'/'K' are folded
+ * into the entry they describe): {name, type, size, mode, mtime, uid, gid, linkname, xattrs, offset} where `offset` is
+ * the start of the entry's data. `type` is the typeflag character ('0' for the old NUL flag).
+ */
+export async function* scanTar(read, totalLength) {
+  let off = 0
+  let pax = null
+  let gnuName = null
+  let gnuLink = null
+  const need = async (o, n) => {
+    const b = await read(o, n)
+    if (b.length < n) throw new Error('tar: truncated archive')
+    return b
+  }
+  while (off + BLOCK <= totalLength) {
+    const h = await need(off, BLOCK)
+    if (h.every((b) => b === 0)) return
+    let sum = 0
+    for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 0x20 : h[i]
+    if (sum !== parseNum(h, 148, 8)) throw new Error(`tar: bad header checksum at ${off}`)
+    const type = h[156] === 0 ? '0' : String.fromCharCode(h[156])
+    let size = parseNum(h, 124, 12)
+    off += BLOCK
+    if (type === 'x' || type === 'g' || type === 'L' || type === 'K') {
+      const data = await need(off, size)
+      off += Math.ceil(size / BLOCK) * BLOCK
+      if (type === 'x') pax = { ...(pax || {}), ...parsePax(data) }
+      else if (type === 'L') gnuName = cstr(data, 0, data.length)
+      else if (type === 'K') gnuLink = cstr(data, 0, data.length)
+      continue
+    }
+    const prefix = cstr(h, 345, 155)
+    let name = cstr(h, 0, 100)
+    if (prefix) name = `${prefix}/${name}`
+    let linkname = cstr(h, 157, 100)
+    let mtime = parseNum(h, 136, 12)
+    let uid = parseNum(h, 108, 8)
+    let gid = parseNum(h, 116, 8)
+    if (gnuName !== null) name = gnuName
+    if (gnuLink !== null) linkname = gnuLink
+    if (pax) {
+      if (pax.path !== undefined) name = pax.path
+      if (pax.linkpath !== undefined) linkname = pax.linkpath
+      if (pax.size !== undefined) size = Number(pax.size)
+      if (pax.mtime !== undefined) mtime = Math.floor(Number(pax.mtime))
+      if (pax.uid !== undefined) uid = Number(pax.uid)
+      if (pax.gid !== undefined) gid = Number(pax.gid)
+    }
+    const xattrs = {}
+    if (pax) for (const [k, v] of Object.entries(pax)) if (k.startsWith('SCHILY.xattr.')) xattrs[k.slice(13)] = v
+    pax = gnuName = gnuLink = null
+    const hasData = type === '0' || type === '7' || (type !== '1' && type !== '2' && type !== '5' && size > 0)
+    yield { name, type, size, mode: parseNum(h, 100, 8), mtime, uid, gid, linkname, xattrs, offset: off }
+    if (hasData) off += Math.ceil(size / BLOCK) * BLOCK
+  }
+}
+
+/** scanTar over a Uint8Array. */
+export function scanTarBytes(u8) {
+  return scanTar((o, n) => u8.subarray(o, o + n), u8.length)
+}

package/src/host/ctl.js

@@ -719,6 +719,14 @@ export const DEFAULT_CAPS = [
   'CAP_CHOWN', 'CAP_DAC_OVERRIDE', 'CAP_FSETID', 'CAP_FOWNER', 'CAP_MKNOD', 'CAP_NET_RAW', 'CAP_SETGID',
   'CAP_SETUID', 'CAP_SETFCAP', 'CAP_SETPCAP', 'CAP_NET_BIND_SERVICE', 'CAP_SYS_CHROOT', 'CAP_KILL', 'CAP_AUDIT_WRITE',
 ]
+/** Every capability, as internal/engine/ocispec.go allCaps (Linux 6.1: 0..40); what --privileged grants. */
+export const ALL_CAPS = [
+  'CHOWN', 'DAC_OVERRIDE', 'DAC_READ_SEARCH', 'FOWNER', 'FSETID', 'KILL', 'SETGID', 'SETUID', 'SETPCAP', 'LINUX_IMMUTABLE',
+  'NET_BIND_SERVICE', 'NET_BROADCAST', 'NET_ADMIN', 'NET_RAW', 'IPC_LOCK', 'IPC_OWNER', 'SYS_MODULE', 'SYS_RAWIO', 'SYS_CHROOT',
+  'SYS_PTRACE', 'SYS_PACCT', 'SYS_ADMIN', 'SYS_BOOT', 'SYS_NICE', 'SYS_RESOURCE', 'SYS_TIME', 'SYS_TTY_CONFIG', 'MKNOD', 'LEASE',
+  'AUDIT_WRITE', 'AUDIT_CONTROL', 'SETFCAP', 'MAC_OVERRIDE', 'MAC_ADMIN', 'SYSLOG', 'WAKE_ALARM', 'BLOCK_SUSPEND', 'AUDIT_READ',
+  'PERFMON', 'BPF', 'CHECKPOINT_RESTORE',
+].map(c => `CAP_${c}`)
 const MASKED = ['/proc/acpi', '/proc/asound', '/proc/kcore', '/proc/keys', '/proc/latency_stats', '/proc/timer_list', '/proc/timer_stats', '/proc/sched_debug', '/proc/scsi', '/sys/firmware']
 const READONLY = ['/proc/bus', '/proc/fs', '/proc/irq', '/proc/sys', '/proc/sysrq-trigger']
 const ETC_FILES = ['hosts', 'resolv.conf', 'hostname']
@@ -751,6 +759,8 @@ function defaultMounts() {
  * @param {{destination:string, files:Record<string,{data:Uint8Array|string, mode?:number}|string>, file?:string, readOnly?:boolean}[]} [o.mounts]
  *        each mount is a share vol<i> with these files; `file` binds only that file of the share
  * @param {string} [o.arch]          for imageconfig.json (default arm64)
+ * @param {boolean} [o.privileged]   docker run --privileged as dockerd writes it: all capabilities, no masked or
+ *        read-only paths, /sys read-write, every device allowed (default false; only builder machines use it)
  * @param {boolean} [o.init]         docker run --init: tini at /sbin/docker-init runs the command
  * @param {boolean} [o.keepRunning]  wmd's keep-running init at /sbin/docker-init: the container stays up
  *        after its command ends; args may be [] (nothing runs until ControlClient.startCommand). Send the
@@ -785,6 +795,13 @@ export function containerBundle(o) {
       resources: { devices: [{ allow: false, access: 'rwm' }] },
     },
   }
+  if (o.privileged) {
+    spec.process.capabilities = { bounding: ALL_CAPS, effective: ALL_CAPS, inheritable: ALL_CAPS, permitted: ALL_CAPS }
+    spec.linux.maskedPaths = []
+    spec.linux.readonlyPaths = []
+    spec.linux.resources = { devices: [{ allow: true, access: 'rwm' }] }
+    for (const m of spec.mounts) if (m.destination === '/sys') m.options = m.options.filter(x => x !== 'ro')
+  }
   if (o.init || o.keepRunning) {
     // as dockerd binds its docker-init (internal/engine/ocispec.go): the guest's tini, or its own init for keep-running
     spec.mounts.push({ destination: DOCKER_INIT, type: 'bind', source: o.keepRunning ? '/sbin/init' : '/sbin/tini', options: ['bind', 'ro'] })