@threadedstack/wmd-web 0.1.16 → 0.1.18

package/src/build/context.js

@@ -0,0 +1,349 @@
+// The build context (docs/plans/web-buildkit-impl.md 5.6): turn a file source, a plain object or a tar into the
+// one tar the page sends to the builder. .dockerignore is applied here with a port of moby's patternmatcher, so
+// ignored files are never read; BuildKit applies it again and stays the source of truth.
+import { matcher, parseDockerignore } from './dockerignore.js'
+import { decodeText, scanTar, tarWriter } from './tar.js'
+
+const MiB = 1024 * 1024
+/** Defaults from the plan (section 3): wmn's old limits. */
+export const DEFAULT_LIMITS = { bytes: 128 * MiB, files: 20000, fileBytes: 64 * MiB }
+
+/** `reason`: 'too-big' | 'too-many' | 'file-too-big' | 'unsafe-path' | 'bad-dockerignore' | 'bad-source'. */
+export class ContextError extends Error {
+  constructor(reason, path, message) {
+    super(message || `build context: ${reason}${path ? ` (${path})` : ''}`)
+    this.name = 'ContextError'
+    this.reason = reason
+    this.path = path
+  }
+}
+
+const enc = new TextEncoder()
+const dec = new TextDecoder()
+const DOCKERIGNORE = '.dockerignore'
+const DOCKERFILE = 'Dockerfile'
+
+/**
+ * Normalizes a context path: drops '.' and empty components ('./a', 'a//b', 'a/'), rejects '..', absolute
+ * paths, backslash-free NUL-free. Returns null for the context root itself.
+ */
+export function normalizeContextPath(p) {
+  if (typeof p !== 'string') throw new ContextError('unsafe-path', String(p), 'build context: path is not a string')
+  if (p.includes('\0')) throw new ContextError('unsafe-path', p, 'build context: NUL in path')
+  if (p.startsWith('/')) throw new ContextError('unsafe-path', p, `build context: absolute path ${p}`)
+  const out = []
+  for (const part of p.split('/')) {
+    if (part === '' || part === '.') continue
+    if (part === '..') throw new ContextError('unsafe-path', p, `build context: '..' in path ${p}`)
+    out.push(part)
+  }
+  return out.length ? out.join('/') : null
+}
+
+/**
+ * Path order by component, so a folder's whole subtree follows it ('d', 'd/a', 'd-x'), as `tar -c` writes: GNU tar
+ * sets a folder's mtime once a later entry is outside it, so a child written after that ('d-x' before 'd/a' in
+ * plain string order) would leave the folder with the time of the unpack.
+ */
+export function byComponents(a, b) {
+  const x = a.split('/')
+  const y = b.split('/')
+  for (let i = 0; i < Math.min(x.length, y.length); i++) if (x[i] !== y[i]) return x[i] < y[i] ? -1 : 1
+  return x.length - y.length
+}
+
+/**
+ * True when a path goes through one of the context's symlinks. Symlinks themselves are kept whatever they point
+ * at, as the docker CLI sends them (BuildKit resolves them inside its own copy of the context); a path under one
+ * would write through the link when the builder unpacks the tar.
+ */
+function underLink(p, links) {
+  for (let i = p.indexOf('/'); i > 0; i = p.indexOf('/', i + 1)) if (links.has(p.slice(0, i))) return true
+  return false
+}
+
+function limitsOf(o) {
+  const l = { ...DEFAULT_LIMITS }
+  for (const k of ['bytes', 'files', 'fileBytes']) if (o && o[k] !== undefined) l[k] = o[k]
+  return l
+}
+
+function isBlob(x) {
+  return typeof Blob !== 'undefined' && x instanceof Blob
+}
+
+function isFileSource(x) {
+  return x && typeof x.list === 'function' && typeof x.stat === 'function' && typeof x.read === 'function'
+}
+
+/** Adapts {[path]: Uint8Array|string} to the file-source shape. A key ending in '/' is a directory. */
+function objectSource(obj) {
+  const m = new Map()
+  for (const [k, v] of Object.entries(obj)) {
+    if (k.endsWith('/')) {
+      m.set(k, null)
+      continue
+    }
+    if (typeof v === 'string') m.set(k, enc.encode(v))
+    else if (v instanceof Uint8Array) m.set(k, v)
+    else throw new ContextError('bad-source', k, `build context: value of ${k} is not a string or Uint8Array`)
+  }
+  return {
+    async list() {
+      return [...m.keys()]
+    },
+    async stat(p) {
+      const v = m.get(p)
+      if (v === undefined) throw new ContextError('bad-source', p, `build context: no such path ${p}`)
+      return v === null ? { type: 'dir', size: 0 } : { type: 'file', size: v.length }
+    },
+    async read(p) {
+      return m.get(p)
+    },
+  }
+}
+
+function tarReader(input) {
+  const blob = isBlob(input)
+  const u8 = input instanceof ArrayBuffer ? new Uint8Array(input) : input
+  const total = blob ? input.size : u8.length
+  const read = blob
+    ? async (o, n) => new Uint8Array(await input.slice(o, o + n).arrayBuffer())
+    : (o, n) => u8.subarray(o, o + n)
+  return { blob, u8, total, read }
+}
+
+/**
+ * The bytes of one regular file of a tar (Uint8Array, ArrayBuffer or Blob), or null when the path is absent or
+ * not a regular file. Names are matched after normalisation, so './a' and 'a' are the same; pax and GNU long
+ * names are handled by scanTar. When a path appears twice the last entry wins, as when the tar is extracted.
+ */
+export async function readTarFile(tar, path) {
+  const want = normalizeContextPath(path)
+  const { read, total } = tarReader(tar)
+  let hit = null
+  for await (const e of scanTar(read, total)) {
+    let n = null
+    try {
+      n = normalizeContextPath(e.name)
+    } catch {
+      continue
+    }
+    if (n === want) hit = e.type === '0' || e.type === '7' ? e : null
+  }
+  if (!hit) return null
+  return hit.size ? new Uint8Array(await read(hit.offset, hit.size)) : new Uint8Array(0)
+}
+
+async function packTar(input, dockerfile, lim) {
+  const { blob, u8, total, read } = tarReader(input)
+  let files = 0
+  let bytes = 0
+  const links = new Set()
+  // every folder an entry so far sits in: a symlink made later in one of them would sit over files already written
+  const parents = new Set()
+  try {
+    for await (const e of scanTar(read, total)) {
+      const name = normalizeContextPath(e.name) ?? ''
+      if (underLink(name, links)) throw new ContextError('unsafe-path', e.name, `build context: ${e.name} is under a symlink`)
+      for (let i = name.indexOf('/'); i > 0; i = name.indexOf('/', i + 1)) parents.add(name.slice(0, i))
+      if (e.type === '5') continue
+      if (e.type === '3' || e.type === '4' || e.type === '6') {
+        throw new ContextError('unsafe-path', e.name, `build context: device or fifo entry ${e.name}`)
+      }
+      if (e.type === '2') {
+        if (parents.has(name)) throw new ContextError('unsafe-path', e.name, `build context: symlink ${e.name} comes after entries inside it`)
+        links.add(name)
+      }
+      if (e.type === '1') {
+        // a target outside the context, or reached through a symlink, would make tar link a file outside it
+        let target = null
+        try {
+          target = normalizeContextPath(e.linkname)
+        } catch {}
+        if (!target || underLink(target, links)) {
+          throw new ContextError('unsafe-path', e.name, `build context: hardlink ${e.name} -> ${e.linkname} leaves the context`)
+        }
+      }
+      if (++files > lim.files) throw new ContextError('too-many', e.name)
+      const data = e.type === '0' || e.type === '7' ? e.size : 0
+      if (data > lim.fileBytes) throw new ContextError('file-too-big', e.name)
+      bytes += data
+      if (bytes > lim.bytes) throw new ContextError('too-big', e.name)
+    }
+  } catch (err) {
+    if (err instanceof ContextError) throw err
+    throw new ContextError('bad-source', '', `build context: ${err.message}`)
+  }
+  const tar = blob ? new Uint8Array(await input.arrayBuffer()) : u8
+  return { tar, files, bytes, ignored: 0, dockerfile }
+}
+
+function mtimeOf(st) {
+  return st.mtime
+}
+
+/**
+ * Packs a build context.
+ * @param source an async file source {list, stat, read}, a plain {[path]: Uint8Array|string}, or a tar
+ *   (Uint8Array, ArrayBuffer or Blob), which is passed through after the limit checks.
+ * @param opts.dockerfile  Dockerfile text given separately: the entry named 'Dockerfile' is then left out of
+ *   the tar (the builder sends it on its own) and the text is returned as `dockerfile`.
+ * @param opts.dockerfileName  the Dockerfile's path in the context (default 'Dockerfile'). Like BuildKit, a
+ *   `<dockerfileName>.dockerignore` in the source is used instead of `.dockerignore` (both stay in the tar).
+ * @param opts.limits  {bytes, files, fileBytes}, default 128 MiB, 20,000, 64 MiB.
+ * @returns {{tar: Uint8Array, files: number, bytes: number, ignored: number, dockerfile?: string}}
+ *   files = regular files and symlinks in the tar; bytes = their content; ignored = paths .dockerignore removed.
+ *
+ * .dockerignore and the Dockerfile stay in the tar even when a pattern matches them: the docker CLI always sends
+ * both (builder/remotecontext), and BuildKit's frontend reads them from the context.
+ * File mtime is in seconds (or a Date); a number is not guessed to be milliseconds.
+ */
+export async function packContext(source, { dockerfile, dockerfileName = DOCKERFILE, limits } = {}) {
+  const lim = limitsOf(limits)
+  if (dockerfile !== undefined && typeof dockerfile !== 'string') {
+    throw new ContextError('bad-source', '', 'build context: dockerfile must be text')
+  }
+  if (source instanceof Uint8Array || source instanceof ArrayBuffer || isBlob(source)) {
+    return packTar(source, dockerfile, lim)
+  }
+  const src = isFileSource(source) ? source : source && typeof source === 'object' ? objectSource(source) : null
+  if (!src) throw new ContextError('bad-source', '', 'build context: source must be a file source, an object or a tar')
+
+  // 1. the paths, normalized and sorted (determinism: the tar does not depend on listing order)
+  const raw = await src.list()
+  const byNorm = new Map() // normalized path -> the path to give src.stat/read
+  const explicitDirs = new Set()
+  let rootRaw // how the source spells its own root ('.', './', ''), if it lists it
+  for (const p of raw) {
+    const n = normalizeContextPath(p)
+    if (n === null) {
+      rootRaw ??= p
+      continue
+    }
+    if (!byNorm.has(n)) byNorm.set(n, p)
+    if (p.endsWith('/')) explicitDirs.add(n)
+  }
+  const sorted = [...byNorm.keys()].sort(byComponents)
+
+  // 2. .dockerignore, read before anything else
+  let ignoredFn = () => false
+  const specific = `${dockerfileName}${DOCKERIGNORE}`
+  const ignoreName = byNorm.has(specific) ? specific : DOCKERIGNORE
+  if (byNorm.has(ignoreName)) {
+    const text = decodeText(await src.read(byNorm.get(ignoreName)))
+    try {
+      ignoredFn = matcher(parseDockerignore(text)).ignored
+    } catch (e) {
+      throw new ContextError('bad-dockerignore', ignoreName, `build context: ${ignoreName}: ${e.message}`)
+    }
+  }
+  const mustKeep = (p) => p === DOCKERIGNORE || p === specific || p === dockerfileName
+  const omit = (p) => dockerfile !== undefined && p === dockerfileName
+
+  // 3. filter, before stat/read
+  let ignored = 0
+  const kept = []
+  for (const p of sorted) {
+    if (omit(p)) continue
+    if (!mustKeep(p) && ignoredFn(p, explicitDirs.has(p))) {
+      ignored++
+      continue
+    }
+    kept.push(p)
+  }
+
+  // 4. stat and limit checks, all before any read
+  const entries = []
+  let files = 0
+  let bytes = 0
+  const statOf = async (p) => {
+    const st = await src.stat(byNorm.get(p))
+    if (!st || (st.type !== 'file' && st.type !== 'dir' && st.type !== 'symlink')) {
+      throw new ContextError('bad-source', p, `build context: bad stat for ${p}`)
+    }
+    return st
+  }
+  const stats = []
+  for (let i = 0; i < kept.length; i += 32) {
+    stats.push(...(await Promise.all(kept.slice(i, i + 32).map(statOf))))
+  }
+  const links = new Set(kept.filter((p, i) => stats[i].type === 'symlink'))
+  for (const p of kept) if (underLink(p, links)) throw new ContextError('unsafe-path', p, `build context: ${p} is under a symlink`)
+  kept.forEach((p, i) => {
+    const st = stats[i]
+    if (st.type === 'dir') {
+      entries.push({ path: p, st })
+      return
+    }
+    if (++files > lim.files) throw new ContextError('too-many', p)
+    if (st.type === 'symlink') {
+      if (!st.target || st.target.includes('\0')) throw new ContextError('bad-source', p, `build context: bad symlink ${p}`)
+
+    } else if (st.size !== undefined) {
+      if (st.size > lim.fileBytes) throw new ContextError('file-too-big', p)
+      bytes += st.size
+      if (bytes > lim.bytes) throw new ContextError('too-big', p)
+    }
+    entries.push({ path: p, st })
+  })
+
+  // 5. write; parents of kept entries that were not kept (or not listed) become directory entries
+  const w = tarWriter()
+  // The root folder entry './' goes first, as `tar -c .` writes it: a COPY of the context's root gives its destination folder this
+  // time and mode, and the builder's tar extraction sets the context folder from it. Mode and mtime come from the source's own
+  // root when it lists one, else mode 0755 and the newest mtime of what is in the tar (none known: 0).
+  let rootSt = {}
+  if (rootRaw !== undefined) {
+    try {
+      rootSt = (await src.stat(rootRaw)) || {}
+    } catch {
+      rootSt = {}
+    }
+    if (rootSt.type && rootSt.type !== 'dir') rootSt = {}
+  }
+  const secs = (t) => (t instanceof Date ? Math.floor(t.getTime() / 1000) : Number(t) || 0)
+  let newest = 0
+  for (const { st } of entries) newest = Math.max(newest, secs(mtimeOf(st)))
+  w.dir('./', { mode: rootSt.mode !== undefined ? rootSt.mode & 0o7777 : 0o755, mtime: rootSt.mtime !== undefined ? rootSt.mtime : newest })
+  const made = new Set()
+  const keptDirs = new Set(entries.filter((e) => e.st.type === 'dir').map((e) => e.path))
+  const ensureDirs = async (p) => {
+    const parts = p.split('/')
+    for (let i = 1; i < parts.length; i++) {
+      const d = parts.slice(0, i).join('/')
+      if (made.has(d) || keptDirs.has(d)) continue
+      made.add(d)
+      let st = {}
+      if (byNorm.has(d)) {
+        try {
+          st = await statOf(d)
+        } catch {
+          st = {}
+        }
+      }
+      w.dir(d, { mode: st.mode !== undefined ? st.mode & 0o7777 : 0o755, mtime: mtimeOf(st) })
+    }
+  }
+  // entries are sorted, so a kept dir precedes its children; implicit parents are emitted just before first use
+  let written = 0
+  for (const { path, st } of entries) {
+    await ensureDirs(path)
+    if (st.type === 'dir') {
+      made.add(path)
+      w.dir(path, { mode: st.mode !== undefined ? st.mode & 0o7777 : 0o755, mtime: mtimeOf(st) })
+    } else if (st.type === 'symlink') {
+      w.symlink(path, st.target, { mtime: mtimeOf(st) })
+    } else {
+      const data = await src.read(byNorm.get(path))
+      if (!(data instanceof Uint8Array)) throw new ContextError('bad-source', path, `build context: read(${path}) is not bytes`)
+      if (data.length > lim.fileBytes) throw new ContextError('file-too-big', path)
+      if (st.size === undefined || st.size !== data.length) {
+        written += data.length - (st.size ?? 0)
+        if (bytes + written > lim.bytes) throw new ContextError('too-big', path)
+      }
+      w.file(path, data, { mode: st.mode !== undefined ? st.mode & 0o7777 : 0o644, mtime: mtimeOf(st) })
+    }
+  }
+  return { tar: w.finish(), files, bytes: bytes + written, ignored, dockerfile }
+}