@taliesinb/mash 0.0.0-stage → 0.1.0

package/src/vm/external.ts

@@ -0,0 +1,265 @@
+/**
+ * Running external commands as real child processes.
+ *
+ * stdio is connected through the VM's open files: pipes and captures are
+ * pumped (so trace listeners see the bytes), /dev/null maps to `ignore`, and
+ * untraced host files are passed as raw fds. Process-substitution paths
+ * (`/dev/fd/N` naming a VM fd) are mapped onto extra child stdio slots
+ * (`devfd`, macOS) or onto named FIFOs (`fifo`, Linux; see fifo.ts).
+ */
+
+import { spawn as nodeSpawn, type ChildProcess, type StdioOptions } from 'node:child_process'
+import * as os from 'node:os'
+import type { Interpreter } from './interp.ts'
+import { FifoSet } from './fifo.ts'
+import { BrokenPipeError, HostFile, NullFile, type OpenFile } from './io.ts'
+import type { Proc } from './proc.ts'
+import { HOST_UMASK } from './state.ts'
+
+export interface SpawnRequest {
+  file: string
+  args: string[]
+  argv0: string
+  cwd: string
+  env: Record<string, string>
+  stdio: StdioOptions
+}
+
+/** Spawn a child. Replace it to confine children (e.g. through the harness's `ctx.sandbox.confine`). */
+export type SpawnFn = (req: SpawnRequest) => ChildProcess | Promise<ChildProcess>
+
+export const defaultSpawn: SpawnFn = (req) =>
+  nodeSpawn(req.file, req.args, { argv0: req.argv0, cwd: req.cwd, env: req.env, stdio: req.stdio })
+
+export interface ExternalRequest {
+  execId: number
+  path: string
+  argv: string[]
+  env: Record<string, string>
+  cwd: string
+  proc: Proc
+  readsStdin: boolean | 'maybe'
+}
+
+function signalNumber(sig: NodeJS.Signals | null): number {
+  if (!sig) return 0
+  return (os.constants.signals as Record<string, number>)[sig] ?? 15
+}
+
+type Slot =
+  | { kind: 'ignore' }
+  | { kind: 'fd'; fd: number }
+  | { kind: 'pump-in'; file: OpenFile }
+  | { kind: 'pump-out'; file: OpenFile }
+
+function outSlot(file: OpenFile | undefined, traced: boolean): Slot {
+  if (!file || !file.writable) return { kind: 'ignore' }
+  if (file instanceof NullFile) return { kind: 'ignore' }
+  if (file instanceof HostFile && !traced) return { kind: 'fd', fd: file.fd }
+  return { kind: 'pump-out', file }
+}
+
+function inSlot(file: OpenFile | undefined, traced: boolean, readsStdin: boolean | 'maybe'): Slot {
+  if (!file || !file.readable || readsStdin === false) return { kind: 'ignore' }
+  if (file instanceof NullFile) return { kind: 'ignore' }
+  if (file instanceof HostFile && !traced && file.buffered === 0) return { kind: 'fd', fd: file.fd }
+  return { kind: 'pump-in', file }
+}
+
+/** Arguments that open one of the child's own stdio fds by name. */
+const STD_NAMES: Record<string, number> = { '/dev/stdin': 0, '/dev/stdout': 1, '/dev/stderr': 2, '/dev/fd/0': 0, '/dev/fd/1': 1, '/dev/fd/2': 2 }
+
+/** An argument served through a named FIFO (`fifo` fd passing). */
+interface FifoArg {
+  arg: number
+  file: OpenFile
+  /** The child reads it (mash feeds the FIFO); else the child writes it (mash drains it). */
+  feed: boolean
+  /** The VM fd it names (≥ 3), for the `/dev/fd/K` fallback when no FIFO can be made. */
+  vmFd?: number
+  /** stdin slot this argument displaced (restored on fallback). */
+  displaced?: Slot
+}
+
+export async function runExternal(interp: Interpreter, req: ExternalRequest): Promise<number> {
+  const rt = interp.rt
+  const { proc } = req
+  const traced = rt.listeners.length > 0
+  // a behavior's "file operands, so no stdin" does not hold for an operand that names stdin
+  const readsStdin = req.readsStdin === false && req.argv.some((a, i) => i > 0 && STD_NAMES[a] === 0) ? 'maybe' : req.readsStdin
+  const slots: Slot[] = [
+    inSlot(proc.fds.get(0), traced, readsStdin),
+    outSlot(proc.fds.get(1), traced),
+    outSlot(proc.fds.get(2), traced),
+  ]
+  const fifoMode = rt.fdPassing === 'fifo'
+  const fifoArgs: FifoArg[] = []
+  const extraSlot = (file: OpenFile): string => {
+    const k = slots.length
+    slots.push(file.readable && !file.writable ? { kind: 'pump-in', file } : { kind: 'pump-out', file })
+    return `/dev/fd/${k}`
+  }
+  // /dev/fd/N arguments naming VM fds (process substitution, `exec 3<...`)
+  const args = req.argv.slice(1).map((a, i) => {
+    const m = /^\/dev\/fd\/(\d+)$/.exec(a)
+    const n = m ? Number(m[1]) : fifoMode ? STD_NAMES[a] : undefined
+    if (n === undefined) return a
+    if (n < 3) {
+      // The child's own fd 0-2. Under `fifo` (Linux) a pumped slot is a socket, which the
+      // child cannot open by name: serve the argument through a FIFO on the same VM file.
+      const slot = slots[n]!
+      if (!fifoMode || (slot.kind !== 'pump-in' && slot.kind !== 'pump-out')) return a
+      const feed = slot.kind === 'pump-in'
+      // one consumer for the stdin data: the child's fd 0 becomes /dev/null
+      if (feed) slots[n] = { kind: 'ignore' }
+      fifoArgs.push({ arg: i, file: slot.file, feed, displaced: feed ? slot : undefined })
+      return a
+    }
+    const file = proc.fds.get(n)
+    if (!file) return a
+    if (!fifoMode) return extraSlot(file)
+    fifoArgs.push({ arg: i, file, feed: file.readable && !file.writable, vmFd: n })
+    return a
+  })
+  let fifos: { set: FifoSet; paths: string[] } | undefined
+  if (fifoArgs.length > 0) {
+    try {
+      fifos = await FifoSet.create(fifoArgs.length)
+      fifoArgs.forEach((f, j) => (args[f.arg] = fifos!.paths[j]!))
+    } catch {
+      // no FIFO possible (mkfifo missing, temp dir not writable): fall back to socket slots
+      for (const f of fifoArgs) {
+        if (f.vmFd !== undefined) args[f.arg] = extraSlot(f.file)
+        else if (f.displaced) slots[0] = f.displaced
+      }
+      fifoArgs.length = 0
+    }
+  }
+  try {
+    return await spawnAndPump(interp, req, slots, args, fifos ? fifoArgs.map((f, j) => ({ ...f, path: fifos!.paths[j]! })) : [], fifos?.set)
+  } finally {
+    fifos?.set.cleanup()
+  }
+}
+
+async function spawnAndPump(
+  interp: Interpreter,
+  req: ExternalRequest,
+  slots: Slot[],
+  args: string[],
+  fifoArgs: (FifoArg & { path: string })[],
+  fifoSet: FifoSet | undefined,
+): Promise<number> {
+  const rt = interp.rt
+  const { proc } = req
+  const stdio: StdioOptions = slots.map((s) => (s.kind === 'ignore' ? 'ignore' : s.kind === 'fd' ? s.fd : 'pipe'))
+
+  // Node cannot start a child with a closed fd or another umask: use a tiny sh exec prelude when needed
+  const prelude: string[] = []
+  if (proc.state.umask !== HOST_UMASK) prelude.push(`umask ${proc.state.umask.toString(8).padStart(4, '0')}`)
+  for (const fd of [0, 1, 2]) if (!proc.fds.get(fd)) prelude.push(`exec ${fd}>&-`)
+  let file = req.path
+  let spawnArgs = args
+  if (prelude.length > 0) {
+    file = '/bin/sh'
+    spawnArgs = ['-c', `${prelude.join('; ')}; exec "$@"`, 'sh', req.path, ...args]
+  }
+
+  let child: ChildProcess
+  try {
+    child = await rt.spawn({ file, args: spawnArgs, argv0: prelude.length > 0 ? 'sh' : req.argv[0], cwd: req.cwd, env: req.env, stdio })
+  } catch (e) {
+    interp.errorMessage(proc, `${req.argv[0]}: ${(e as Error).message}`)
+    return 126
+  }
+
+  const exited = new AbortController()
+  const pumps: Promise<void>[] = []
+  let spawnError: NodeJS.ErrnoException | null = null
+
+  const exit = new Promise<number>((resolve) => {
+    child.once('error', (e: NodeJS.ErrnoException) => {
+      spawnError = e
+      exited.abort()
+      resolve(e.code === 'ENOENT' ? 127 : 126)
+    })
+    child.once('exit', (code, signal) => {
+      exited.abort()
+      resolve(code ?? 128 + signalNumber(signal))
+    })
+  })
+
+  rt.emit({ type: 'command:spawn', execId: req.execId, osPid: child.pid })
+
+  const onAbort = (): void => {
+    child.kill('SIGTERM')
+    setTimeout(() => child.kill('SIGKILL'), 2000).unref()
+  }
+  rt.signal.addEventListener('abort', onAbort, { once: true })
+  proc.jobChildren?.add(child)
+  if (proc.jobSignal?.aborted) child.kill('SIGTERM')
+
+  slots.forEach((slot, i) => {
+    const stream = child.stdio[i]
+    if (!stream) return
+    if (slot.kind === 'pump-in') {
+      const w = stream as NodeJS.WritableStream & { destroyed?: boolean; writableNeedDrain?: boolean }
+      w.on('error', () => {}) // EPIPE when the child exits without reading everything
+      pumps.push((async () => {
+        try {
+          for (;;) {
+            const chunk = await slot.file.read(exited.signal)
+            if (chunk === null) {
+              if (!exited.signal.aborted) w.end()
+              return
+            }
+            if (exited.signal.aborted || w.destroyed) {
+              slot.file.unread(chunk)
+              return
+            }
+            if (!w.write(chunk)) {
+              await new Promise<void>((resolve) => {
+                const done = (): void => {
+                  w.removeListener('drain', done)
+                  exited.signal.removeEventListener('abort', done)
+                  resolve()
+                }
+                w.once('drain', done)
+                exited.signal.addEventListener('abort', done, { once: true })
+              })
+            }
+          }
+        } catch {
+          // reading the source failed: give the child EOF
+          w.end()
+        }
+      })())
+    } else if (slot.kind === 'pump-out') {
+      const r = stream as NodeJS.ReadableStream & { destroy(): void }
+      pumps.push((async () => {
+        try {
+          for await (const chunk of r) {
+            await slot.file.write(chunk as Uint8Array)
+          }
+        } catch (e) {
+          if (e instanceof BrokenPipeError) r.destroy()
+          else r.destroy()
+        }
+      })())
+    }
+  })
+
+  if (fifoSet) {
+    for (const f of fifoArgs) pumps.push(f.feed ? fifoSet.feed(f.path, f.file, exited.signal) : fifoSet.drain(f.path, f.file, exited.signal))
+  }
+
+  const status = await exit
+  // stdout/stderr pumps finish at EOF; stdin pumps stop at exit
+  await Promise.all(pumps)
+  rt.signal.removeEventListener('abort', onAbort)
+  proc.jobChildren?.delete(child)
+  if (spawnError) {
+    interp.errorMessage(proc, `${req.argv[0]}: ${(spawnError as NodeJS.ErrnoException).message}`)
+  }
+  return status
+}

package/src/vm/fifo.ts

@@ -0,0 +1,226 @@
+/**
+ * Named-FIFO fallback for handing VM file descriptors to external commands.
+ *
+ * Node creates child stdio pipes as UNIX socketpairs. macOS lets a child open
+ * `/dev/fd/N` on a socket (it dups the descriptor); Linux reopens the file
+ * through `/proc/self/fd/N`, which fails with ENXIO for sockets. So on Linux a
+ * process substitution (`diff <(a) <(b)`) or an `exec 3<f; cmd /dev/fd/3`
+ * passed to an external command cannot use the socket slot. Instead the child
+ * gets the path of a named FIFO in a private temp dir, and mash pumps between
+ * the FIFO and the VM file (so taps and tracing still see the bytes).
+ *
+ * Opening a FIFO blocks until the other side opens it, and a blocking open
+ * would hold one of libuv's four threadpool threads. So mash never does a
+ * blocking open: everything here uses `O_NONBLOCK` descriptors and polls with
+ * a short backoff (immediate while data flows, at most {@link MAX_DELAY} ms
+ * when idle).
+ *
+ * - **The child reads** (`<(cmd)`): mash polls `open(O_WRONLY|O_NONBLOCK)`,
+ *   which fails with ENXIO until the child has opened the FIFO for reading.
+ *   Only then does it write, so no data can be lost in a pipe nobody holds,
+ *   and closing it at the VM file's EOF gives the child EOF. If the child
+ *   exits without opening it, nothing is consumed from the VM file.
+ * - **The child writes** (`>(cmd)`): mash opens the read side
+ *   `O_RDONLY|O_NONBLOCK` up front (that never blocks, and lets the child's
+ *   own open for writing succeed at once) and polls `read`. A read of 0 means
+ *   "no writer right now", so the pump only stops once the child has exited
+ *   and every writer (including grandchildren) has closed.
+ *
+ * The strategy is chosen per machine: {@link defaultFdPassing} (Linux and the
+ * other non-macOS systems → `fifo`, macOS → `devfd`), overridable with the
+ * `MASH_PROCSUBST=fifo|devfd` environment variable or `MachineOptions.procSubst`.
+ */
+
+import { execFile } from 'node:child_process'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import type { OpenFile } from './io.ts'
+
+/** How VM fds named by `/dev/fd/N` arguments reach an external command. */
+export type FdPassing = 'devfd' | 'fifo'
+
+/** `MASH_PROCSUBST` if set to `fifo`/`devfd`, else `devfd` on macOS and `fifo` elsewhere. */
+export function defaultFdPassing(env: Record<string, string | undefined> = process.env): FdPassing {
+  const v = env.MASH_PROCSUBST
+  if (v === 'fifo' || v === 'devfd') return v
+  return process.platform === 'darwin' ? 'devfd' : 'fifo'
+}
+
+const MAX_DELAY = 16
+
+/** Exponential backoff for polling: 0 (a macrotask turn), 1, 2, 4 … MAX_DELAY ms. */
+class Backoff {
+  private delay = 0
+  reset(): void {
+    this.delay = 0
+  }
+  /** Sleep; if `signal` aborts *during* the sleep, wake early. */
+  wait(signal?: AbortSignal): Promise<void> {
+    const ms = this.delay
+    this.delay = Math.min(MAX_DELAY, ms === 0 ? 1 : ms * 2)
+    return new Promise((resolve) => {
+      if (signal?.aborted) signal = undefined
+      const done = (): void => {
+        clearTimeout(t)
+        signal?.removeEventListener('abort', done)
+        resolve()
+      }
+      const t = ms === 0 ? setTimeout(done, 0) : setTimeout(done, ms)
+      signal?.addEventListener('abort', done, { once: true })
+    })
+  }
+}
+
+function unlink(p: string): void {
+  try {
+    fs.unlinkSync(p)
+  } catch {
+    // already gone
+  }
+}
+
+function code(e: unknown): string | undefined {
+  return (e as NodeJS.ErrnoException | null)?.code
+}
+
+/** A private temp dir holding the FIFOs of one external command. */
+export class FifoSet {
+  readonly dir: string
+  private readonly feeds: string[] = []
+  private constructor(dir: string) {
+    this.dir = dir
+  }
+
+  /** Create a private dir with `n` FIFOs (`0`, `1`, …), mode 0600, via one `mkfifo` call. */
+  static async create(n: number): Promise<{ set: FifoSet; paths: string[] }> {
+    const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'mash-fifo-'))
+    const set = new FifoSet(dir)
+    const paths = Array.from({ length: n }, (_, i) => path.join(dir, String(i)))
+    try {
+      await new Promise<void>((resolve, reject) => {
+        execFile('mkfifo', ['-m', '600', ...paths], { env: { PATH: `/usr/bin:/bin:${process.env.PATH ?? ''}` } }, (err) => (err ? reject(err) : resolve()))
+      })
+    } catch (e) {
+      set.cleanup()
+      throw e
+    }
+    return { set, paths }
+  }
+
+  /**
+   * Feed the FIFO at `p` from `file` (the child reads it). Resolves at the
+   * file's EOF, when the child stops reading, or once `exited` aborts.
+   */
+  async feed(p: string, file: OpenFile, exited: AbortSignal): Promise<void> {
+    this.feeds.push(p)
+    const C = fs.constants
+    const backoff = new Backoff()
+    let fd = -1
+    while (fd < 0) {
+      if (exited.aborted) return
+      try {
+        fd = fs.openSync(p, C.O_WRONLY | C.O_NONBLOCK)
+      } catch (e) {
+        if (code(e) !== 'ENXIO') return // no reader yet is ENXIO; anything else: give up
+        await backoff.wait(exited)
+      }
+    }
+    backoff.reset()
+    try {
+      for (;;) {
+        const chunk = await file.read(exited)
+        if (chunk === null) return // EOF (or the child exited: nothing consumed)
+        let off = 0
+        while (off < chunk.length) {
+          if (exited.aborted) {
+            file.unread(chunk.subarray(off))
+            return
+          }
+          try {
+            off += fs.writeSync(fd, chunk, off, chunk.length - off)
+            backoff.reset()
+          } catch (e) {
+            if (code(e) === 'EAGAIN') {
+              await backoff.wait(exited)
+              continue
+            }
+            // EPIPE: the child closed it; leave the rest for other readers of a shared file
+            file.unread(chunk.subarray(off))
+            return
+          }
+        }
+      }
+    } catch {
+      // reading the source failed: the child gets EOF
+    } finally {
+      fs.closeSync(fd)
+      // a second open for reading would wait for a writer forever: make it fail instead
+      unlink(p)
+    }
+  }
+
+  /**
+   * Drain the FIFO at `p` into `file` (the child writes it). Resolves once
+   * the child has exited and no writer holds the FIFO, or when `file` has no
+   * readers left (the child's writes then fail with EPIPE/SIGPIPE).
+   */
+  async drain(p: string, file: OpenFile, exited: AbortSignal): Promise<void> {
+    const C = fs.constants
+    let fd: number
+    try {
+      fd = fs.openSync(p, C.O_RDONLY | C.O_NONBLOCK)
+    } catch {
+      return
+    }
+    const buf = Buffer.allocUnsafe(65536)
+    const backoff = new Backoff()
+    try {
+      for (;;) {
+        let n: number
+        try {
+          n = fs.readSync(fd, buf, 0, buf.length, null)
+        } catch (e) {
+          if (code(e) !== 'EAGAIN') return
+          await backoff.wait() // a writer is connected but has nothing yet
+          continue
+        }
+        if (n > 0) {
+          backoff.reset()
+          await file.write(Buffer.from(buf.subarray(0, n)))
+          continue
+        }
+        // n === 0: no writer at the moment (not opened yet, or closed)
+        if (exited.aborted) return
+        await backoff.wait(exited)
+      }
+    } catch {
+      // BrokenPipeError: nobody reads any more. A later open for writing must fail, not block
+      unlink(p)
+    } finally {
+      fs.closeSync(fd)
+    }
+  }
+
+  /**
+   * Remove the FIFOs and the dir. A reader of a fed FIFO still blocked in
+   * `open` (a grandchild that opened it late) is released first: a brief
+   * non-blocking writer open lets its open return, and it reads EOF.
+   */
+  cleanup(): void {
+    const C = fs.constants
+    for (const p of this.feeds) {
+      if (!fs.existsSync(p)) continue
+      try {
+        fs.closeSync(fs.openSync(p, C.O_WRONLY | C.O_NONBLOCK))
+      } catch {
+        // ENXIO: no reader waiting
+      }
+    }
+    try {
+      fs.rmSync(this.dir, { recursive: true, force: true })
+    } catch {
+      // best effort
+    }
+  }
+}