@taliesinb/mash 0.0.0-stage → 0.1.0

package/src/vm/io.ts

@@ -0,0 +1,570 @@
+/**
+ * The VM's I/O model, modelled on POSIX file descriptions.
+ *
+ * An {@link OpenFile} is an open file description: a pipe end, a real file,
+ * a capture buffer (command substitution), /dev/null, or a host stream. It is
+ * reference-counted: an {@link FdTable} (one per shell "process") holds
+ * references, subshells and pipeline stages clone the table (adding
+ * references), and closing the last write reference of a pipe delivers EOF to
+ * its reader — exactly the behaviour scripts rely on.
+ *
+ * Every OpenFile belongs to a {@link Channel}, the identity trace consumers
+ * see. Data events are reported once per byte: on write for output channels
+ * (pipes, files, captures, stdout/stderr) and on read for input-only channels
+ * (input files, here-docs).
+ */
+
+import * as fs from 'node:fs'
+
+export type ChannelKind = 'pipe' | 'file' | 'capture' | 'null' | 'host' | 'heredoc' | 'procsubst'
+
+export interface Channel {
+  /** Unique id within one machine. */
+  id: number
+  kind: ChannelKind
+  /** Human-readable label, e.g. `pipe(grep foo → wc -l)` or `> out.txt`. */
+  label: string
+  /** File path for file channels. */
+  path?: string
+  /** AST node that created the channel (pipeline, redirect, substitution). */
+  nodeId?: number
+}
+
+export type DataListener = (channel: Channel, chunk: Uint8Array, direction: 'write' | 'read') => void
+
+/** Thrown by writes to a pipe whose read end is fully closed (EPIPE / SIGPIPE). */
+export class BrokenPipeError extends Error {
+  constructor() {
+    super('Broken pipe')
+    this.name = 'BrokenPipeError'
+  }
+}
+
+const encoder = new TextEncoder()
+
+export function toBytes(data: string | Uint8Array): Uint8Array {
+  return typeof data === 'string' ? encoder.encode(data) : data
+}
+
+function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
+  if (a.length === 0) return b
+  if (b.length === 0) return a
+  const out = new Uint8Array(a.length + b.length)
+  out.set(a, 0)
+  out.set(b, a.length)
+  return out
+}
+
+/**
+ * An open file description. Subclasses implement the raw operations; this
+ * base class adds reference counting, a shared read-ahead buffer (so `read`
+ * can consume one line and leave the rest for the next reader) and tracing.
+ */
+export abstract class OpenFile {
+  refs = 1
+  readonly channel: Channel
+  readonly readable: boolean
+  readonly writable: boolean
+  protected listeners: DataListener[]
+  private pushback: Uint8Array = new Uint8Array(0)
+  private closedForGood = false
+  /** Report reads to listeners (input-only channels). */
+  protected traceReads = false
+
+  constructor(channel: Channel, readable: boolean, writable: boolean, listeners: DataListener[]) {
+    this.channel = channel
+    this.readable = readable
+    this.writable = writable
+    this.listeners = listeners
+  }
+
+  /** Add a reference (fd duplication / table clone). */
+  retain(): this {
+    this.refs++
+    return this
+  }
+
+  /** Drop a reference; the last one closes the description. */
+  release(): void {
+    if (this.refs <= 0) return
+    this.refs--
+    if (this.refs === 0 && !this.closedForGood) {
+      this.closedForGood = true
+      this.onLastClose()
+    }
+  }
+
+  get closed(): boolean {
+    return this.closedForGood
+  }
+
+  protected abstract rawRead(): Promise<Uint8Array | null>
+  protected abstract rawWrite(chunk: Uint8Array): Promise<void>
+  protected abstract onLastClose(): void
+
+  async write(data: string | Uint8Array): Promise<void> {
+    if (!this.writable) throw new Error(`${this.channel.label}: not open for writing`)
+    const bytes = toBytes(data)
+    if (bytes.length === 0) return
+    await this.rawWrite(bytes)
+    for (const l of this.listeners) l(this.channel, bytes, 'write')
+  }
+
+  private inflight: Promise<void> | null = null
+  private sawEof = false
+  private readError: unknown = null
+
+  /** Bytes already read from the underlying source but not yet consumed. */
+  get buffered(): number {
+    return this.pushback.length
+  }
+
+  /**
+   * Next chunk, or null at EOF. With `signal`, returns null early when the
+   * signal aborts — without consuming anything: data that arrives later stays
+   * buffered for the next reader. (This is what lets the VM stop feeding a
+   * child process that has exited without losing the rest of a shared stdin.)
+   */
+  async read(signal?: AbortSignal): Promise<Uint8Array | null> {
+    if (!this.readable) throw new Error(`${this.channel.label}: not open for reading`)
+    for (;;) {
+      if (this.pushback.length > 0) {
+        const b = this.pushback
+        this.pushback = new Uint8Array(0)
+        return b
+      }
+      if (this.readError !== null) {
+        const e = this.readError
+        this.readError = null
+        throw e
+      }
+      if (this.sawEof) return null
+      if (signal?.aborted) return null
+      if (!this.inflight) {
+        this.inflight = this.rawRead().then(
+          (chunk) => {
+            this.inflight = null
+            if (chunk === null) this.sawEof = true
+            else {
+              this.pushback = concat(this.pushback, chunk)
+              if (this.traceReads) for (const l of this.listeners) l(this.channel, chunk, 'read')
+            }
+          },
+          (err: unknown) => {
+            this.inflight = null
+            this.readError = err
+          },
+        )
+      }
+      if (signal) {
+        await new Promise<void>((resolve) => {
+          const onAbort = (): void => resolve()
+          signal.addEventListener('abort', onAbort, { once: true })
+          this.inflight!.then(() => {
+            signal.removeEventListener('abort', onAbort)
+            resolve()
+          })
+        })
+      } else {
+        await this.inflight
+      }
+    }
+  }
+
+  /** Return unconsumed bytes to the front of the stream. */
+  unread(bytes: Uint8Array): void {
+    if (bytes.length > 0) this.pushback = concat(bytes, this.pushback)
+  }
+
+  /**
+   * Read up to and including `delim` (a byte), or to EOF. Returns null at EOF
+   * with nothing read. Leaves the remainder buffered for the next reader.
+   */
+  async readUntil(delim: number | null, maxBytes = Infinity): Promise<Uint8Array | null> {
+    let acc: Uint8Array = new Uint8Array(0)
+    for (;;) {
+      const chunk = await this.read()
+      if (chunk === null) return acc.length > 0 ? acc : null
+      const idx = delim === null ? -1 : chunk.indexOf(delim)
+      if (idx >= 0 && acc.length + idx + 1 <= maxBytes) {
+        acc = concat(acc, chunk.subarray(0, idx + 1))
+        this.unread(chunk.subarray(idx + 1))
+        return acc
+      }
+      if (acc.length + chunk.length >= maxBytes) {
+        const take = maxBytes - acc.length
+        acc = concat(acc, chunk.subarray(0, take))
+        this.unread(chunk.subarray(take))
+        return acc
+      }
+      acc = concat(acc, chunk)
+    }
+  }
+
+  /** Read everything to EOF. */
+  async readAll(): Promise<Uint8Array> {
+    let acc: Uint8Array = new Uint8Array(0)
+    for (;;) {
+      const chunk = await this.read()
+      if (chunk === null) return acc
+      acc = concat(acc, chunk)
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Pipes
+// ---------------------------------------------------------------------------
+
+/** An in-memory pipe with bounded buffering (backpressure) and POSIX EOF/EPIPE semantics. */
+export class Pipe {
+  readonly channel: Channel
+  private chunks: Uint8Array[] = []
+  private buffered = 0
+  private readonly capacity: number
+  writersOpen = true
+  readersOpen = true
+  private readWaiters: (() => void)[] = []
+  private writeWaiters: (() => void)[] = []
+  readonly readEnd: PipeEnd
+  readonly writeEnd: PipeEnd
+
+  constructor(channel: Channel, listeners: DataListener[], capacity = 64 * 1024) {
+    this.channel = channel
+    this.capacity = capacity
+    this.readEnd = new PipeEnd(this, 'read', listeners)
+    this.writeEnd = new PipeEnd(this, 'write', listeners)
+  }
+
+  private wakeReaders(): void {
+    const w = this.readWaiters
+    this.readWaiters = []
+    for (const f of w) f()
+  }
+
+  private wakeWriters(): void {
+    const w = this.writeWaiters
+    this.writeWaiters = []
+    for (const f of w) f()
+  }
+
+  async write(chunk: Uint8Array): Promise<void> {
+    while (this.readersOpen && this.buffered >= this.capacity) {
+      await new Promise<void>((r) => this.writeWaiters.push(r))
+    }
+    if (!this.readersOpen) throw new BrokenPipeError()
+    this.chunks.push(chunk)
+    this.buffered += chunk.length
+    this.wakeReaders()
+  }
+
+  async read(): Promise<Uint8Array | null> {
+    while (this.chunks.length === 0 && this.writersOpen) {
+      await new Promise<void>((r) => this.readWaiters.push(r))
+    }
+    const chunk = this.chunks.shift()
+    if (chunk === undefined) return null
+    this.buffered -= chunk.length
+    this.wakeWriters()
+    return chunk
+  }
+
+  closeWriters(): void {
+    this.writersOpen = false
+    this.wakeReaders()
+  }
+
+  closeReaders(): void {
+    this.readersOpen = false
+    this.chunks = []
+    this.buffered = 0
+    this.wakeWriters()
+  }
+}
+
+export class PipeEnd extends OpenFile {
+  private readonly pipe: Pipe
+
+  constructor(pipe: Pipe, end: 'read' | 'write', listeners: DataListener[]) {
+    super(pipe.channel, end === 'read', end === 'write', listeners)
+    this.pipe = pipe
+  }
+
+  protected rawRead(): Promise<Uint8Array | null> {
+    return this.pipe.read()
+  }
+
+  protected rawWrite(chunk: Uint8Array): Promise<void> {
+    return this.pipe.write(chunk)
+  }
+
+  protected onLastClose(): void {
+    if (this.writable) this.pipe.closeWriters()
+    else this.pipe.closeReaders()
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Files
+// ---------------------------------------------------------------------------
+
+export type FileMode = 'read' | 'write' | 'append' | 'readwrite'
+
+/** A real file opened on the host. */
+export class HostFile extends OpenFile {
+  readonly fd: number
+  private eof = false
+
+  private constructor(channel: Channel, fd: number, mode: FileMode, listeners: DataListener[]) {
+    super(channel, mode === 'read' || mode === 'readwrite', mode !== 'read', listeners)
+    this.fd = fd
+    this.traceReads = mode === 'read'
+  }
+
+  static async open(channel: Channel, path: string, mode: FileMode, listeners: DataListener[], noclobber = false, umask?: number): Promise<HostFile> {
+    const existed = mode !== 'read' && umask !== undefined && fs.existsSync(path)
+    const flags = mode === 'read' ? 'r' : mode === 'write' ? (noclobber ? 'wx' : 'w') : mode === 'append' ? 'a' : 'r+'
+    let fd: number
+    try {
+      fd = await new Promise<number>((resolve, reject) => fs.open(path, mode === 'readwrite' && !fs.existsSync(path) ? 'w+' : flags, 0o666, (err, f) => (err ? reject(err) : resolve(f))))
+    } catch (e) {
+      throw describeFsError(e, path)
+    }
+    if (mode !== 'read' && umask !== undefined && !existed) {
+      try {
+        fs.fchmodSync(fd, 0o666 & ~umask)
+      } catch {
+        // ignore
+      }
+    }
+    return new HostFile(channel, fd, mode, listeners)
+  }
+
+  protected rawRead(): Promise<Uint8Array | null> {
+    if (this.eof) return Promise.resolve(null)
+    return new Promise((resolve, reject) => {
+      const buf = Buffer.allocUnsafe(64 * 1024)
+      fs.read(this.fd, buf, 0, buf.length, null, (err, n) => {
+        if (err) return reject(err)
+        if (n === 0) {
+          this.eof = true
+          return resolve(null)
+        }
+        resolve(new Uint8Array(buf.buffer, buf.byteOffset, n).slice())
+      })
+    })
+  }
+
+  protected rawWrite(chunk: Uint8Array): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.write(this.fd, chunk, 0, chunk.length, null, (err) => (err ? reject(err) : resolve()))
+    })
+  }
+
+  protected onLastClose(): void {
+    fs.close(this.fd, () => {})
+  }
+}
+
+export class FsError extends Error {
+  readonly code: string
+  constructor(message: string, code: string) {
+    super(message)
+    this.code = code
+  }
+}
+
+export function describeFsError(e: unknown, path: string): FsError {
+  const code = (e as NodeJS.ErrnoException).code ?? 'EIO'
+  const text: Record<string, string> = {
+    ENOENT: 'No such file or directory',
+    EACCES: 'Permission denied',
+    EPERM: 'Operation not permitted',
+    EISDIR: 'Is a directory',
+    ENOTDIR: 'Not a directory',
+    EEXIST: 'cannot overwrite existing file',
+    EROFS: 'Read-only file system',
+  }
+  return new FsError(`${path}: ${text[code] ?? code}`, code)
+}
+
+// ---------------------------------------------------------------------------
+// Captures, null, constant input, host streams
+// ---------------------------------------------------------------------------
+
+/** Collects written bytes (command substitution, `run()` output capture). */
+export class Capture extends OpenFile {
+  private parts: Uint8Array[] = []
+  size = 0
+  /** Bytes beyond this are discarded (but counted). */
+  readonly limit: number
+  truncated = false
+
+  constructor(channel: Channel, listeners: DataListener[], limit = 64 * 1024 * 1024) {
+    super(channel, false, true, listeners)
+    this.limit = limit
+  }
+
+  protected rawRead(): Promise<Uint8Array | null> {
+    return Promise.resolve(null)
+  }
+
+  protected async rawWrite(chunk: Uint8Array): Promise<void> {
+    if (this.size + chunk.length > this.limit) {
+      this.truncated = true
+      const room = Math.max(0, this.limit - this.size)
+      if (room > 0) this.parts.push(chunk.slice(0, room))
+      this.size += chunk.length
+      return
+    }
+    this.parts.push(chunk)
+    this.size += chunk.length
+  }
+
+  protected onLastClose(): void {}
+
+  bytes(): Uint8Array {
+    let n = 0
+    for (const p of this.parts) n += p.length
+    const out = new Uint8Array(n)
+    let o = 0
+    for (const p of this.parts) {
+      out.set(p, o)
+      o += p.length
+    }
+    return out
+  }
+
+  text(): string {
+    return new TextDecoder().decode(this.bytes())
+  }
+}
+
+export class NullFile extends OpenFile {
+  constructor(channel: Channel, listeners: DataListener[]) {
+    super(channel, true, true, listeners)
+  }
+
+  protected rawRead(): Promise<Uint8Array | null> {
+    return Promise.resolve(null)
+  }
+
+  protected rawWrite(): Promise<void> {
+    return Promise.resolve()
+  }
+
+  protected onLastClose(): void {}
+}
+
+/** Fixed input (here-docs, here-strings). */
+export class ConstInput extends OpenFile {
+  private data: Uint8Array | null
+
+  constructor(channel: Channel, data: string | Uint8Array, listeners: DataListener[]) {
+    super(channel, true, false, listeners)
+    this.data = toBytes(data)
+    this.traceReads = true
+  }
+
+  protected rawRead(): Promise<Uint8Array | null> {
+    const d = this.data
+    this.data = null
+    return Promise.resolve(d && d.length > 0 ? d : null)
+  }
+
+  protected rawWrite(): Promise<void> {
+    return Promise.reject(new Error('not writable'))
+  }
+
+  protected onLastClose(): void {}
+}
+
+/** Adapts a host Node stream (or callback) as the VM's stdin/stdout/stderr. */
+export class HostStream extends OpenFile {
+  private readonly onWrite?: (chunk: Uint8Array) => void | Promise<void>
+  private readonly iterator?: AsyncIterator<Uint8Array | string>
+
+  constructor(
+    channel: Channel,
+    listeners: DataListener[],
+    opts: { write?: (chunk: Uint8Array) => void | Promise<void>; input?: AsyncIterable<Uint8Array | string> },
+  ) {
+    super(channel, opts.input !== undefined, opts.write !== undefined, listeners)
+    this.onWrite = opts.write
+    this.iterator = opts.input?.[Symbol.asyncIterator]()
+    this.traceReads = opts.input !== undefined
+  }
+
+  protected async rawRead(): Promise<Uint8Array | null> {
+    if (!this.iterator) return null
+    const r = await this.iterator.next()
+    if (r.done) return null
+    return toBytes(r.value)
+  }
+
+  protected async rawWrite(chunk: Uint8Array): Promise<void> {
+    await this.onWrite?.(chunk)
+  }
+
+  protected onLastClose(): void {}
+}
+
+// ---------------------------------------------------------------------------
+// Fd tables
+// ---------------------------------------------------------------------------
+
+/** One shell process's file descriptor table. */
+export class FdTable {
+  private readonly fds = new Map<number, OpenFile>()
+
+  get(fd: number): OpenFile | undefined {
+    return this.fds.get(fd)
+  }
+
+  /** Install `file` at `fd` (taking ownership of one reference), closing what was there. */
+  set(fd: number, file: OpenFile): void {
+    const old = this.fds.get(fd)
+    this.fds.set(fd, file)
+    old?.release()
+  }
+
+  close(fd: number): void {
+    const old = this.fds.get(fd)
+    if (old) {
+      this.fds.delete(fd)
+      old.release()
+    }
+  }
+
+  /** `n>&m` */
+  dup(from: number, to: number): boolean {
+    const f = this.fds.get(from)
+    if (!f) return false
+    if (from === to) return true
+    this.set(to, f.retain())
+    return true
+  }
+
+  /** A copy sharing every open file (each gains a reference). */
+  clone(): FdTable {
+    const t = new FdTable()
+    for (const [fd, f] of this.fds) t.fds.set(fd, f.retain())
+    return t
+  }
+
+  closeAll(): void {
+    for (const f of this.fds.values()) f.release()
+    this.fds.clear()
+  }
+
+  entries(): [number, OpenFile][] {
+    return [...this.fds.entries()]
+  }
+
+  /** Lowest free fd ≥ `min` (for `{var}>file`, bash starts at 10). */
+  freeFd(min = 10): number {
+    let n = min
+    while (this.fds.has(n)) n++
+    return n
+  }
+}