@nimbus-sh/core 0.5.0 → 0.7.0

package/src/substrate/lifo/commands/types.ts

@@ -4,9 +4,8 @@ import type { VfsCred } from '../../../runtime/os-contracts.js';
 export interface CommandOutputStream {
   write(text: string): void;
   /**
-   * Present on sinks that store bytes verbatim (files, `/dev/null`). Textual
-   * sinks — the terminal, shell pipes — omit it and take decoded text, since
-   * they have no way to carry a byte that is not text.
+   * Present on sinks that store bytes verbatim — files, `/dev/null`, and
+   * byte-capable shell pipes. Sinks without it take decoded text instead.
    */
   writeBytes?(bytes: Uint8Array): void;
   /**
@@ -22,9 +21,13 @@ export interface CommandInputStream {
   read(): Promise<string | null>;   // null = EOF
   readAll(): Promise<string>;
   readLine?(): Promise<string | null>;
-  readBytes?(maxLength: number): Promise<string | null>;
+  /**
+   * Bounded raw-byte read. Byte-capable sources (shell pipes, dumps) return
+   * the original bytes; text-only sources encode what they hold. Null means
+   * EOF with nothing returned.
+   */
+  readBytes?(maxLength: number): Promise<Uint8Array | null>;
 }
-
 export interface TerminalInputStream extends CommandInputStream {
   rawMode: boolean;
 }

package/src/substrate/lifo/node-compat/zlib.ts

@@ -2,6 +2,79 @@ import { Buffer } from './buffer.js';
 
 type ZlibCallback = (err: Error | null, result?: Buffer) => void;
 
+/**
+ * Node's async surface: the callback is required, with options optional in
+ * front of it. There is no promise return to fall back on.
+ */
+interface ZlibAsync {
+  (data: ZlibInput, callback: ZlibCallback): void;
+  (data: ZlibInput, options: object, callback: ZlibCallback): void;
+}
+
+/**
+ * Everything a zlib entry point accepts: strings encode to UTF-8;
+ * ArrayBuffers and every ArrayBufferView carry their own bytes.
+ */
+type ZlibInput = string | ArrayBufferView | ArrayBuffer;
+
+/**
+ * One funnel for every entry point. The result views the caller's exact
+ * byte region — byteOffset and byteLength preserved, element values never
+ * reinterpreted — so sniffing and the codec see identical bytes for a
+ * Uint8Array, a Uint16Array, an offset DataView, or a bare ArrayBuffer.
+ */
+function toBytes(data: ZlibInput): Uint8Array {
+  if (typeof data === 'string') return new TextEncoder().encode(data);
+  if (data instanceof Uint8Array) return data;
+  if (ArrayBuffer.isView(data)) {
+    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+  }
+  return new Uint8Array(data);
+}
+
+/**
+ * Node splits decompression failures two ways, and the engine's own failure
+ * is the only thing that can tell them apart: an input that ran out is
+ * Z_BUF_ERROR / errno -5, while everything else — bad header, bad block, and
+ * a checksum or length trailer that can only fail once the member is
+ * complete — is Z_DATA_ERROR / errno -3 carrying the engine's message. Which
+ * side of a write or a close the failure lands on says nothing about that,
+ * so it is not consulted.
+ */
+function isUnexpectedEnd(cause: unknown): boolean {
+  const message = cause instanceof Error && typeof cause.message === 'string' ? cause.message : String(cause);
+  return /unexpected end|end of (?:file|input|stream)|premature|truncated|\bEOF\b/i.test(message);
+}
+
+function zDataError(cause: unknown): Error {
+  const message = cause instanceof Error && cause.message ? cause.message : String(cause);
+  return Object.assign(new Error(message), { code: 'Z_DATA_ERROR', errno: -3, cause });
+}
+
+function zBufError(cause: unknown): Error {
+  return Object.assign(new Error('unexpected end of file'), {
+    code: 'Z_BUF_ERROR',
+    errno: -5,
+    cause,
+  });
+}
+
+/**
+ * Node throws this synchronously rather than deferring a callback that does
+ * not exist.
+ */
+function invalidCallback(value: unknown): TypeError {
+  const received =
+    value === undefined ? 'undefined'
+    : value === null ? 'null'
+    : typeof value === 'object' ? `an instance of ${value.constructor?.name ?? 'Object'}`
+    : `type ${typeof value} (${String(value)})`;
+  return Object.assign(
+    new TypeError(`The "callback" argument must be of type function. Received ${received}`),
+    { code: 'ERR_INVALID_ARG_TYPE' },
+  );
+}
+
 async function processStream(
   data: Uint8Array,
   format: CompressionFormat,
@@ -13,15 +86,35 @@ async function processStream(
       : new DecompressionStream(format);
 
   const writer = stream.writable.getWriter();
-  writer.write(data as unknown as ArrayBuffer);
-  writer.close();
-
   const reader = stream.readable.getReader();
   const chunks: Uint8Array[] = [];
-  for (;;) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    chunks.push(value);
+  // Feeding and draining run together: a single write large enough to fill
+  // the codec's output queue only completes while somebody reads. Both sides
+  // are contained so neither rejection surfaces unhandled, and the first
+  // failure is the one reported.
+  const failures: unknown[] = [];
+  const record = (reason: unknown): void => {
+    if (failures.length === 0) failures.push(reason);
+  };
+  const drain = (async () => {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) return;
+      chunks.push(value);
+    }
+  })().catch(record);
+  const feed = (async () => {
+    await writer.write(data);
+    await writer.close();
+  })().catch(record);
+  await Promise.all([drain, feed]);
+
+  if (failures.length > 0) {
+    const reason = failures[0];
+    if (type === 'decompress') {
+      throw isUnexpectedEnd(reason) ? zBufError(reason) : zDataError(reason);
+    }
+    throw reason instanceof Error ? reason : new Error(String(reason));
   }
 
   let totalLength = 0;
@@ -35,14 +128,28 @@ async function processStream(
   return result;
 }
 
-function wrapAsync(format: CompressionFormat, type: 'compress' | 'decompress') {
-  return function (data: Uint8Array | string, optionsOrCb: unknown, cb?: ZlibCallback): void {
-    const callback = typeof optionsOrCb === 'function' ? (optionsOrCb as ZlibCallback) : cb!;
-    const input = typeof data === 'string' ? new TextEncoder().encode(data) : data;
-    const raw = input instanceof Buffer ? new Uint8Array(input) : input;
-    processStream(raw, format, type)
-      .then((result) => callback(null, result))
-      .catch((err) => callback(err instanceof Error ? err : new Error(String(err))));
+/**
+ * Node's Unzip sniffs the wrapper itself: the gzip magic bytes mean gunzip,
+ * anything else is treated as zlib-wrapped deflate.
+ */
+function looksLikeGzip(data: Uint8Array): boolean {
+  return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
+}
+
+function wrapAsync(
+  format: CompressionFormat | ((data: Uint8Array) => CompressionFormat),
+  type: 'compress' | 'decompress',
+): ZlibAsync {
+  return function (data: ZlibInput, optionsOrCb: object | ZlibCallback, cb?: ZlibCallback): void {
+    const callback = typeof optionsOrCb === 'function' ? optionsOrCb : cb;
+    if (typeof callback !== 'function') throw invalidCallback(callback);
+    const bytes = toBytes(data);
+    // Two arms, not .then().catch(): a callback that throws on success must
+    // not be handed its own exception as a second, failed invocation.
+    processStream(bytes, typeof format === 'function' ? format(bytes) : format, type).then(
+      (result) => callback(null, result),
+      (err) => callback(err instanceof Error ? err : new Error(String(err))),
+    );
   };
 }
 
@@ -52,20 +159,40 @@ export const deflate = wrapAsync('deflate', 'compress');
 export const inflate = wrapAsync('deflate', 'decompress');
 export const deflateRaw = wrapAsync('deflate-raw', 'compress');
 export const inflateRaw = wrapAsync('deflate-raw', 'decompress');
-export const unzip = wrapAsync('gzip', 'decompress');
+export const unzip = wrapAsync((data) => (looksLikeGzip(data) ? 'gzip' : 'deflate'), 'decompress');
+
+/**
+ * The Sync surface needs a synchronous compress primitive; browser-class
+ * runtimes only offer the async CompressionStream. Each Sync name refuses
+ * with the working async replacement spelled out instead of failing as a
+ * missing export or a vague "not supported".
+ */
+function syncUnavailable(name: string, asyncName: string): never {
+  throw Object.assign(new Error(
+    `zlib.${name}: synchronous compression is not available on this runtime. Use the async zlib.${asyncName}(data, callback) form instead.`,
+  ), { code: 'ERR_ZLIB_SYNC_UNAVAILABLE' });
+}
 
-// Sync variants are not supported in the browser
-export function gzipSync(): never {
-  throw new Error('zlib sync operations are not supported in Lifo');
+export function gzipSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('gzipSync', 'gzip');
+}
+export function gunzipSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('gunzipSync', 'gunzip');
+}
+export function deflateSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('deflateSync', 'deflate');
+}
+export function inflateSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('inflateSync', 'inflate');
 }
-export function gunzipSync(): never {
-  throw new Error('zlib sync operations are not supported in Lifo');
+export function deflateRawSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('deflateRawSync', 'deflateRaw');
 }
-export function deflateSync(): never {
-  throw new Error('zlib sync operations are not supported in Lifo');
+export function inflateRawSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('inflateRawSync', 'inflateRaw');
 }
-export function inflateSync(): never {
-  throw new Error('zlib sync operations are not supported in Lifo');
+export function unzipSync(buffer?: ZlibInput, options?: object): never {
+  syncUnavailable('unzipSync', 'unzip');
 }
 
 export const constants = {
@@ -90,5 +217,7 @@ export const constants = {
 
 export default {
   gzip, gunzip, deflate, inflate, deflateRaw, inflateRaw, unzip,
-  gzipSync, gunzipSync, deflateSync, inflateSync, constants,
+  gzipSync, gunzipSync, deflateSync, inflateSync,
+  deflateRawSync, inflateRawSync, unzipSync,
+  constants,
 };

package/src/substrate/lifo/shell/interpreter.ts

@@ -40,6 +40,7 @@ import { exitCodeForAbortSignal } from './signals.js';
 import { resolve } from '../utils/path.js';
 import { encode } from '../utils/encoding.js';
 import { globMatch } from '../utils/glob.js';
+import { staticStdinReader } from '../../../shell/stdin-adapter.js';
 
 /**
  * Bytes a file-backed descriptor holds before committing. Matches the stream
@@ -1391,7 +1392,7 @@ export class Interpreter {
         const body = heredoc.quoted
           ? heredoc.body
           : await expandWord([{ text: heredoc.body, quoted: 'double' }], expandCtx);
-        this.setInputFd(fds, redir.fd ?? 0, { stream: this.createStringReader(body), terminal: false });
+        this.setInputFd(fds, redir.fd ?? 0, { stream: staticStdinReader(body), terminal: false });
         continue;
       }
 
@@ -1533,11 +1534,17 @@ export class Interpreter {
       return { stream: terminalStdin, terminal: true };
     }
     if (target === '/dev/stdin') return this.resolveInputFd('0', fds.inputFds, fds.terminalInputFds);
+    const targetPath = resolve(this.config.getCwd(), target);
+    const vfs = io.vfs ?? this.config.vfs;
     try {
-      return {
-        stream: this.createFileReader(io.vfs ?? this.config.vfs, resolve(this.config.getCwd(), target)),
-        terminal: false,
-      };
+      // Open-authorize before the command runs, the way open(2) would: a
+      // missing, unreadable, or directory target fails the redirection even
+      // when the command never reads a byte.
+      if (vfs.stat(targetPath).type === 'directory') {
+        throw Object.assign(new Error(`EISDIR: ${targetPath}`), { code: 'EISDIR' });
+      }
+      vfs.access(targetPath, 0o4);
+      return { stream: this.createFileReader(vfs, targetPath), terminal: false };
     } catch (error) {
       throw new RedirectionOpenError(target, error);
     }
@@ -1607,37 +1614,77 @@ export class Interpreter {
     for (const stream of fds.outputFds.values()) stream.flush?.();
   }
 
+  /**
+   * Byte-faithful redirected input: bounded range reads keep >64 KiB
+   * redirections intact and preserve bytes that are not valid UTF-8, while
+   * the text view still decodes progressively across chunk boundaries.
+   *
+   * Only a zero-length range means EOF. Devices and some mounts answer with
+   * fewer bytes than asked whenever their internal bound is hit; those short
+   * nonempty reads advance the offset and continue, exactly like read(2).
+   */
   private createFileReader(vfs: VFS, path: string): CommandInputStream {
-    const content = vfs.readFileString(path);
-    return this.createStringReader(content);
-  }
-
-  private createStringReader(content: string): CommandInputStream {
+    const decoder = new TextDecoder('utf-8');
     let offset = 0;
+    let eof = false;
+    const pull = (max: number): Uint8Array | null => {
+      if (eof) return null;
+      const chunk = vfs.readRange(path, offset, max);
+      if (chunk.length === 0) {
+        eof = true;
+        return null;
+      }
+      offset += chunk.length;
+      return chunk;
+    };
     return {
+      readBytes: async (maxLength: number) => {
+        if (maxLength <= 0) return new Uint8Array(0);
+        return pull(maxLength);
+      },
       read: async () => {
-        if (offset >= content.length) return null;
-        const chunk = content.slice(offset);
-        offset = content.length;
-        return chunk;
+        while (true) {
+          const bytes = pull(65536);
+          if (bytes === null) {
+            const tail = decoder.decode();
+            return tail.length > 0 ? tail : null;
+          }
+          const text = decoder.decode(bytes, { stream: true });
+          if (text.length > 0) return text;
+        }
       },
       readAll: async () => {
-        if (offset >= content.length) return '';
-        const chunk = content.slice(offset);
-        offset = content.length;
-        return chunk;
+        let out = '';
+        while (true) {
+          const bytes = pull(65536);
+          if (bytes === null) break;
+          out += decoder.decode(bytes, { stream: true });
+        }
+        out += decoder.decode();
+        return out;
       },
       readLine: async () => {
-        if (offset >= content.length) return null;
-        const newline = content.indexOf('\n', offset);
-        if (newline < 0) {
-          const line = content.slice(offset);
-          offset = content.length;
-          return line;
+        let line = '';
+        let sawAny = false;
+        while (true) {
+          const bytes = pull(65536);
+          if (bytes === null) break;
+          sawAny = true;
+          // Split on the raw 0x0A byte so pushback returns ORIGINAL bytes;
+          // decoded-text pushback would corrupt multibyte sequences that
+          // straddle the chunk boundary or mis-measure invalid UTF-8.
+          const newline = bytes.indexOf(0x0a);
+          if (newline >= 0) {
+            offset -= bytes.length - (newline + 1);
+            eof = false;
+            line += decoder.decode(bytes.subarray(0, newline), { stream: true });
+            const flushed = decoder.decode();
+            return line + flushed;
+          }
+          line += decoder.decode(bytes, { stream: true });
         }
-        const line = content.slice(offset, newline);
-        offset = newline + 1;
-        return line;
+        line += decoder.decode();
+        return sawAny || line.length > 0 ? line : null;
       },
     };
   }

package/src/substrate/lifo/shell/pipe.ts

@@ -1,14 +1,26 @@
 import type { CommandOutputStream, CommandInputStream } from '../commands/types.js';
+import { encode } from '../utils/encoding.js';
 
+/**
+ * A shell pipe that carries the producer's exact bytes. Text writes are
+ * encoded once at the write side, `writeBytes` stores bytes verbatim, and
+ * the text view (`read`/`readAll`/`readLine`) decodes progressively so a
+ * multi-byte UTF-8 sequence split across chunks survives intact.
+ */
 export class PipeChannel {
-  private buffer: string[] = [];
+  private buffer: Uint8Array[] = [];
   private closed = false;
-  private waiting: Array<(value: string | null) => void> = [];
+  private waiting: Array<(value: Uint8Array | null) => void> = [];
+  private decoder = new TextDecoder('utf-8');
 
   readonly writer: CommandOutputStream = {
     write: (text: string) => {
       if (this.closed) return;
-      this.deliver(text, 'back');
+      this.deliver(encode(text), 'back');
+    },
+    writeBytes: (bytes: Uint8Array) => {
+      if (this.closed) return;
+      this.deliver(bytes, 'back');
     },
   };
 
@@ -19,18 +31,31 @@ export class PipeChannel {
     readBytes: (maxLength: number) => this.readBytes(maxLength),
   };
 
-  private read(): Promise<string | null> {
+  /** Next queued chunk, a waiter's delivery, or null once closed and empty. */
+  private pull(): Promise<Uint8Array | null> {
     if (this.buffer.length > 0) {
       return Promise.resolve(this.buffer.shift() ?? null);
     }
     if (this.closed) {
       return Promise.resolve(null);
     }
-    return new Promise<string | null>((resolve) => {
+    return new Promise<Uint8Array | null>((resolve) => {
       this.waiting.push(resolve);
     });
   }
 
+  private async read(): Promise<string | null> {
+    while (true) {
+      const bytes = await this.pull();
+      if (bytes === null) {
+        const tail = this.decoder.decode();
+        return tail.length > 0 ? tail : null;
+      }
+      const text = this.decoder.decode(bytes, { stream: true });
+      if (text.length > 0) return text;
+    }
+  }
+
   private async readAll(): Promise<string> {
     const parts: string[] = [];
     while (true) {
@@ -43,30 +68,45 @@ export class PipeChannel {
 
   private async readLine(): Promise<string | null> {
     let line = '';
+    let sawAny = false;
     while (true) {
-      const chunk = await this.read();
-      if (chunk === null) return line.length > 0 ? line : null;
-
-      const newline = chunk.indexOf('\n');
+      const bytes = await this.pull();
+      if (bytes === null) break;
+      sawAny = true;
+      // Split on the raw 0x0A byte so pushback returns ORIGINAL bytes; a
+      // multibyte sequence straddling the chunk boundary then survives as
+      // é instead of collapsing into a replacement character.
+      const newline = bytes.indexOf(0x0a);
       if (newline >= 0) {
-        const rest = chunk.slice(newline + 1);
+        const rest = bytes.subarray(newline + 1);
         if (rest.length > 0) this.deliver(rest, 'front');
-        return line + chunk.slice(0, newline);
+        line += this.decoder.decode(bytes.subarray(0, newline), { stream: true });
+        const flushed = this.decoder.decode();
+        return line + flushed;
       }
-
-      line += chunk;
+      line += this.decoder.decode(bytes, { stream: true });
     }
+    // A trailing incomplete sequence still surfaces as U+FFFD at EOF.
+    const tail = this.decoder.decode();
+    line += tail;
+    return sawAny || tail.length > 0 ? line : null;
   }
 
-  private async readBytes(maxLength: number): Promise<string | null> {
-    if (maxLength <= 0) return '';
+  /**
+   * Bounded byte read: returns whatever the producer has already delivered,
+   * capped at maxLength. maxLength bounds the result, it is never a fill
+   * target — waiting to complete it would stall every consumer downstream of
+   * a live open producer. A larger chunk keeps only its first maxLength
+   * bytes; the remainder stays queued in original order.
+   */
+  private async readBytes(maxLength: number): Promise<Uint8Array | null> {
+    if (maxLength <= 0) return new Uint8Array(0);
 
-    const chunk = await this.read();
+    const chunk = await this.pull();
     if (chunk === null) return null;
     if (chunk.length <= maxLength) return chunk;
-
-    this.deliver(chunk.slice(maxLength), 'front');
-    return chunk.slice(0, maxLength);
+    this.buffer.unshift(chunk.subarray(maxLength));
+    return chunk.subarray(0, maxLength);
   }
 
   close(): void {
@@ -77,16 +117,17 @@ export class PipeChannel {
     }
   }
 
-  private deliver(text: string, position: 'front' | 'back'): void {
-    if (text.length === 0) return;
+  private deliver(bytes: Uint8Array, position: 'front' | 'back'): void {
+    if (bytes.length === 0) return;
 
     if (this.waiting.length > 0) {
       const resolve = this.waiting.shift()!;
-      resolve(text);
+      resolve(bytes);
       return;
     }
 
-    if (position === 'front') this.buffer.unshift(text);
-    else this.buffer.push(text);
+    if (position === 'front') this.buffer.unshift(bytes);
+    else this.buffer.push(bytes);
   }
+
 }