@smthrs/std 1.0.0-rc.7 → 1.0.0-rc.9

package/src/internal/Exec.ts

@@ -30,7 +30,7 @@ import { elideMiddle } from "./Text.ts"
  * @category models
  * @since 1.0.0
  */
-export const ExecErrorCode = Schema.Literals(["timeout", "spawn_error", "capture_overflow"])
+export const ExecErrorCode = Schema.Literals(["timeout", "spawn_error", "arguments_too_long", "capture_overflow"])
 
 /**
  * Failure codes a buffered execution can report.
@@ -44,8 +44,10 @@ export type ExecErrorCode = typeof ExecErrorCode.Type
  * A command that never produced an exit code.
  *
  * A non-zero exit is a successful {@link ExecResult}, not a failure; this
- * error is reserved for a command that could not start or was cut off by its
- * timeout, or whose captured output exceeded a refusing capture bound.
+ * error is reserved for a command that could not start (`arguments_too_long`
+ * when the operating system refused it for the size of its arguments) or was
+ * cut off by its timeout, or whose captured output exceeded a refusing capture
+ * bound.
  *
  * @category errors
  * @since 1.0.0
@@ -269,6 +271,23 @@ const capture = <E>(
     ? boundedEnds(stream, maxBytes)
     : boundedTail(stream, maxBytes, overflow)
 
+/**
+ * A command whose arguments exceed the operating system's limit fails `execve`
+ * with `E2BIG` before it starts: one argument over 128 KiB on Linux, all of
+ * them over 1 MiB on macOS. A heredoc in a command line, or a file's text
+ * passed as an argument, is one such argument. The platform error's own text
+ * repeats the whole command line, so this text is written here instead, and a
+ * caller may show it as it stands.
+ */
+const argumentsTooLong = (error: { readonly cause?: unknown }): ExecError | undefined =>
+  (error.cause as { readonly code?: unknown } | undefined)?.code === "E2BIG"
+    ? new ExecError({
+      code: "arguments_too_long",
+      message:
+        "Command not started: its arguments exceed the operating system's size limit (E2BIG). Send long text on standard input or in a file, not in the command line or its arguments."
+    })
+    : undefined
+
 const unbounded = (
   command: string,
   options: ExecOptions
@@ -327,7 +346,7 @@ const unbounded = (
   }).pipe(
     Effect.scoped,
     Effect.mapError((error) =>
-      error instanceof ExecError ? error : new ExecError({
+      error instanceof ExecError ? error : argumentsTooLong(error) ?? new ExecError({
         code: "spawn_error",
         message: `exec: ${command}: ${error.message}`
       })
@@ -368,20 +387,23 @@ export const exec = (
  *
  * A timeout keeps its own code, because "the command never finished" and "the
  * command never started" are different things to a caller deciding whether to
- * retry. Every other way of producing no exit code is a host failure.
+ * retry. Arguments the operating system refused for their size are the
+ * caller's own input, which a shell flow publishes, so they read as
+ * `invalid_input`. Every other way of producing no exit code is a host failure.
  *
  * @category errors
  * @since 1.0.0
  */
 export const toStdErrorCode = (error: ExecError): StdError.Code =>
-  error.code === "timeout" ? "timeout" : "command_failed"
+  error.code === "timeout" ? "timeout" : error.code === "arguments_too_long" ? "invalid_input" : "command_failed"
 
 /**
  * The model-facing failure one execution error becomes.
  *
  * A timeout names the command, which is what the caller asked for and all a
- * timeout has to say. Any other failure carries the message the execution
- * produced, which already names the command.
+ * timeout has to say. Arguments too long to start carry the reason and the
+ * fix, and nothing of the command. Any other failure carries the message the
+ * execution produced, which already names the command.
  *
  * @category errors
  * @since 1.0.0
@@ -391,6 +413,8 @@ export const toStdError = (command: string, error: ExecError): StdError.StdError
     code: toStdErrorCode(error),
     message: error.code === "timeout"
       ? `Command timed out: ${command}`
+      : error.code === "arguments_too_long"
+      ? error.message
       : `Command failed to start: ${error.message}`,
     ...(error.limitMillis === undefined ? {} : { limitMillis: error.limitMillis })
   })

package/src/internal/RemoteTree.ts

@@ -0,0 +1,159 @@
+/**
+ * A tree another machine listed, served as a read-only `FileSystem`.
+ *
+ * `RemoteSearch` lists a root on the machine with `find` and brings back the
+ * bytes the search reads with `tar`; this module holds what came back and
+ * answers what the walk, the globs, and the matcher ask of a filesystem, so
+ * `PortableSearch` can search the copy as it searches a host.
+ *
+ * @since 1.0.0-rc.8
+ */
+
+import type * as Path from "@smthrs/kernel/Path"
+import { Effect, Option, Stream } from "effect"
+import * as ByteSize from "effect/ByteSize"
+import * as FileSystem from "effect/FileSystem"
+import * as PlatformError from "effect/PlatformError"
+import * as Tar from "./Tar.ts"
+
+/**
+ * What one listed path is: `d` a directory, `u` a directory the machine's
+ * user may not list, `f` a file, `o` anything else, and a symbolic link to a
+ * directory (`D`), to a file (`F`), or to neither (`L`).
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export type Kind = "d" | "u" | "f" | "o" | "D" | "F" | "L"
+
+/**
+ * One machine's listing of a root, and the bytes it brought back, by
+ * absolute path.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export interface Tree {
+  readonly kinds: ReadonlyMap<string, Kind>
+  readonly children: ReadonlyMap<string, ReadonlyArray<string>>
+  readonly contents: ReadonlyMap<string, Uint8Array>
+}
+
+const kinds: ReadonlySet<string> = new Set(["d", "u", "f", "o", "D", "F", "L"])
+
+/**
+ * The member names `tar` wrote for absolute paths, which it strips of their
+ * leading `/`, as those paths again.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const absolute = (members: ReadonlyMap<string, Uint8Array>): ReadonlyMap<string, Uint8Array> =>
+  new Map([...members].map(([name, bytes]) => [`/${name}`, bytes]))
+
+/**
+ * Reads NUL-terminated `<kind><path>` records up to an `E` record, then the
+ * tar stream of `.gitignore` files after it. `root` is the listed root, which
+ * is nobody's child. `undefined` when the bytes are not that shape.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const parse = (bytes: Uint8Array, root: string, path: Path.Path): Tree | undefined => {
+  const decoder = new TextDecoder()
+  const listed = new Map<string, Kind>()
+  const children = new Map<string, Array<string>>()
+  let offset = 0
+  for (;;) {
+    const end = bytes.indexOf(0, offset)
+    if (end === -1) return undefined
+    const record = decoder.decode(bytes.subarray(offset, end))
+    offset = end + 1
+    if (record === "E") break
+    const kind = record.slice(0, 1)
+    if (!kinds.has(kind)) return undefined
+    const entry = record.slice(1)
+    listed.set(entry, kind as Kind)
+    if (entry === root) continue
+    const parent = path.dirname(entry)
+    const siblings = children.get(parent)
+    if (siblings === undefined) children.set(parent, [path.basename(entry)])
+    else siblings.push(path.basename(entry))
+  }
+  const ignores = Tar.files(bytes.subarray(offset))
+  return ignores === undefined ? undefined : { kinds: listed, children, contents: absolute(ignores) }
+}
+
+const refusal = (tag: PlatformError.SystemErrorTag, method: string, path: string): PlatformError.PlatformError =>
+  PlatformError.systemError({ _tag: tag, module: "FileSystem", method, pathOrDescriptor: path })
+
+/** What `stat` says each kind is: links resolved, a dangling one absent. */
+const types: Readonly<Record<Kind, FileSystem.File.Type | undefined>> = {
+  d: "Directory",
+  u: "Directory",
+  D: "Directory",
+  f: "File",
+  F: "File",
+  o: "Unknown",
+  L: undefined
+}
+
+const info = (type: FileSystem.File.Type, size: number): FileSystem.File.Info => ({
+  type,
+  size: ByteSize.bytes(BigInt(size)),
+  mtime: Option.none(),
+  atime: Option.none(),
+  birthtime: Option.none(),
+  dev: 0,
+  ino: Option.none(),
+  mode: 0,
+  nlink: Option.none(),
+  uid: Option.none(),
+  gid: Option.none(),
+  rdev: Option.none(),
+  blksize: Option.none(),
+  blocks: Option.none()
+})
+
+/**
+ * The tree as a read-only `FileSystem`, resolving paths with `path`. A file
+ * whose bytes did not come back is one the machine's user could not read,
+ * which the in-process peer skips exactly as it skips such a file on a host.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const fileSystem = (tree: Tree, path: Path.Path): FileSystem.FileSystem => {
+  const kindOf = (value: string) => tree.kinds.get(path.resolve(value))
+  const readFile = (value: string) => {
+    const bytes = tree.contents.get(path.resolve(value))
+    if (bytes !== undefined) return Effect.succeed(bytes)
+    const type = types[kindOf(value) ?? "L"]
+    return Effect.fail(
+      refusal(type === "File" ? "PermissionDenied" : type === undefined ? "NotFound" : "BadResource", "readFile", value)
+    )
+  }
+  return FileSystem.makeNoop({
+    stat: (value) => {
+      const type = types[kindOf(value) ?? "L"]
+      return type === undefined
+        ? Effect.fail(refusal("NotFound", "stat", value))
+        : Effect.succeed(info(type, tree.contents.get(path.resolve(value))?.length ?? 0))
+    },
+    readDirectory: (value) => {
+      const kind = kindOf(value)
+      if (kind === "d") return Effect.succeed([...tree.children.get(path.resolve(value)) ?? []])
+      const tag = kind === "u" ? "PermissionDenied" : kind === undefined || kind === "L" ? "NotFound" : "BadResource"
+      return Effect.fail(refusal(tag, "readDirectory", value))
+    },
+    readLink: (value) => {
+      const kind = kindOf(value)
+      return kind === "D" || kind === "F" || kind === "L"
+        ? Effect.succeed("")
+        : Effect.fail(refusal(kind === undefined ? "NotFound" : "BadResource", "readLink", value))
+    },
+    readFile,
+    readFileString: (value) => Effect.map(readFile(value), (bytes) => new TextDecoder().decode(bytes)),
+    stream: (value) => Stream.fromEffect(readFile(value))
+  })
+}

package/src/internal/Tar.ts

@@ -0,0 +1,95 @@
+/**
+ * The regular files of a tar stream.
+ *
+ * `RemoteSearch` brings a machine's files back with its own `tar`, because
+ * every image ships one and it frames each file with its name and size. GNU
+ * and BusyBox `tar` write GNU long names, `bsdtar` writes POSIX extended
+ * headers, and `xargs` may start `tar` more than once, so the stream can hold
+ * several archives one after another. This reads all of them.
+ *
+ * @since 1.0.0-rc.8
+ */
+
+const decoder = new TextDecoder()
+
+/** Bytes up to the first NUL, as text. */
+const text = (bytes: Uint8Array): string => {
+  const end = bytes.indexOf(0)
+  return decoder.decode(end === -1 ? bytes : bytes.subarray(0, end))
+}
+
+/** An octal header field, or `undefined` when it holds anything else. */
+const octal = (bytes: Uint8Array): number | undefined => {
+  const digits = text(bytes).trim()
+  return /^[0-7]+$/.test(digits) ? Number.parseInt(digits, 8) : undefined
+}
+
+/** The header's checksum, computed with the checksum field read as spaces. */
+const checksum = (header: Uint8Array): number => {
+  let sum = 0
+  for (let index = 0; index < 512; index++) sum += index >= 148 && index < 156 ? 32 : header[index]!
+  return sum
+}
+
+/** The `path` record of a POSIX extended header: `<length> <key>=<value>\n` records. */
+const paxPath = (data: Uint8Array): string | undefined => {
+  let offset = 0
+  let path: string | undefined
+  while (offset < data.length) {
+    const space = data.indexOf(32, offset)
+    const length = space === -1 ? Number.NaN : Number(decoder.decode(data.subarray(offset, space)))
+    if (!Number.isSafeInteger(length) || length <= space - offset || offset + length > data.length) return path
+    const record = decoder.decode(data.subarray(space + 1, offset + length - 1))
+    if (record.startsWith("path=")) path = record.slice("path=".length)
+    offset += length
+  }
+  return path
+}
+
+/**
+ * Every regular file the stream holds, by member name, or `undefined` when
+ * the stream is not whole tar archives: a header whose checksum does not
+ * hold, a member cut short, or a last archive with no end. A `tar` that
+ * stopped between two members wrote no end, so its missing files are not
+ * mistaken for files it could not read.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const files = (stream: Uint8Array): ReadonlyMap<string, Uint8Array> | undefined => {
+  const found = new Map<string, Uint8Array>()
+  let longName: string | undefined
+  let extended: string | undefined
+  let ended = true
+  for (let offset = 0; offset + 512 <= stream.length;) {
+    const header = stream.subarray(offset, offset + 512)
+    offset += 512
+    // Zero blocks end one archive; `xargs` may have started another.
+    ended = header.every((byte) => byte === 0)
+    if (ended) continue
+    const size = octal(header.subarray(124, 136))
+    if (size === undefined || octal(header.subarray(148, 156)) !== checksum(header)) return undefined
+    if (offset + size > stream.length) return undefined
+    const data = stream.subarray(offset, offset + size)
+    offset += Math.ceil(size / 512) * 512
+    const type = String.fromCharCode(header[156]!)
+    if (type === "L") {
+      longName = text(data)
+      continue
+    }
+    if (type === "x") {
+      extended = paxPath(data) ?? extended
+      continue
+    }
+    // A long link name and a global header describe no member of their own.
+    if (type === "K" || type === "g") continue
+    if (type === "0" || type === "\0" || type === "7") {
+      const prefix = decoder.decode(header.subarray(257, 263)) === "ustar\0" ? text(header.subarray(345, 500)) : ""
+      const short = text(header.subarray(0, 100))
+      found.set(extended ?? longName ?? (prefix === "" ? short : `${prefix}/${short}`), data.slice())
+    }
+    longName = undefined
+    extended = undefined
+  }
+  return ended ? found : undefined
+}