@smthrs/std 1.0.0-rc.8 → 1.0.0-rc.9

package/docs/troubleshooting.md

@@ -149,6 +149,14 @@ not covered by its own `reads` or `writes`. The message names the resolved path.
 check is lexical and fail-closed: it reads the tokens in the command line, not
 what the process will actually touch.
 
+## invalid_input: a command nested deeper than 32 levels
+
+**What happened.** A hermetic `bash` call nested command substitutions, quotes,
+shell `-c` arguments, or `eval` calls more than 32 levels deep. The pre-check
+refuses text it cannot read in bounded time rather than run it unchecked.
+
+**What to change.** Flatten the command, or use `mode: "unhermetic"`.
+
 ## invalid_input: a hermetic call that names a container
 
 **What happened.** `mode: "hermetic"` together with `container`. A container has

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@smthrs/std",
   "type": "module",
-  "version": "1.0.0-rc.8",
+  "version": "1.0.0-rc.9",
   "license": "MIT",
   "smthrs": {
     "group": "agent"
@@ -252,6 +252,16 @@
         "default": "./dist/cjs/Relocate.js"
       }
     },
+    "./RemoteSearch": {
+      "import": {
+        "types": "./dist/esm/RemoteSearch.d.ts",
+        "default": "./dist/esm/RemoteSearch.js"
+      },
+      "require": {
+        "types": "./dist/cjs/RemoteSearch.d.ts",
+        "default": "./dist/cjs/RemoteSearch.js"
+      }
+    },
     "./Search": {
       "import": {
         "types": "./dist/esm/Search.d.ts",
@@ -401,11 +411,11 @@
     "tag": "next"
   },
   "dependencies": {
-    "@smthrs/capability": "1.0.0-rc.8",
-    "@smthrs/control": "1.0.0-rc.8",
-    "@smthrs/core": "1.0.0-rc.8",
-    "@smthrs/kernel": "1.0.0-rc.8",
-    "@smthrs/model": "1.0.0-rc.8"
+    "@smthrs/capability": "1.0.0-rc.9",
+    "@smthrs/control": "1.0.0-rc.9",
+    "@smthrs/core": "1.0.0-rc.9",
+    "@smthrs/kernel": "1.0.0-rc.9",
+    "@smthrs/model": "1.0.0-rc.9"
   },
   "devDependencies": {
     "@effect/language-service": "0.87.2",

package/src/PortableSearch.ts

@@ -240,6 +240,26 @@ const glob = (
     }
   })
 
+/**
+ * The files this peer's `grep` reads for `input`, in the order it reads
+ * them: the walk under `input.root` with its `.gitignore` rules and
+ * `input.globs` applied and symbolic links left out, or the root itself when
+ * it names one file. A peer that has to bring a tree's bytes to this process
+ * before it can search them (`RemoteSearch`) fetches exactly these.
+ *
+ * @category queries
+ * @since 1.0.0-rc.8
+ */
+export const searched = (
+  input: Pick<Search.GrepInput, "root" | "hidden" | "noIgnore" | "globs">
+): Effect.Effect<ReadonlyArray<string>, StdError.StdError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fileSystem = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const walked = yield* Walk.files(fileSystem, path, input.root, input.hidden, input.noIgnore)
+    return yield* candidates(fileSystem, path, walked, input.root, input.globs)
+  })
+
 /**
  * Captures filesystem and path services in the portable peer.
  *

package/src/RemoteSearch.ts

@@ -0,0 +1,199 @@
+/**
+ * The search contract for a tree on another machine, in two round trips.
+ *
+ * `PortableSearch` asks `FileSystem` one question per directory, per entry
+ * and per file. On a machine reached through a transport, such as a
+ * benchmark's task container behind `docker exec`, every question is a
+ * process start, so one `grep` over a 6,000-file repository is some 20,000 of
+ * them. This peer asks the machine's own shell twice instead. `find` lists
+ * the tree under the root, leaving out the directories every walk skips, and
+ * `tar` returns its `.gitignore` files with the listing. The in-process peer
+ * then decides which files the search reads, a second `tar` returns exactly
+ * those, and `PortableSearch` answers over that copy. Every result, notice,
+ * and refusal is therefore the in-process peer's own: this module decides
+ * only where the bytes come from.
+ *
+ * It runs through the machine's own `ChildProcessSpawner`, the one
+ * `Sandbox.hostContext` serves beside the machine's `FileSystem`. The machine
+ * needs `sh`, `find`, `xargs`, `tar`, `head`, and `base64`; GNU, BusyBox, and
+ * BSD userlands all have them. Output crosses as base64, so a transport whose
+ * channel carries only text returns it intact. A search reads at most
+ * {@link maxBytes} of file content and refuses beyond that, naming the root
+ * to narrow.
+ *
+ * @since 1.0.0-rc.8
+ */
+
+import type * as ChildProcessSpawner from "@smthrs/kernel/ChildProcessSpawner"
+import * as Path from "@smthrs/kernel/Path"
+import { Context, Effect } from "effect"
+import * as Base64 from "effect/encoding/Base64"
+import * as FileSystem from "effect/FileSystem"
+import * as Result from "effect/Result"
+import * as Exec from "./internal/Exec.ts"
+import * as Prune from "./internal/Prune.ts"
+import * as RemoteTree from "./internal/RemoteTree.ts"
+import * as Tar from "./internal/Tar.ts"
+import * as PortableSearch from "./PortableSearch.ts"
+import * as Search from "./Search.ts"
+import * as StdError from "./StdError.ts"
+
+/**
+ * The most file content one search brings back from the machine.
+ *
+ * The copy is held in memory while the in-process peer reads it, so the
+ * bound is on this process's memory rather than on the search: a root that
+ * holds more is refused with `response_too_large` and a narrower root or
+ * globs answer it.
+ *
+ * @category constants
+ * @since 1.0.0-rc.8
+ */
+export const maxBytes = 256 * 1024 * 1024
+
+/** The most a listing, with its `.gitignore` files, may take. */
+const maxListingBytes = 64 * 1024 * 1024
+
+const quoted = (value: string): string => `'${value.replaceAll("'", `'\\''`)}'`
+
+const pruned = `\\( ${Prune.repositoryDirectories.map((name) => `-name ${quoted(name)}`).join(" -o ")} \\) -prune`
+
+/** The refusal for a machine without one of the tools a search runs. */
+const missing = (tool: string) => `searching this machine needs ${tool}, which it does not have`
+
+const requireTools = `for t in find xargs tar head; do command -v "$t" >/dev/null || { ` +
+  `echo "${missing("$t")}" >&2; exit 3; }; done`
+
+/**
+ * `$1` is the root, `$2` the byte bound, and `$3` is `1` to send the
+ * `.gitignore` files after the listing. Records are NUL-terminated `<kind><path>`;
+ * an `E` record ends the listing, and a tar stream of the ignore files follows.
+ */
+const listing = [
+  "set -f",
+  requireTools,
+  "r=$1",
+  "{",
+  "  if [ -d \"$r\" ]; then if [ -r \"$r\" ] && [ -x \"$r\" ]; then printf 'd%s\\0' \"$r\"; else printf 'u%s\\0' \"$r\"; fi",
+  "  elif [ -f \"$r\" ]; then printf 'f%s\\0' \"$r\"",
+  "  elif [ -e \"$r\" ]; then printf 'o%s\\0' \"$r\"",
+  "  fi",
+  // No `-exec … {} +` inside parentheses: BusyBox `find` never runs one there.
+  `  find -H "$r" -mindepth 1 ${pruned} -o -type d -exec printf 'd%s\\0' {} + ` +
+  `-o -type f -exec printf 'f%s\\0' {} + ` +
+  `-o -type l -exec sh -c 'for p do if [ -d "$p" ]; then k=D; elif [ -f "$p" ]; then k=F; else k=L; fi; ` +
+  `printf "%s%s\\0" "$k" "$p"; done' sh {} + -o -exec printf 'o%s\\0' {} + 2>/dev/null`,
+  "  printf 'E\\0'",
+  `  if [ "$3" = 1 ]; then find -H "$r" -mindepth 1 ${pruned} -o -type f -name .gitignore -print0 2>/dev/null | ` +
+  "(cd / && COPYFILE_DISABLE=1 xargs -0 -r tar -cf - -- 2>/dev/null); fi",
+  "} | head -c \"$2\" | base64"
+].join("\n")
+
+/** `$1` is the byte bound; standard input is the NUL-terminated absolute paths to send. */
+const fetching = [
+  requireTools,
+  "cd / && COPYFILE_DISABLE=1 xargs -0 -r tar -chf - -- 2>/dev/null | head -c \"$1\" | base64"
+].join("\n")
+
+const failed = (root: string, reason: string): StdError.StdError =>
+  new StdError.StdError({
+    code: "command_failed",
+    message: `Could not search ${root} on this machine: ${reason}`,
+    path: root
+  })
+
+const tooLarge = (root: string, bound: number): StdError.StdError =>
+  new StdError.StdError({
+    code: "response_too_large",
+    message: `The files under ${root} come to more than ${bound} bytes; narrow the root or the globs`,
+    path: root
+  })
+
+/**
+ * Reads the program from the first line of standard input, as base64, and
+ * runs it with the arguments and the rest of standard input. A program in
+ * the argv would be quoted once by the spawner and again by the transport,
+ * and a sealed host refuses to start a command line past 4,096 characters;
+ * this line stays short whatever the program says.
+ */
+const bootstrap = `command -v base64 >/dev/null || { echo "${missing("base64")}" >&2; exit 3; }; ` +
+  "IFS= read -r p && p=$(printf %s \"$p\" | base64 -d) && exec sh -c \"$p\" sh \"$@\""
+
+/**
+ * Runs one script on the machine and decodes the base64 it prints, refusing
+ * output that reached `bound`: `head` cut it, so it is not the whole answer.
+ */
+const run = (
+  root: string,
+  script: string,
+  args: ReadonlyArray<string>,
+  bound: number,
+  stdin = ""
+): Effect.Effect<Uint8Array, StdError.StdError, ChildProcessSpawner.ChildProcessSpawner> =>
+  Effect.gen(function*() {
+    const result = yield* Exec.exec("sh", {
+      args: ["-c", bootstrap, "sh", ...args],
+      stdin: `${Base64.encode(new TextEncoder().encode(script))}\n${stdin}`
+    }).pipe(Effect.mapError((error) => failed(root, error.message)))
+    if (result.exitCode !== 0) {
+      return yield* Effect.fail(failed(root, result.stderr.trim() || `exit ${result.exitCode}`))
+    }
+    const decoded = Base64.decode(result.stdout.replaceAll(/\s/g, ""))
+    if (Result.isFailure(decoded)) return yield* Effect.fail(failed(root, "the machine answered with invalid base64"))
+    if (decoded.success.length >= bound) return yield* Effect.fail(tooLarge(root, bound - 1))
+    return decoded.success
+  })
+
+/**
+ * Builds the peer over a machine's spawner and its `Path`, which resolves a
+ * relative root where the machine's file operations do. `maxBytes` lowers
+ * the bound on the file content one search brings back; the default is
+ * {@link maxBytes}.
+ *
+ * @category constructors
+ * @since 1.0.0-rc.8
+ */
+export const make = (
+  services: Context.Context<ChildProcessSpawner.ChildProcessSpawner | Path.Path>,
+  options?: { readonly maxBytes?: number | undefined }
+): Search.Search => {
+  const path = Context.get(services, Path.Path)
+  const contentBound = (options?.maxBytes ?? maxBytes) + 1
+  const copy = (tree: RemoteTree.Tree) =>
+    Context.make(FileSystem.FileSystem, RemoteTree.fileSystem(tree, path)).pipe(Context.add(Path.Path, path))
+  const listed = (root: string, noIgnore: boolean | undefined) => {
+    const resolved = path.resolve(root)
+    const bound = maxListingBytes + 1
+    return Effect.flatMap(
+      run(root, listing, [resolved, String(bound), noIgnore === true ? "0" : "1"], bound),
+      (bytes) => {
+        const tree = RemoteTree.parse(bytes, resolved, path)
+        return tree === undefined ? Effect.fail(failed(root, "the listing came back malformed")) : Effect.succeed(tree)
+      }
+    )
+  }
+  return Search.make({
+    grep: (input) =>
+      Effect.gen(function*() {
+        const tree = yield* listed(input.root, input.noIgnore)
+        const read = yield* PortableSearch.searched(input).pipe(Effect.provide(copy(tree)))
+        const files = [...new Set(read.map((file) => path.resolve(file)))]
+        const bytes = files.length === 0 ? new Uint8Array() : yield* run(
+          input.root,
+          fetching,
+          [String(contentBound)],
+          contentBound,
+          files.map((file) => `${file}\0`).join("")
+        )
+        // `tar` ends every archive it writes, even one whose files it could not read.
+        const members = files.length === 0 || bytes.length > 0 ? Tar.files(bytes) : undefined
+        if (members === undefined) return yield* Effect.fail(failed(input.root, "the files came back malformed"))
+        const contents = new Map([...tree.contents, ...RemoteTree.absolute(members)])
+        return yield* PortableSearch.make(copy({ ...tree, contents })).grep(input)
+      }).pipe(Effect.provide(services)),
+    glob: (input) =>
+      Effect.flatMap(listed(input.root, input.noIgnore), (tree) => PortableSearch.make(copy(tree)).glob(input)).pipe(
+        Effect.provide(services)
+      )
+  })
+}

package/src/index.ts

@@ -56,6 +56,9 @@ export * as PortableSearch from "./PortableSearch.ts"
 /** @category layers @since 1.0.0 */
 export * as NativeSearch from "./NativeSearch.ts"
 
+/** @category layers @since 1.0.0-rc.8 */
+export * as RemoteSearch from "./RemoteSearch.ts"
+
 /**
  * @category flows
  * @since 1.0.0

package/src/internal/EnvelopePrecheck.ts

@@ -115,6 +115,9 @@ const reservedWords = new Set([
 const writeCommands = new Set(["chmod", "chown", "mkdir", "rm", "rmdir", "tee", "touch", "truncate", "unlink"])
 const destinationCommands = new Set(["cp", "install", "mv"])
 const prefixWrappers = new Set(["env", "nice", "time", "sudo", "nohup", "xargs", "command", "exec"])
+// A shell runs the argument of its `-c` flag as shell text of its own.
+const shells = new Set(["ash", "bash", "dash", "fish", "ksh", "mksh", "sh", "zsh"])
+const commandFlag = /^-[A-Za-z]*c[A-Za-z]*$/
 
 /**
  * One lexical unit of shell text: an operator, or a word with its quotes
@@ -310,7 +313,11 @@ const isPathToken = (path: Path.Path, token: string): boolean =>
   /^[A-Za-z]:[\\/]/.test(token) ||
   token.includes("/")
 
-const segmentReferences = (path: Path.Path, tokens: ReadonlyArray<Token>): ReadonlyArray<PathReference> => {
+const segmentReferences = (
+  path: Path.Path,
+  tokens: ReadonlyArray<Token>,
+  nesting: number
+): ReadonlyArray<PathReference> => {
   // The word after a redirection operator is its target, never the command.
   const isTarget = (index: number): boolean => index > 0 && isRedirection(tokens[index - 1]!)
   const isWord = (index: number): boolean => !tokens[index]!.operator && !isTarget(index)
@@ -334,10 +341,23 @@ const segmentReferences = (path: Path.Path, tokens: ReadonlyArray<Token>): Reado
   }
 
   const command = commandIndex === -1 ? "" : name(commandIndex)
+  // Shell text handed to `eval` or to a shell's `-c` flag is a command, not a
+  // path: reading `sh -c 'rm /work/target'` as one quoted word let a delete
+  // pass as a read of a file named `rm /work/target`.
+  const scripts = new Set<number>()
+  if (command === "eval") {
+    for (let index = commandIndex + 1; index < tokens.length; index++) if (isWord(index)) scripts.add(index)
+  } else if (shells.has(command)) {
+    const flag = tokens.findIndex((token, index) =>
+      index > commandIndex && isWord(index) && commandFlag.test(token.text)
+    )
+    if (flag !== -1 && flag + 1 < tokens.length && isWord(flag + 1)) scripts.add(flag + 1)
+  }
   const candidates: Array<{ readonly index: number; readonly value: string }> = []
   for (let index = 0; index < tokens.length; index++) {
     const token = tokens[index]!
-    if (index === commandIndex || token.operator || (!isTarget(index) && token.text.startsWith("-"))) continue
+    if (index === commandIndex || scripts.has(index) || token.operator) continue
+    if (!isTarget(index) && token.text.startsWith("-")) continue
     if (isTarget(index) && textRedirections.has(tokens[index - 1]!.text)) continue
     const value = pathValue(token.text)
     if (value !== "" && isPathToken(path, value)) candidates.push({ index, value })
@@ -347,7 +367,9 @@ const segmentReferences = (path: Path.Path, tokens: ReadonlyArray<Token>): Reado
     ? candidates.filter(({ index }) => !isTarget(index)).at(-1)?.index
     : undefined
 
-  return candidates.map(({ index, value }) => {
+  const script = [...scripts].map((index) => tokens[index]!.text).join(" ")
+  const nested = script === "" ? [] : references(path, script, nesting + 1)
+  const named = candidates.map(({ index, value }): PathReference => {
     const access: Access = isTarget(index)
       ? writeRedirections.has(tokens[index - 1]!.text) ? "write" : "read"
       : writeCommands.has(command) || destinationIndex === index
@@ -355,21 +377,23 @@ const segmentReferences = (path: Path.Path, tokens: ReadonlyArray<Token>): Reado
       : "read"
     return { access, value }
   })
+  return [...named, ...nested]
 }
 
 const references = (path: Path.Path, command: string, nesting: number): ReadonlyArray<PathReference> => {
+  if (nesting > maxNesting) throw tooDeep()
   const { substitutions, tokens } = lex(command, nesting)
   const found: Array<PathReference> = []
   let segment: Array<Token> = []
   for (const token of tokens) {
     if (token.operator && commandSeparators.has(token.text)) {
-      found.push(...segmentReferences(path, segment))
+      found.push(...segmentReferences(path, segment, nesting))
       segment = []
     } else {
       segment.push(token)
     }
   }
-  found.push(...segmentReferences(path, segment))
+  found.push(...segmentReferences(path, segment, nesting))
   // A substitution runs its body as a command; its paths count like any other.
   for (const body of substitutions) found.push(...references(path, body, nesting + 1))
   return found

package/src/internal/RemoteTree.ts

@@ -0,0 +1,159 @@
+/**
+ * A tree another machine listed, served as a read-only `FileSystem`.
+ *
+ * `RemoteSearch` lists a root on the machine with `find` and brings back the
+ * bytes the search reads with `tar`; this module holds what came back and
+ * answers what the walk, the globs, and the matcher ask of a filesystem, so
+ * `PortableSearch` can search the copy as it searches a host.
+ *
+ * @since 1.0.0-rc.8
+ */
+
+import type * as Path from "@smthrs/kernel/Path"
+import { Effect, Option, Stream } from "effect"
+import * as ByteSize from "effect/ByteSize"
+import * as FileSystem from "effect/FileSystem"
+import * as PlatformError from "effect/PlatformError"
+import * as Tar from "./Tar.ts"
+
+/**
+ * What one listed path is: `d` a directory, `u` a directory the machine's
+ * user may not list, `f` a file, `o` anything else, and a symbolic link to a
+ * directory (`D`), to a file (`F`), or to neither (`L`).
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export type Kind = "d" | "u" | "f" | "o" | "D" | "F" | "L"
+
+/**
+ * One machine's listing of a root, and the bytes it brought back, by
+ * absolute path.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export interface Tree {
+  readonly kinds: ReadonlyMap<string, Kind>
+  readonly children: ReadonlyMap<string, ReadonlyArray<string>>
+  readonly contents: ReadonlyMap<string, Uint8Array>
+}
+
+const kinds: ReadonlySet<string> = new Set(["d", "u", "f", "o", "D", "F", "L"])
+
+/**
+ * The member names `tar` wrote for absolute paths, which it strips of their
+ * leading `/`, as those paths again.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const absolute = (members: ReadonlyMap<string, Uint8Array>): ReadonlyMap<string, Uint8Array> =>
+  new Map([...members].map(([name, bytes]) => [`/${name}`, bytes]))
+
+/**
+ * Reads NUL-terminated `<kind><path>` records up to an `E` record, then the
+ * tar stream of `.gitignore` files after it. `root` is the listed root, which
+ * is nobody's child. `undefined` when the bytes are not that shape.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const parse = (bytes: Uint8Array, root: string, path: Path.Path): Tree | undefined => {
+  const decoder = new TextDecoder()
+  const listed = new Map<string, Kind>()
+  const children = new Map<string, Array<string>>()
+  let offset = 0
+  for (;;) {
+    const end = bytes.indexOf(0, offset)
+    if (end === -1) return undefined
+    const record = decoder.decode(bytes.subarray(offset, end))
+    offset = end + 1
+    if (record === "E") break
+    const kind = record.slice(0, 1)
+    if (!kinds.has(kind)) return undefined
+    const entry = record.slice(1)
+    listed.set(entry, kind as Kind)
+    if (entry === root) continue
+    const parent = path.dirname(entry)
+    const siblings = children.get(parent)
+    if (siblings === undefined) children.set(parent, [path.basename(entry)])
+    else siblings.push(path.basename(entry))
+  }
+  const ignores = Tar.files(bytes.subarray(offset))
+  return ignores === undefined ? undefined : { kinds: listed, children, contents: absolute(ignores) }
+}
+
+const refusal = (tag: PlatformError.SystemErrorTag, method: string, path: string): PlatformError.PlatformError =>
+  PlatformError.systemError({ _tag: tag, module: "FileSystem", method, pathOrDescriptor: path })
+
+/** What `stat` says each kind is: links resolved, a dangling one absent. */
+const types: Readonly<Record<Kind, FileSystem.File.Type | undefined>> = {
+  d: "Directory",
+  u: "Directory",
+  D: "Directory",
+  f: "File",
+  F: "File",
+  o: "Unknown",
+  L: undefined
+}
+
+const info = (type: FileSystem.File.Type, size: number): FileSystem.File.Info => ({
+  type,
+  size: ByteSize.bytes(BigInt(size)),
+  mtime: Option.none(),
+  atime: Option.none(),
+  birthtime: Option.none(),
+  dev: 0,
+  ino: Option.none(),
+  mode: 0,
+  nlink: Option.none(),
+  uid: Option.none(),
+  gid: Option.none(),
+  rdev: Option.none(),
+  blksize: Option.none(),
+  blocks: Option.none()
+})
+
+/**
+ * The tree as a read-only `FileSystem`, resolving paths with `path`. A file
+ * whose bytes did not come back is one the machine's user could not read,
+ * which the in-process peer skips exactly as it skips such a file on a host.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const fileSystem = (tree: Tree, path: Path.Path): FileSystem.FileSystem => {
+  const kindOf = (value: string) => tree.kinds.get(path.resolve(value))
+  const readFile = (value: string) => {
+    const bytes = tree.contents.get(path.resolve(value))
+    if (bytes !== undefined) return Effect.succeed(bytes)
+    const type = types[kindOf(value) ?? "L"]
+    return Effect.fail(
+      refusal(type === "File" ? "PermissionDenied" : type === undefined ? "NotFound" : "BadResource", "readFile", value)
+    )
+  }
+  return FileSystem.makeNoop({
+    stat: (value) => {
+      const type = types[kindOf(value) ?? "L"]
+      return type === undefined
+        ? Effect.fail(refusal("NotFound", "stat", value))
+        : Effect.succeed(info(type, tree.contents.get(path.resolve(value))?.length ?? 0))
+    },
+    readDirectory: (value) => {
+      const kind = kindOf(value)
+      if (kind === "d") return Effect.succeed([...tree.children.get(path.resolve(value)) ?? []])
+      const tag = kind === "u" ? "PermissionDenied" : kind === undefined || kind === "L" ? "NotFound" : "BadResource"
+      return Effect.fail(refusal(tag, "readDirectory", value))
+    },
+    readLink: (value) => {
+      const kind = kindOf(value)
+      return kind === "D" || kind === "F" || kind === "L"
+        ? Effect.succeed("")
+        : Effect.fail(refusal(kind === undefined ? "NotFound" : "BadResource", "readLink", value))
+    },
+    readFile,
+    readFileString: (value) => Effect.map(readFile(value), (bytes) => new TextDecoder().decode(bytes)),
+    stream: (value) => Stream.fromEffect(readFile(value))
+  })
+}

package/src/internal/Tar.ts

@@ -0,0 +1,95 @@
+/**
+ * The regular files of a tar stream.
+ *
+ * `RemoteSearch` brings a machine's files back with its own `tar`, because
+ * every image ships one and it frames each file with its name and size. GNU
+ * and BusyBox `tar` write GNU long names, `bsdtar` writes POSIX extended
+ * headers, and `xargs` may start `tar` more than once, so the stream can hold
+ * several archives one after another. This reads all of them.
+ *
+ * @since 1.0.0-rc.8
+ */
+
+const decoder = new TextDecoder()
+
+/** Bytes up to the first NUL, as text. */
+const text = (bytes: Uint8Array): string => {
+  const end = bytes.indexOf(0)
+  return decoder.decode(end === -1 ? bytes : bytes.subarray(0, end))
+}
+
+/** An octal header field, or `undefined` when it holds anything else. */
+const octal = (bytes: Uint8Array): number | undefined => {
+  const digits = text(bytes).trim()
+  return /^[0-7]+$/.test(digits) ? Number.parseInt(digits, 8) : undefined
+}
+
+/** The header's checksum, computed with the checksum field read as spaces. */
+const checksum = (header: Uint8Array): number => {
+  let sum = 0
+  for (let index = 0; index < 512; index++) sum += index >= 148 && index < 156 ? 32 : header[index]!
+  return sum
+}
+
+/** The `path` record of a POSIX extended header: `<length> <key>=<value>\n` records. */
+const paxPath = (data: Uint8Array): string | undefined => {
+  let offset = 0
+  let path: string | undefined
+  while (offset < data.length) {
+    const space = data.indexOf(32, offset)
+    const length = space === -1 ? Number.NaN : Number(decoder.decode(data.subarray(offset, space)))
+    if (!Number.isSafeInteger(length) || length <= space - offset || offset + length > data.length) return path
+    const record = decoder.decode(data.subarray(space + 1, offset + length - 1))
+    if (record.startsWith("path=")) path = record.slice("path=".length)
+    offset += length
+  }
+  return path
+}
+
+/**
+ * Every regular file the stream holds, by member name, or `undefined` when
+ * the stream is not whole tar archives: a header whose checksum does not
+ * hold, a member cut short, or a last archive with no end. A `tar` that
+ * stopped between two members wrote no end, so its missing files are not
+ * mistaken for files it could not read.
+ *
+ * @private
+ * @since 1.0.0-rc.8
+ */
+export const files = (stream: Uint8Array): ReadonlyMap<string, Uint8Array> | undefined => {
+  const found = new Map<string, Uint8Array>()
+  let longName: string | undefined
+  let extended: string | undefined
+  let ended = true
+  for (let offset = 0; offset + 512 <= stream.length;) {
+    const header = stream.subarray(offset, offset + 512)
+    offset += 512
+    // Zero blocks end one archive; `xargs` may have started another.
+    ended = header.every((byte) => byte === 0)
+    if (ended) continue
+    const size = octal(header.subarray(124, 136))
+    if (size === undefined || octal(header.subarray(148, 156)) !== checksum(header)) return undefined
+    if (offset + size > stream.length) return undefined
+    const data = stream.subarray(offset, offset + size)
+    offset += Math.ceil(size / 512) * 512
+    const type = String.fromCharCode(header[156]!)
+    if (type === "L") {
+      longName = text(data)
+      continue
+    }
+    if (type === "x") {
+      extended = paxPath(data) ?? extended
+      continue
+    }
+    // A long link name and a global header describe no member of their own.
+    if (type === "K" || type === "g") continue
+    if (type === "0" || type === "\0" || type === "7") {
+      const prefix = decoder.decode(header.subarray(257, 263)) === "ustar\0" ? text(header.subarray(345, 500)) : ""
+      const short = text(header.subarray(0, 100))
+      found.set(extended ?? longName ?? (prefix === "" ? short : `${prefix}/${short}`), data.slice())
+    }
+    longName = undefined
+    extended = undefined
+  }
+  return ended ? found : undefined
+}