@smthrs/std 1.0.0-rc.7 → 1.0.0-rc.9

package/docs/api.md

@@ -180,10 +180,14 @@ resolves its own services when it is built.
 The in-process peer. Walks the injected `FileSystem`, so it needs no external
 binary and is browser-safe.
 
-| Export  | Type                                                       |
-| ------- | ---------------------------------------------------------- |
-| `make`  | `(services: Context<FileSystem \| Path>) => Search.Search` |
-| `layer` | `Layer<Search.Search, never, FileSystem \| Path>`          |
+| Export     | Type                                                                                                                                   |
+| ---------- | -------------------------------------------------------------------------------------------------------------------------------------- |
+| `make`     | `(services: Context<FileSystem \| Path>) => Search.Search`                                                                             |
+| `layer`    | `Layer<Search.Search, never, FileSystem \| Path>`                                                                                      |
+| `searched` | `(input: Pick<GrepInput, "root" \| "hidden" \| "noIgnore" \| "globs">) => Effect<ReadonlyArray<string>, StdError, FileSystem \| Path>` |
+
+`searched` names the files `grep` reads for an input, in the order it reads
+them; `RemoteSearch` fetches exactly those.
 
 ## NativeSearch
 
@@ -201,6 +205,26 @@ start `rg` is `provider_unavailable`. The default layer uses the host environmen
 allowlist. `make` accepts explicit environment declarations as its second
 argument; those names override the allowlist.
 
+## RemoteSearch
+
+The peer for a tree on another machine, such as a container or a sandbox
+machine. It lists the root with the machine's `find`, brings back its
+`.gitignore` files and then the files the search reads with `tar`, and lets
+`PortableSearch` answer over that copy, so every result, notice, and refusal is
+the in-process peer's. Each search is two round trips through the machine's
+spawner, whatever the tree's size.
+
+| Export     | Type                                                                                                 | Meaning                                                                      |
+| ---------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| `maxBytes` | `268_435_456`                                                                                        | The most file content one search brings back; past it, `response_too_large`. |
+| `make`     | `(services: Context<ChildProcessSpawner \| Path>, options?: { maxBytes?: number }) => Search.Search` | The spawner runs on the machine; `Path` resolves a relative root there.      |
+
+The machine needs `sh`, `find`, `xargs`, `tar`, `head`, and `base64`; GNU,
+BusyBox, and BSD userlands all have them, and a machine without one refuses
+with `command_failed` naming it. The program goes in on standard input and
+the answer comes back as base64, so a transport whose argv is bounded or whose
+output channel carries only text answers intact.
+
 ## SearchContract
 
 The validation and matching rules both peers build on, exported so a third peer
@@ -307,23 +331,26 @@ How the project under test runs its suite, declared once by the host.
 
 Pinned trees, and the scratch checkouts a call runs against.
 
-| Export             | Type                                                                       | Meaning                                                          |
-| ------------------ | -------------------------------------------------------------------------- | ---------------------------------------------------------------- |
-| `Checkpoints`      | interface and `Context.Service` tag                                        | `{ capture, materialize }`.                                      |
-| `Snapshot`         | `Schema.Class`                                                             | `{ id, ref }`, identified as `@smthrs/std/Checkpoints/Snapshot`. |
-| `Materialized`     | interface                                                                  | `id`, `host`, `guest`, `root`, `guestRoot`.                      |
-| `baseId`           | `"base"`                                                                   | The id naming the tree the run opened on.                        |
-| `scratchDirectory` | `".flows-checkpoints"`                                                     | Where a checkpoint is materialized, relative to the root.        |
-| `configSection`    | `"flows-checkpoint"`                                                       | The git-config section minted checkpoints are recorded under.    |
-| `make`             | `(service: Checkpoints) => Checkpoints`                                    |                                                                  |
-| `unavailable`      | `(id: string) => StdError`                                                 | The refusal a host that pins nothing answers with.               |
-| `makeNoop`         | `() => Checkpoints`                                                        | Fails both calls with `unavailable`.                             |
-| `layerNoop`        | `Layer<Checkpoints>`                                                       |                                                                  |
-| `GitOptions`       | interface                                                                  | `root`, `cwd?`, `baseRef?`.                                      |
-| `makeGit`          | `(options: GitOptions) => Effect<Checkpoints, never, ChildProcessSpawner>` |                                                                  |
-| `layerGit`         | `(options: GitOptions) => Layer<Checkpoints, never, ChildProcessSpawner>`  |                                                                  |
-| `Relocation`       | tagged union                                                               | `Relocated`, `UnsupportedFlow`, `AbsolutePath`, `OutsideTree`.   |
-| `relocate`         | `(flow: string, input: Json, materialized: Materialized) => Relocation`    | Rewrites one call's input onto a checkpoint.                     |
+| Export             | Type                                                                                          | Meaning                                                          |
+| ------------------ | --------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- |
+| `Checkpoints`      | interface and `Context.Service` tag                                                           | `{ capture, materialize }`.                                      |
+| `Snapshot`         | `Schema.Class`                                                                                | `{ id, ref }`, identified as `@smthrs/std/Checkpoints/Snapshot`. |
+| `Materialized`     | interface                                                                                     | `id`, `host`, `guest`, `root`, `guestRoot`.                      |
+| `baseId`           | `"base"`                                                                                      | The id naming the tree the run opened on.                        |
+| `scratchDirectory` | `".flows-checkpoints"`                                                                        | Where a checkpoint is materialized, relative to the root.        |
+| `configSection`    | `"flows-checkpoint"`                                                                          | The git-config section minted checkpoints are recorded under.    |
+| `make`             | `(service: Checkpoints) => Checkpoints`                                                       |                                                                  |
+| `unavailable`      | `(id: string) => StdError`                                                                    | The refusal a host that pins nothing answers with.               |
+| `makeNoop`         | `() => Checkpoints`                                                                           | Fails both calls with `unavailable`.                             |
+| `layerNoop`        | `Layer<Checkpoints>`                                                                          |                                                                  |
+| `GitOptions`       | interface                                                                                     | `root`, `cwd?`, `baseRef?`.                                      |
+| `makeGit`          | `(options: GitOptions) => Effect<Checkpoints, never, ChildProcessSpawner>`                    |                                                                  |
+| `layerGit`         | `(options: GitOptions) => Layer<Checkpoints, never, ChildProcessSpawner>`                     |                                                                  |
+| `ContainerOptions` | interface                                                                                     | `container`, `cwd?`.                                             |
+| `makeContainer`    | `(options: ContainerOptions) => Effect<Checkpoints, never, ChildProcessSpawner \| Container>` | Runs git inside the container `bash` runs in.                    |
+| `layerContainer`   | `(options: ContainerOptions) => Layer<Checkpoints, never, ChildProcessSpawner \| Container>`  |                                                                  |
+| `Relocation`       | tagged union                                                                                  | `Relocated`, `UnsupportedFlow`, `AbsolutePath`, `OutsideTree`.   |
+| `relocate`         | `(flow: string, input: Json, materialized: Materialized) => Relocation`                       | Rewrites one call's input onto a checkpoint.                     |
 
 `capture(id)` returns a `Snapshot`. `materialize(id, use)` is scoped: it hands
 the tree to `use` and removes the checkout however that effect ends. Host git

package/docs/concepts/effects-and-capabilities.md

@@ -112,7 +112,9 @@ handler checks the command against them before spawning anything. Read what the
 check does, because the name promises more than it delivers.
 
 The check is **lexical**. It tokenizes the command line, keeping quoted spans
-whole, treats each physical line as its own command, classifies each path token
+whole, treats each physical line as its own command, reads the body of a
+command substitution, the argument of a shell's `-c` flag, and the arguments of
+`eval` as commands of their own, classifies each path token
 as a read or a write from the command that governs it, resolves both the token
 and every declaration to a canonical absolute path, and refuses the call with
 `outside_declared_reads` or `outside_declared_writes` when one falls outside.

package/docs/guides/bind-the-standard-flows.md

@@ -36,7 +36,9 @@ const host = Layer.mergeAll(
 
 Swap `NativeSearch.layer` for `PortableSearch.layer` to search in process with
 no external binary. Both satisfy the same `Search` service, and nothing above
-the layer changes.
+the layer changes. When the files live on another machine, such as a container
+or a sandbox machine, `RemoteSearch.make` over that machine's spawner and `Path`
+answers in two round trips where the portable peer would ask once per file.
 
 A host that cannot serve a service binds its refusal layer instead of leaving it
 out, so the failure names the missing service rather than a missing context:

package/docs/guides/pin-a-checkpoint.md

@@ -32,6 +32,26 @@ const store = Checkpoints.layerGit({
 `Checkpoints.layerNoop`, whose refusal says to take the reading on the live tree
 instead.
 
+## Bind the container store
+
+When the repository lives only in a container and `bash` runs there, bind
+`layerContainer` instead. It runs git inside that container through the
+`Container` transport `bash` uses, so `ctx.base` and `ctx.checkpoint()` read the
+container's tree. A git store would look for the repository on the host and
+refuse every call that names a checkpoint.
+
+```ts
+const store = Checkpoints.layerContainer({
+  container: "task", // the container `bash` runs in
+  cwd: "/app" // optional; absent is the container's working directory
+})
+```
+
+`layerContainer` requires a `ChildProcessSpawner` and a `Container`. It records
+and names a checkpoint as the git store does, and checks a tree out at the same
+`<top>/.flows-checkpoints/<id>-<lease>` path inside the container, so a
+relocated `bash` call keeps its subdirectory.
+
 ## Capture and materialize
 
 ```ts

package/docs/guides/run-a-shell-command.md

@@ -91,7 +91,10 @@ tracked in [#1790](https://github.com/smithersai/smithers/issues/1790).
 A host sealed to one container is the exception. `SMITHERS_BASH_CONTAINER` names
 the container, and `bash` and `test` then run in it through the `docker` CLI.
 Every other container, and a call that names none, fails with
-`outside_container`. The transport process runs outside the OS sandbox, which
+`outside_container`. The file flows (`read`, `edit`, `apply_patch`, `grep` and
+the rest) act on the container's files through the same transport, resolving a
+relative path under `SMITHERS_TEST_CWD`, and `grep` and `glob` answer through
+`RemoteSearch`. The transport process runs outside the OS sandbox, which
 denies the daemon's socket, and receives the host's `DOCKER_HOST`,
 `DOCKER_CONTEXT`, `DOCKER_CONFIG`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH`,
 so it reaches a rootless daemon.

package/docs/installation.md

@@ -7,7 +7,11 @@ sidebar:
 
 ## Install the package
 
-Not on npm yet; see [Installation](https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/flow/docs/installation.md#use-the-libraries).
+```bash
+npm install @smthrs/std@next effect
+```
+
+The `next` tag carries the 1.0 release candidates.
 
 It requires
 Node.js 26.4.0 or later and ships as both ESM and CommonJS with TypeScript
@@ -92,14 +96,14 @@ Six flows reach the host through a service rather than through a platform layer.
 Each one has a refusal layer for a host that cannot serve it, so a flow whose
 service is missing says so instead of succeeding quietly.
 
-| Service          | Refusal layer              | Working layer                                   |
-| ---------------- | -------------------------- | ----------------------------------------------- |
-| `Search`         | `Search.layerNoop`         | `PortableSearch.layer` or `NativeSearch.layer`  |
-| `Container`      | `Container.layerNoop`      | `Container.layerCommand({ program: "docker" })` |
-| `TestRunner`     | `TestRunner.layerNoop`     | `TestRunner.layer(runner)`                      |
-| `Checkpoints`    | `Checkpoints.layerNoop`    | `Checkpoints.layerGit({ root })`                |
-| `WebSearch`      | `WebSearch.layerNoop`      | `ExaWebSearch.layer(credentialId)`              |
-| `LanguageServer` | `LanguageServer.layerNoop` | `NodeLanguageServer.layer(config)`              |
+| Service          | Refusal layer              | Working layer                                                                   |
+| ---------------- | -------------------------- | ------------------------------------------------------------------------------- |
+| `Search`         | `Search.layerNoop`         | `PortableSearch.layer` or `NativeSearch.layer`                                  |
+| `Container`      | `Container.layerNoop`      | `Container.layerCommand({ program: "docker" })`                                 |
+| `TestRunner`     | `TestRunner.layerNoop`     | `TestRunner.layer(runner)`                                                      |
+| `Checkpoints`    | `Checkpoints.layerNoop`    | `Checkpoints.layerGit({ root })` or `Checkpoints.layerContainer({ container })` |
+| `WebSearch`      | `WebSearch.layerNoop`      | `ExaWebSearch.layer(credentialId)`                                              |
+| `LanguageServer` | `LanguageServer.layerNoop` | `NodeLanguageServer.layer(config)`                                              |
 
 `NativeSearch.layer` drives the `rg` executable, so a host that binds it needs
 ripgrep on the path. `PortableSearch.layer` needs nothing beyond `FileSystem`

package/docs/quickstart.md

@@ -14,7 +14,7 @@ without an external binary and without an API key.
 ## Prerequisites
 
 - Node.js 26.4.0 or later.
-- A package with the dependencies installed. Not on npm yet; see [Installation](https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/flow/docs/installation.md#use-the-libraries).
+- A project with the packages installed: `npm install @smthrs/std@next @effect/platform-node effect`.
 
 ## Make a tree to search
 

package/docs/troubleshooting.md

@@ -149,6 +149,14 @@ not covered by its own `reads` or `writes`. The message names the resolved path.
 check is lexical and fail-closed: it reads the tokens in the command line, not
 what the process will actually touch.
 
+## invalid_input: a command nested deeper than 32 levels
+
+**What happened.** A hermetic `bash` call nested command substitutions, quotes,
+shell `-c` arguments, or `eval` calls more than 32 levels deep. The pre-check
+refuses text it cannot read in bounded time rather than run it unchecked.
+
+**What to change.** Flatten the command, or use `mode: "unhermetic"`.
+
 ## invalid_input: a hermetic call that names a container
 
 **What happened.** `mode: "hermetic"` together with `container`. A container has
@@ -160,6 +168,18 @@ The same rule refuses a hermetic `script` whose `interpreter` is not a shell:
 the pre-check reads shell text, and Python is not shell text. Run it
 unhermetically, or express it as shell.
 
+## invalid_input: Command not started: its arguments exceed the operating system's size limit (E2BIG)
+
+**What happened.** The operating system refused to start the command because
+its arguments are too large: one argument over 128 KiB on Linux, or all of them
+over 1 MiB on macOS. A long heredoc in `command`, or a file's text passed in
+`args`, is one such argument. For a `container` call the host's `docker exec`
+carries the same arguments, so the same limit applies.
+
+**What to change.** Send the text on standard input, which has no size limit.
+Put a program and its data in `script`, or keep `command` short and pass the
+data in `stdin`. Never put long text in `command` or `args`.
+
 ## provider_unavailable: this host has no container transport
 
 **What happened.** A call named a `container` and no `Container` service was

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@smthrs/std",
   "type": "module",
-  "version": "1.0.0-rc.7",
+  "version": "1.0.0-rc.9",
   "license": "MIT",
   "smthrs": {
     "group": "agent"
@@ -252,6 +252,16 @@
         "default": "./dist/cjs/Relocate.js"
       }
     },
+    "./RemoteSearch": {
+      "import": {
+        "types": "./dist/esm/RemoteSearch.d.ts",
+        "default": "./dist/esm/RemoteSearch.js"
+      },
+      "require": {
+        "types": "./dist/cjs/RemoteSearch.d.ts",
+        "default": "./dist/cjs/RemoteSearch.js"
+      }
+    },
     "./Search": {
       "import": {
         "types": "./dist/esm/Search.d.ts",
@@ -401,11 +411,11 @@
     "tag": "next"
   },
   "dependencies": {
-    "@smthrs/capability": "1.0.0-rc.7",
-    "@smthrs/control": "1.0.0-rc.7",
-    "@smthrs/core": "1.0.0-rc.7",
-    "@smthrs/kernel": "1.0.0-rc.7",
-    "@smthrs/model": "1.0.0-rc.7"
+    "@smthrs/capability": "1.0.0-rc.9",
+    "@smthrs/control": "1.0.0-rc.9",
+    "@smthrs/core": "1.0.0-rc.9",
+    "@smthrs/kernel": "1.0.0-rc.9",
+    "@smthrs/model": "1.0.0-rc.9"
   },
   "devDependencies": {
     "@effect/language-service": "0.87.2",

package/src/Checkpoints.ts

@@ -72,6 +72,8 @@
 
 import { ChildProcessSpawner } from "@smthrs/kernel/ChildProcessSpawner"
 import { Context, Effect, Layer, Schema } from "effect"
+import * as Container from "./Container.ts"
+import * as Exec from "./internal/Exec.ts"
 import { GitWorktree } from "./internal/GitWorktree.ts"
 import { withoutTrailingSlash } from "./internal/Paths.ts"
 import * as Prune from "./internal/Prune.ts"
@@ -409,6 +411,141 @@ const gitStore = (options: GitOptions, spawner: ChildProcessSpawner["Service"]):
  */
 export const layerGit = (options: GitOptions): Layer.Layer<Checkpoints, never, ChildProcessSpawner> =>
   Layer.effect(Checkpoints)(makeGit(options))
+
+/**
+ * What {@link makeContainer} needs to know about the container it pins in.
+ *
+ * @category models
+ * @since 1.0.0-rc.8
+ */
+export interface ContainerOptions {
+  /** The container `bash` runs in. */
+  readonly container: string
+  /** A directory of the repository inside it; absent is the container's own working directory. */
+  readonly cwd?: string | undefined
+}
+
+/**
+ * `$1` is the id. Records the working tree with `stash create` over a copy of
+ * the index, names the commit in config as {@link makeGit} does, and prints it.
+ */
+const pinScript = [
+  "set -e",
+  "cd \"$(git rev-parse --show-toplevel)\"",
+  "copy=$(mktemp -d)",
+  "trap 'rm -rf -- \"$copy\"' EXIT",
+  "index=$(git rev-parse --git-path index)",
+  "if [ -e \"$index\" ]; then cp -p -- \"$index\" \"$copy/index\"; fi",
+  "commit=$(GIT_INDEX_FILE=\"$copy/index\" git -c user.name=flows -c user.email=flows@localhost stash create \"flows checkpoint $1\")",
+  "[ -n \"$commit\" ] || commit=$(git rev-parse --verify 'HEAD^{commit}')",
+  `git config "${configSection}.$1" "$commit"`,
+  "printf '%s\\n' \"$commit\""
+].join("\n")
+
+/**
+ * `$1` is the id, `$2` the lease. Checks the tree out at
+ * `<top>/.flows-checkpoints/<id>-<lease>` in a repository of its own whose
+ * objects are the workspace's, and prints the top and the checkout.
+ */
+const checkoutScript = [
+  "set -e",
+  "top=$(git rev-parse --show-toplevel)",
+  "cd \"$top\"",
+  "common=$(cd \"$(git rev-parse --git-common-dir)\" && pwd -P)",
+  `if [ "$1" = ${baseId} ]; then`,
+  `  commit=$(git rev-parse --verify --quiet '${TestRunner.captureBase}^{commit}' || git rev-parse --verify --quiet 'HEAD^{commit}') || { echo "No commit resolves from ${TestRunner.captureBase} or HEAD in $top." >&2; exit 1; }`,
+  "else",
+  `  commit=$(git config --get "${configSection}.$1") || { echo "No checkpoint is stored under $1 in $top. Take the reading on the live tree instead." >&2; exit 1; }`,
+  "fi",
+  `dir="$top/${scratchDirectory}/$1-$2"`,
+  "trap 'rm -rf -- \"$dir\"' EXIT",
+  `mkdir -p -- "$top/${scratchDirectory}"`,
+  "git init --quiet \"$dir\"",
+  "printf '%s/objects\\n' \"$common\" > \"$dir/.git/objects/info/alternates\"",
+  "git -C \"$dir\" checkout --quiet --detach \"$commit\"",
+  "trap - EXIT",
+  "printf '%s\\n%s\\n' \"$top\" \"$dir\""
+].join("\n")
+
+/**
+ * Builds the store for a host whose `bash` runs in one container.
+ *
+ * The repository is the container's, so every git command runs there, through
+ * the same {@link Container.Container} transport `bash` takes. The host store
+ * looked in the CLI's own directory instead and refused every `at` in 112 of
+ * 123 trials of the 2026-10-14 SWE-bench Pro run. This keeps that store's
+ * shape: a checkpoint is a `stash create` commit named in config, and a tree
+ * is checked out under the repository for one call, so a relocated `bash`
+ * call keeps its subdirectory and still finds what the workspace installs
+ * beside it. That git may run what the repository configures, but only inside
+ * the container the agent already commands.
+ *
+ * @category constructors
+ * @since 1.0.0-rc.8
+ */
+export const makeContainer = (
+  options: ContainerOptions
+): Effect.Effect<Checkpoints, never, ChildProcessSpawner | Container.Container> =>
+  Effect.gen(function*() {
+    const spawner = yield* ChildProcessSpawner
+    const transport = yield* Container.Container
+    const sh = (script: string, args: ReadonlyArray<string>) =>
+      transport.exec({
+        container: options.container,
+        file: "sh",
+        args: ["-c", script, "sh", ...args],
+        cwd: options.cwd,
+        stdin: false
+      }).pipe(
+        Effect.flatMap((plan) =>
+          Exec.exec(plan.file, { args: plan.args, env: plan.env }).pipe(
+            Effect.mapError((error) => Exec.toStdError(plan.file, error))
+          )
+        ),
+        Effect.flatMap((result) =>
+          result.exitCode === 0
+            ? Effect.succeed(result.stdout.trim().split("\n"))
+            : Effect.fail(failed(`git in container ${options.container} failed: ${result.stderr.trim()}`))
+        ),
+        Effect.provideService(ChildProcessSpawner, spawner)
+      )
+    const named = (id: string) =>
+      namable.test(id)
+        ? Effect.void
+        : Effect.fail(failed(`A checkpoint id must match ${namable.source}; ${id} does not.`, "invalid_input"))
+    return make({
+      capture: (id) =>
+        named(id).pipe(
+          Effect.andThen(sh(pinScript, [id])),
+          Effect.map(([commit]) => new Snapshot({ id, ref: commit! }))
+        ),
+      materialize: (id, use) =>
+        named(id).pipe(Effect.andThen(Effect.acquireUseRelease(
+          sh(checkoutScript, [id, globalThis.crypto.randomUUID()]),
+          ([root, checkout]) => use({ id, host: checkout!, guest: checkout!, root: root!, guestRoot: root! }),
+          ([, checkout]) =>
+            sh("rm -rf -- \"$1\"", [checkout!]).pipe(
+              Effect.catch((error) =>
+                Effect.logWarning("could not remove a scratch checkout").pipe(
+                  Effect.annotateLogs({ path: checkout, reason: error.message })
+                )
+              )
+            )
+        )))
+    })
+  })
+
+/**
+ * Provides {@link makeContainer}.
+ *
+ * @category layers
+ * @since 1.0.0-rc.8
+ */
+export const layerContainer = (
+  options: ContainerOptions
+): Layer.Layer<Checkpoints, never, ChildProcessSpawner | Container.Container> =>
+  Layer.effect(Checkpoints)(makeContainer(options))
+
 /**
  * Rewrites one call's input so the call runs against a materialized checkpoint.
  *

package/src/PortableSearch.ts

@@ -240,6 +240,26 @@ const glob = (
     }
   })
 
+/**
+ * The files this peer's `grep` reads for `input`, in the order it reads
+ * them: the walk under `input.root` with its `.gitignore` rules and
+ * `input.globs` applied and symbolic links left out, or the root itself when
+ * it names one file. A peer that has to bring a tree's bytes to this process
+ * before it can search them (`RemoteSearch`) fetches exactly these.
+ *
+ * @category queries
+ * @since 1.0.0-rc.8
+ */
+export const searched = (
+  input: Pick<Search.GrepInput, "root" | "hidden" | "noIgnore" | "globs">
+): Effect.Effect<ReadonlyArray<string>, StdError.StdError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fileSystem = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const walked = yield* Walk.files(fileSystem, path, input.root, input.hidden, input.noIgnore)
+    return yield* candidates(fileSystem, path, walked, input.root, input.globs)
+  })
+
 /**
  * Captures filesystem and path services in the portable peer.
  *