@smthrs/std 0.0.0-stage → 1.0.0-rc.4

package/src/internal/CodexText.ts

@@ -0,0 +1,182 @@
+/**
+ * Faithful port of Codex CLI output truncation.
+ *
+ * Mirrors `codex-rs/utils/string/src/truncate.rs` and
+ * `codex-rs/utils/output-truncation/src/lib.rs`: middle truncation on UTF-8
+ * boundaries with a `…{n} tokens truncated…` marker and a 4-bytes-per-token
+ * estimate, so ChatGPT-family models see the exact shapes they were
+ * posttrained on.
+ *
+ * @since 1.0.0
+ */
+
+const APPROX_BYTES_PER_TOKEN = 4
+
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+
+/**
+ * UTF-8 byte length of a string.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const byteLength = (text: string): number => encoder.encode(text).byteLength
+
+/**
+ * Approximate token count: ceil(utf8Bytes / 4).
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxTokenCount = (text: string): number =>
+  Math.floor((byteLength(text) + APPROX_BYTES_PER_TOKEN - 1) / APPROX_BYTES_PER_TOKEN)
+
+/**
+ * Approximate byte budget for a token budget.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxBytesForTokens = (tokens: number): number => tokens * APPROX_BYTES_PER_TOKEN
+
+/**
+ * Approximate tokens represented by a byte count: ceil(bytes / 4).
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const approxTokensFromByteCount = (bytes: number): number =>
+  Math.floor((bytes + APPROX_BYTES_PER_TOKEN - 1) / APPROX_BYTES_PER_TOKEN)
+
+const formatTruncationMarker = (useTokens: boolean, removedCount: number): string =>
+  useTokens ? `…${removedCount} tokens truncated…` : `…${removedCount} chars truncated…`
+
+interface Split {
+  readonly removedChars: number
+  readonly before: string
+  readonly after: string
+}
+
+/** A UTF-8 continuation byte is `10xxxxxx`; every other byte starts a character. */
+const isContinuation = (byte: number): boolean => (byte & 0b1100_0000) === 0b1000_0000
+
+/** The last character boundary at or before `offset`. */
+const boundaryAtOrBefore = (bytes: Uint8Array, offset: number): number => {
+  let at = Math.min(offset, bytes.byteLength)
+  while (at > 0 && at < bytes.byteLength && isContinuation(bytes[at]!)) at--
+  return at
+}
+
+/** The first character boundary at or after `offset`. */
+const boundaryAtOrAfter = (bytes: Uint8Array, offset: number): number => {
+  let at = Math.max(offset, 0)
+  while (at < bytes.byteLength && isContinuation(bytes[at]!)) at++
+  return at
+}
+
+/**
+ * Splits `s` into the head that fits `beginningBytes` and the tail that fits
+ * `endBytes`, both cut on UTF-8 character boundaries, and counts the characters
+ * dropped between them.
+ *
+ * The boundaries are read off the encoded bytes rather than by re-encoding one
+ * character at a time: a per-character `TextEncoder.encode` costs about two
+ * microseconds, which is nineteen seconds for the eight megabytes a bounded
+ * capture can hand this function.
+ */
+const splitString = (s: string, beginningBytes: number, endBytes: number): Split => {
+  const bytes = encoder.encode(s)
+  const tailStartTarget = Math.max(bytes.byteLength - endBytes, 0)
+  const prefixEnd = boundaryAtOrBefore(bytes, beginningBytes)
+  const suffixStart = boundaryAtOrAfter(bytes, Math.max(tailStartTarget, prefixEnd))
+  let removedChars = 0
+  for (let at = prefixEnd; at < suffixStart; at++) if (!isContinuation(bytes[at]!)) removedChars++
+  return {
+    removedChars,
+    before: decoder.decode(bytes.subarray(0, prefixEnd)),
+    after: decoder.decode(bytes.subarray(suffixStart))
+  }
+}
+
+const truncateWithByteEstimate = (s: string, maxBytes: number, useTokens: boolean): string => {
+  if (s === "") return ""
+  const totalBytes = byteLength(s)
+  if (maxBytes === 0) {
+    return formatTruncationMarker(useTokens, useTokens ? approxTokensFromByteCount(totalBytes) : [...s].length)
+  }
+  if (totalBytes <= maxBytes) return s
+  const leftBudget = Math.floor(maxBytes / 2)
+  const rightBudget = maxBytes - leftBudget
+  const { after, before, removedChars } = splitString(s, leftBudget, rightBudget)
+  const marker = formatTruncationMarker(
+    useTokens,
+    useTokens ? approxTokensFromByteCount(Math.max(totalBytes - maxBytes, 0)) : removedChars
+  )
+  return `${before}${marker}${after}`
+}
+
+/**
+ * Middle-truncate to a byte budget with a char-count marker.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const truncateMiddleChars = (s: string, maxBytes: number): string => truncateWithByteEstimate(s, maxBytes, false)
+
+/**
+ * Middle-truncate to an approximate token budget, preserving the beginning
+ * and end. Returns the possibly truncated string and the original token
+ * count when truncation occurred.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const truncateMiddleWithTokenBudget = (
+  s: string,
+  maxTokens: number
+): { readonly text: string; readonly originalTokenCount: number | undefined } => {
+  if (s === "") return { text: "", originalTokenCount: undefined }
+  if (maxTokens > 0 && byteLength(s) <= approxBytesForTokens(maxTokens)) {
+    return { text: s, originalTokenCount: undefined }
+  }
+  const truncated = truncateWithByteEstimate(s, approxBytesForTokens(maxTokens), true)
+  if (truncated === s) return { text: s, originalTokenCount: undefined }
+  return { text: truncated, originalTokenCount: approxTokenCount(s) }
+}
+
+const countLines = (content: string): number => {
+  if (content === "") return 0
+  const parts = content.split("\n")
+  return parts[parts.length - 1] === "" ? parts.length - 1 : parts.length
+}
+
+/**
+ * Codex `format_exec_output_for_model`: exit code, wall time, optional line
+ * count disclosure, then the (possibly middle-truncated) output.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const formatExecOutputForModel = (options: {
+  readonly output: string
+  readonly exitCode: number
+  readonly durationSeconds: number
+  readonly maxOutputTokens: number
+  readonly timedOutAfterMs?: number | undefined
+}): string => {
+  const content = options.timedOutAfterMs === undefined
+    ? options.output
+    : `command timed out after ${options.timedOutAfterMs} milliseconds\n${options.output}`
+  const durationSeconds = Math.round(options.durationSeconds * 10) / 10
+  const totalLines = countLines(content)
+  const formatted = truncateMiddleWithTokenBudget(content, options.maxOutputTokens).text
+  const sections = [
+    `Exit code: ${options.exitCode}`,
+    `Wall time: ${durationSeconds} seconds`
+  ]
+  if (totalLines !== countLines(formatted)) sections.push(`Total output lines: ${totalLines}`)
+  sections.push("Output:")
+  sections.push(formatted)
+  return sections.join("\n")
+}

package/src/internal/Declaration.ts

@@ -0,0 +1,60 @@
+/**
+ * Shared declaration helpers for the standard flows.
+ *
+ * @since 1.0.0
+ */
+
+import type * as Capability from "@smthrs/capability/Capability"
+import * as Effects from "@smthrs/core/Effects"
+
+/**
+ * Options accepted by {@link envelope}.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface EnvelopeOptions {
+  readonly tier: "sealed" | "compensable" | "irreversible"
+  readonly mode: "hermetic" | "expected"
+  readonly reads: Iterable<string>
+  readonly writes: Iterable<string>
+}
+
+/**
+ * Builds a deterministic effect declaration for a standard flow.
+ *
+ * Every standard flow serializes on conflict: the library owns no lane
+ * partitioning of its own, so the conservative choice is the safe one.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const envelope = (options: EnvelopeOptions): Effects.Declaration =>
+  Effects.make({
+    reads: options.reads,
+    writes: options.writes,
+    mode: options.mode,
+    onConflict: "serialize",
+    tier: options.tier
+  })
+
+/**
+ * Formats a capability request as the `action:resource` string carried on a
+ * flow declaration.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const capability = (action: Capability.Action, resource: string): string => `${action}:${resource}`
+
+/**
+ * The read glob covering everything under one search root.
+ *
+ * `glob` and `grep` narrow their conservative `/**` declaration to the root the
+ * call actually names, and a declaration that disagreed between the two would
+ * make the same search reserve different paths depending on which flow ran it.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const rootSubtree = (root: string): string => root === "/" ? "/**" : `${root.replace(/\/+$/, "")}/**`

package/src/internal/Diagnostics.ts

@@ -0,0 +1,134 @@
+/**
+ * Keeps a bound language server in step with the files the mutation flows
+ * write, and reads back the errors an edit leaves.
+ *
+ * The server is optional: a host that binds no `LanguageServer` edits files
+ * exactly as before, and a server that fails or times out never fails the
+ * write that already happened.
+ *
+ * @since 1.0.0
+ */
+
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import * as Schema from "effect/Schema"
+import * as LanguageServer from "../LanguageServer.ts"
+
+/**
+ * Most errors attached to one result.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const MAX_ERRORS = 20
+
+/**
+ * One error-severity diagnostic, at its 1-based position.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const Problem = Schema.Struct({
+  line: Schema.Number.annotate({ description: "1-based line" }),
+  character: Schema.Number.annotate({ description: "1-based character" }),
+  message: Schema.String
+})
+
+/**
+ * A decoded {@link Problem}.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export type Problem = typeof Problem.Type
+
+const record = (value: unknown): Readonly<Record<string, unknown>> | undefined =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+    ? value as Readonly<Record<string, unknown>>
+    : undefined
+
+/**
+ * The error-severity items of a document diagnostic report, 1-based.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const errors = (report: unknown): ReadonlyArray<Problem> => {
+  const items = record(report)?.items
+  if (!Array.isArray(items)) return []
+  const found: Array<Problem> = []
+  for (const item of items) {
+    const diagnostic = record(item)
+    const start = record(record(diagnostic?.range)?.start)
+    if (
+      diagnostic?.severity !== 1 || typeof diagnostic.message !== "string" ||
+      typeof start?.line !== "number" || typeof start.character !== "number"
+    ) continue
+    found.push({ line: start.line + 1, character: start.character + 1, message: diagnostic.message })
+    if (found.length === MAX_ERRORS) break
+  }
+  return found
+}
+
+/**
+ * Sends a written file's new text to the bound server, if any. Succeeds with
+ * the server that took it.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const sync = (path: string, text: string): Effect.Effect<LanguageServer.LanguageServer | undefined> =>
+  Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+    Effect.flatMap(Option.match({
+      onNone: () => Effect.succeed(undefined),
+      onSome: (server) =>
+        server.sync(path, text).pipe(
+          Effect.as(server),
+          Effect.orElseSucceed(() => undefined)
+        )
+    }))
+  )
+
+/**
+ * Tells the bound server, if any, that a file is gone.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const close = (path: string): Effect.Effect<void> =>
+  Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+    Effect.flatMap(Option.match({
+      onNone: () => Effect.void,
+      onSome: (server) => server.close(path).pipe(Effect.ignore)
+    }))
+  )
+
+/**
+ * Tells the bound server, if any, to re-read every file it has open, after
+ * something other than the mutation flows may have changed them.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const refresh: Effect.Effect<void> = Effect.serviceOption(LanguageServer.LanguageServer).pipe(
+  Effect.flatMap(Option.match({
+    onNone: () => Effect.void,
+    onSome: (server) => server.refresh.pipe(Effect.ignore)
+  }))
+)
+
+/**
+ * The errors `server` reports for `path`, or `undefined` when it could not
+ * answer.
+ *
+ * @category internal
+ * @since 1.0.0
+ */
+export const errorsOf = (
+  server: LanguageServer.LanguageServer,
+  path: string
+): Effect.Effect<ReadonlyArray<Problem> | undefined> =>
+  server.diagnostics(path).pipe(
+    Effect.map(errors),
+    Effect.orElseSucceed(() => undefined)
+  )

package/src/internal/EnvelopePrecheck.ts

@@ -0,0 +1,233 @@
+/**
+ * The hermetic pre-check: a lexical scan of shell text for the paths it names,
+ * and the refusal when one of them falls outside the declared envelope.
+ *
+ * The scan is fail-closed over explicit path tokens and nothing more. It
+ * cannot prove confinement — a shell can name a path no lexer sees, through a
+ * variable, a substitution, or a program that opens a file of its own — so a
+ * kernel or host sandbox must eventually enforce and report the complete read
+ * and write sets.
+ *
+ * It lives beside the flow rather than inside it because it is its own
+ * concept, with its own vocabulary (Access, PathReference) and its own
+ * security history: three independent bypasses, each sufficient on its own to
+ * defeat it, are closed here.
+ *
+ * @since 1.0.0
+ */
+
+import type * as Path from "@smthrs/kernel/Path"
+import * as StdError from "../StdError.ts"
+import { withinEnvelope } from "./Paths.ts"
+
+/**
+ * The filesystem envelope one invocation declared.
+ *
+ * Structural rather than the hermetic `Bash.Input` itself: the scan needs the
+ * three fields and none of the rest, and depending on the flow module would
+ * make the flow and its pre-check import each other.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Envelope {
+  readonly cwd?: string | undefined
+  readonly reads: ReadonlyArray<string>
+  readonly writes: ReadonlyArray<string>
+}
+
+/**
+ * How a command touches a path it names.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export type Access = "read" | "write"
+
+/**
+ * One path a command names, and how it touches it.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface PathReference {
+  readonly access: Access
+  readonly value: string
+}
+
+const commandSeparators = new Set(["&&", "||", ";", "|", "&", "\n"])
+const redirections = new Set([">", ">>", "<", "<<"])
+const writeCommands = new Set(["chmod", "chown", "mkdir", "rm", "rmdir", "tee", "touch", "truncate", "unlink"])
+const destinationCommands = new Set(["cp", "install", "mv"])
+const prefixWrappers = new Set(["env", "nice", "time", "sudo", "nohup", "xargs", "command", "exec"])
+
+// Shell spaces may disappear, but a physical line ends one command. Retaining
+// LF prevents a command on line one from classifying every later line's paths.
+//
+// A word is one or more runs of bare characters and quoted sections, not a
+// bare run alone: `FOO="a b" rm /work/target` used to tokenize as `FOO="a`,
+// `b"`, `rm`, `/work/target`, and since only the first token carried an `=`
+// the command was read as `b"` rather than `rm` — so the delete was
+// classified as a read and a hermetic call with `writes: []` spawned. The
+// quoted alternatives come first inside the group so a quoted span is taken
+// whole; the bare class excludes quotes so the branches cannot overlap.
+const tokenize = (command: string): ReadonlyArray<string> =>
+  command.match(/\n|>>|<<|&&|\|\||[<>;|&]|(?:"(?:\\.|[^"\\])*"|'[^']*'|[^\s<>;|&"']+)+/g) ?? []
+
+const unquote = (token: string): string => {
+  const first = token[0]
+  const last = token[token.length - 1]
+  return token.length >= 2 && (first === "'" || first === "\"") && last === first
+    ? token.slice(1, -1)
+    : token
+}
+
+const pathValue = (token: string): string =>
+  unquote(token)
+    .replace(/^\$\(+/, "")
+    .replace(/[),]+$/, "")
+
+const isPathToken = (path: Path.Path, token: string): boolean =>
+  token === "." ||
+  token === ".." ||
+  token.startsWith("./") ||
+  token.startsWith("../") ||
+  path.isAbsolute(token) ||
+  /^[A-Za-z]:[\\/]/.test(token) ||
+  token.includes("/")
+
+const segmentReferences = (path: Path.Path, tokens: ReadonlyArray<string>): ReadonlyArray<PathReference> => {
+  const commentIndex = tokens.findIndex((token) => token.startsWith("#"))
+  const active = commentIndex === -1 ? tokens : tokens.slice(0, commentIndex)
+  const hasWrapper = active.some((token) => prefixWrappers.has(path.basename(pathValue(token))))
+  let commandIndex = active.findIndex((token) =>
+    !redirections.has(token) &&
+    !token.includes("=") &&
+    !prefixWrappers.has(path.basename(pathValue(token))) &&
+    !token.startsWith("-") &&
+    token !== ""
+  )
+  if (hasWrapper) {
+    // Wrapper options can carry their own values. Prefer a later known
+    // mutating command so an option value cannot lend it a read classification.
+    const mutatingIndex = active.findIndex((token) => {
+      const command = path.basename(pathValue(token))
+      return writeCommands.has(command) || destinationCommands.has(command)
+    })
+    if (mutatingIndex !== -1) commandIndex = mutatingIndex
+  }
+  if (commandIndex === -1) return []
+
+  const command = path.basename(pathValue(active[commandIndex]!))
+  const candidates: Array<{ readonly index: number; readonly value: string }> = []
+  for (let index = 0; index < active.length; index++) {
+    const token = active[index]!
+    if (index === commandIndex || redirections.has(token) || token.startsWith("-")) continue
+    const value = pathValue(token)
+    if (value !== "" && isPathToken(path, value)) candidates.push({ index, value })
+  }
+
+  const destinationIndex = destinationCommands.has(command)
+    ? candidates[candidates.length - 1]?.index
+    : undefined
+
+  return candidates.map(({ index, value }) => {
+    const previous = active[index - 1]
+    const access: Access = previous === ">" || previous === ">>" ||
+        writeCommands.has(command) || destinationIndex === index
+      ? "write"
+      : "read"
+    return { access, value }
+  })
+}
+
+/**
+ * Every path token the shell text names, one segment at a time.
+ *
+ * @category pre-check
+ * @since 1.0.0
+ */
+export const commandReferences = (path: Path.Path, command: string): ReadonlyArray<PathReference> => {
+  const references: Array<PathReference> = []
+  let segment: Array<string> = []
+  for (const token of tokenize(command)) {
+    if (commandSeparators.has(token)) {
+      references.push(...segmentReferences(path, segment))
+      segment = []
+    } else {
+      segment.push(token)
+    }
+  }
+  references.push(...segmentReferences(path, segment))
+  return references
+}
+
+const resolvePath = (
+  path: Path.Path,
+  cwd: string,
+  value: string
+): string => path.isAbsolute(value) ? path.normalize(value) : path.resolve(cwd, value)
+
+const isDeclared = (
+  path: Path.Path,
+  cwd: string,
+  declared: ReadonlyArray<string>,
+  source: string
+): boolean =>
+  withinEnvelope(
+    declared.map((entry) => resolvePath(path, cwd, entry)),
+    resolvePath(path, cwd, source)
+  )
+
+/**
+ * The refusal shell text earns by naming a path outside its declared
+ * envelope, or `undefined` when every path it names is declared.
+ *
+ * @category pre-check
+ * @since 1.0.0
+ */
+export const outsideEnvelope = (
+  input: Envelope,
+  text: string,
+  path: Path.Path
+): StdError.StdError | undefined => {
+  const base = path.resolve(".")
+  const cwd = path.resolve(input.cwd ?? ".")
+  // Passing the default explicitly is not a declaration the caller owes. A
+  // working directory is where the command runs, not a file it reads, and
+  // `cwd: "."` names exactly what omitting `cwd` names — so refusing it asked
+  // the caller to list the workspace root among its reads to say nothing at
+  // all. Agents hit this constantly: one SWE-bench run spent ten of its
+  // forty-eight tool calls re-issuing the same command after
+  // "Working directory is outside declared reads: <the workspace root>". A
+  // cwd that points somewhere else is still declared or refused.
+  if (input.cwd !== undefined && cwd !== base && !isDeclared(path, cwd, input.reads, cwd)) {
+    return new StdError.StdError({
+      code: "outside_declared_reads",
+      message: `Working directory is outside declared reads: ${cwd}`,
+      path: cwd
+    })
+  }
+
+  // The Shell contract currently has no sandbox or access-reporting surface.
+  // This lexical scan is intentionally only a fail-closed pre-check for
+  // explicit path tokens. It cannot prove confinement; a kernel/host sandbox
+  // must eventually enforce and report the complete read and write sets.
+  for (const reference of commandReferences(path, text)) {
+    const resolved = resolvePath(path, cwd, reference.value)
+    // /dev/* is the process plumbing every shell one-liner leans on:
+    // `2>/dev/null`, `cat /dev/stdin`. Normalize first so a dot-dot segment
+    // cannot disguise an ordinary filesystem path as process plumbing.
+    if (resolved.startsWith("/dev/")) continue
+    const declared = reference.access === "write" ? input.writes : input.reads
+    if (!isDeclared(path, cwd, declared, resolved)) {
+      const code = reference.access === "write" ? "outside_declared_writes" : "outside_declared_reads"
+      return new StdError.StdError({
+        code,
+        message: `Command path is outside declared ${reference.access}s: ${resolved}`,
+        path: resolved
+      })
+    }
+  }
+  return undefined
+}