@smthrs/std 0.0.0-stage → 1.0.0-rc.3

package/src/PortableSearch.ts

@@ -0,0 +1,264 @@
+/**
+ * The in-process implementation of the ripgrep search contract.
+ *
+ * @since 1.0.0
+ */
+
+import * as Path from "@smthrs/kernel/Path"
+import { type Context, Effect, Layer, Stream } from "effect"
+import * as FileSystem from "effect/FileSystem"
+import * as Grouping from "./internal/Grouping.ts"
+import * as LinearRegex from "./internal/LinearRegex.ts"
+import { escapeRegex } from "./internal/SearchContract.ts"
+import { notice, truncateBytes } from "./internal/Text.ts"
+import * as Walk from "./internal/Walk.ts"
+import * as Search from "./Search.ts"
+import * as Contract from "./SearchContract.ts"
+import * as StdError from "./StdError.ts"
+
+/**
+ * Narrows a walk to the files a search reports.
+ *
+ * A root that names one file is that file: `rg` searches a path given on the
+ * command line whatever `-g` says, and follows it even when it is a symlink.
+ * Everything a walk found is filtered by the globs first and probed for
+ * symlinks second, so the probe runs over the far smaller included set.
+ */
+const candidates = (
+  fileSystem: FileSystem.FileSystem,
+  path: Path.Path,
+  walked: Walk.Walked,
+  root: string,
+  globs: ReadonlyArray<string>
+): Effect.Effect<ReadonlyArray<string>> =>
+  Effect.gen(function*() {
+    if (walked.explicitFile) return walked.files
+    const included = walked.files.filter((file) =>
+      Contract.includedByGlobs(globs, Walk.relativeForPattern(path, root, file), path.basename(file))
+    )
+    const links = yield* Walk.symbolicLinks(fileSystem, included)
+    return included.filter((_, index) => links[index] !== true)
+  })
+
+const preview = (line: string): string => {
+  let head = ""
+  let length = 0
+  for (const character of line) {
+    if (length++ === 500) break
+    head += character
+  }
+  return truncateBytes(head, 500, { keep: "head" }).text
+}
+
+const grep = (
+  input: Search.GrepInput
+): Effect.Effect<Search.GrepOutput, StdError.StdError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fileSystem = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const includedByGlobs = (relative: string, basename: string) =>
+      Contract.includedByGlobs(input.globs, relative, basename)
+    const walked = yield* Walk.files(fileSystem, path, input.root, input.hidden, input.noIgnore, includedByGlobs)
+    const insensitive = input.ignoreCase || (input.smartCase && !/[A-Z]/.test(input.pattern))
+    const regex = LinearRegex.compile(
+      input.fixedStrings ? escapeRegex(input.pattern) : input.pattern,
+      insensitive
+    )
+    const shownMatches: Array<Search.GrepMatch> = []
+    const shownFiles: Array<string> = []
+    let total = 0
+    let filesSearched = 0
+    let skippedBinary = 0
+
+    const included = yield* candidates(fileSystem, path, walked, input.root, input.globs)
+    for (const file of included) {
+      // Count every admitted file, including unreadable files and binaries.
+      filesSearched++
+      const remaining = input.filesWithMatches ? 0 : input.limit - shownMatches.length
+      const matched: Array<number> = []
+      const selected = new Map<number, Search.GrepLine>()
+      const recent = new Map<number, Search.GrepLine>()
+      let count = 0
+      let lineNumber = 0
+      let through = 0
+      const scanLine = (text: string) =>
+        Effect.gen(function*() {
+          lineNumber++
+          let matches = false
+          if (input.maxCount === undefined || count < input.maxCount) {
+            const evaluation = regex.test(text)
+            let step = evaluation.next()
+            while (!step.done) {
+              yield* Effect.yieldNow
+              step = evaluation.next()
+            }
+            matches = step.value
+          }
+          if (matches) count++
+          if (remaining <= 0) return
+          // The first overflow hit is a boundary: context nearest to it must
+          // not migrate to the last retained hit when the limit is applied.
+          const retain = matches && matched.length < remaining + 1
+          const awaiting = matched.length < remaining + 1
+          if (retain || lineNumber <= through || (awaiting && input.beforeContext > 0)) {
+            const row: Search.GrepLine = {
+              file,
+              line: lineNumber,
+              text: preview(text),
+              kind: retain ? "match" : "context"
+            }
+            if (retain) {
+              matched.push(lineNumber)
+              for (const [number, context] of recent) selected.set(number, context)
+              through = lineNumber + input.afterContext
+            }
+            if (retain || lineNumber <= through) selected.set(lineNumber, row)
+            if (awaiting && input.beforeContext > 0) recent.set(lineNumber, row)
+          }
+          recent.delete(lineNumber - input.beforeContext)
+          if (matched.length >= remaining + 1) recent.clear()
+        })
+      // Split only on LF (stripping a preceding CR), just like sourceLines.
+      // Stream.splitLines also treats a standalone CR as a delimiter.
+      const decoder = new TextDecoder()
+      let fragments: Array<string> = []
+      const consume = (chunk: string) =>
+        Effect.gen(function*() {
+          let from = 0
+          let newline = chunk.indexOf("\n")
+          while (newline >= 0) {
+            fragments.push(chunk.slice(from, newline))
+            const line = fragments.join("")
+            fragments = []
+            yield* scanLine(line.endsWith("\r") ? line.slice(0, -1) : line)
+            from = newline + 1
+            newline = chunk.indexOf("\n", from)
+          }
+          if (from < chunk.length) fragments.push(chunk.slice(from))
+        })
+      const scanned = yield* Effect.gen(function*() {
+        let started = false
+        // A guarded disk host can safely expose atomic reads while refusing
+        // open streaming handles. Retry through that same filesystem service
+        // only before any bytes arrive, so a failed partial stream is never
+        // combined with a second reading of a file that may have changed.
+        yield* fileSystem.stream(file).pipe(
+          Stream.catch((error) => started ? Stream.fail(error) : Stream.fromEffect(fileSystem.readFile(file))),
+          Stream.runForEach((bytes) => {
+            started = true
+            if (bytes.includes(0)) {
+              return Effect.fail(
+                new StdError.StdError({
+                  code: "binary_file",
+                  message: `Cannot search binary file: ${file}`,
+                  path: file
+                })
+              )
+            }
+            return consume(decoder.decode(bytes, { stream: true }))
+          })
+        )
+        yield* consume(decoder.decode())
+        if (fragments.length > 0) yield* scanLine(fragments.join(""))
+        return true
+      }).pipe(Effect.catch((error) => {
+        if (error instanceof StdError.StdError) {
+          if (walked.explicitFile) return Effect.fail(error)
+          skippedBinary++
+        }
+        return Effect.succeed(false)
+      }))
+      if (!scanned || count === 0) continue
+      total += input.filesWithMatches ? 1 : count
+      if (input.filesWithMatches) {
+        if (shownFiles.length < input.limit) shownFiles.push(file)
+        continue
+      }
+      if (matched.length === 0) continue
+      const grouped = Grouping.group([...selected.values()].sort((left, right) => left.line - right.line))
+        .slice(0, remaining)
+      // Source is needed only for returned symbols, and is released before
+      // moving to the next file. Overflow files never load symbol source.
+      const contents = new Map<string, ReadonlyArray<string>>()
+      if (input.symbols) {
+        const source = yield* fileSystem.readFileString(file).pipe(Effect.orElseSucceed(() => undefined))
+        if (source !== undefined) contents.set(file, Grouping.sourceLines(source))
+      }
+      shownMatches.push(...Grouping.annotate(grouped, contents))
+    }
+
+    const truncated = total > input.limit
+    const unsatisfiable = total > 0 ? undefined : yield* Contract.unsatisfiableNotice({
+      fileSystem,
+      path,
+      root: input.root,
+      globs: input.globs,
+      hidden: input.hidden,
+      noIgnore: input.noIgnore,
+      ignored: walked.ignored
+    })
+    return {
+      matches: shownMatches,
+      files: shownFiles,
+      filesSearched,
+      skippedBinary,
+      truncated,
+      ...(truncated
+        ? { notice: notice(input.filesWithMatches ? "files" : "matches", input.limit, total) }
+        : {}),
+      ...(unsatisfiable === undefined ? {} : { notice: unsatisfiable })
+    }
+  })
+
+const glob = (
+  input: Search.GlobInput
+): Effect.Effect<Search.GlobOutput, StdError.StdError, FileSystem.FileSystem | Path.Path> =>
+  Effect.gen(function*() {
+    const fileSystem = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const includedByGlobs = (relative: string, basename: string) =>
+      Contract.includedByGlobs([input.pattern], relative, basename)
+    const walked = yield* Walk.files(fileSystem, path, input.root, input.hidden, input.noIgnore, includedByGlobs)
+    const included = yield* candidates(fileSystem, path, walked, input.root, [input.pattern])
+    const matching = [...included].sort()
+    const paths = matching.slice(0, input.limit)
+    const unsatisfiable = matching.length > 0 ? undefined : yield* Contract.unsatisfiableNotice({
+      fileSystem,
+      path,
+      root: input.root,
+      globs: [input.pattern],
+      hidden: input.hidden,
+      noIgnore: input.noIgnore,
+      ignored: walked.ignored
+    })
+    return {
+      paths,
+      total: matching.length,
+      truncated: matching.length > input.limit,
+      ...(matching.length > input.limit ? { notice: notice("entries", paths.length, matching.length) } : {}),
+      ...(unsatisfiable === undefined ? {} : { notice: unsatisfiable })
+    }
+  })
+
+/**
+ * Captures filesystem and path services in the portable peer.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const make = (services: Context.Context<FileSystem.FileSystem | Path.Path>): Search.Search =>
+  Search.make({
+    grep: (input) => Effect.provide(grep(input), services),
+    glob: (input) => Effect.provide(glob(input), services)
+  })
+
+/**
+ * Provides the in-process peer.
+ *
+ * @category layers
+ * @since 1.0.0
+ */
+export const layer: Layer.Layer<Search.Search, never, FileSystem.FileSystem | Path.Path> = Layer.effect(
+  Search.Search,
+  Effect.map(Effect.context<FileSystem.FileSystem | Path.Path>(), make)
+)

package/src/Probe.ts

@@ -0,0 +1,284 @@
+/**
+ * Telling an invalid probe from a failing check.
+ *
+ * A non-zero exit is a runner's verdict about the code it ran. It is not the
+ * runner's verdict about the command it was handed. When an invocation names a
+ * test, a file, a module, an environment, or a program that does not exist, the
+ * runner never reaches any code and still exits non-zero — and at the tool
+ * boundary that is indistinguishable from the bug reproducing. A SWE-bench run
+ * lost thirty-nine frames to exactly that: the reproduction named a test method
+ * the class does not define, so it read `exitCode: 1` before the edit and
+ * `exitCode: 1` after it, and the agent could not tell it had already won.
+ *
+ * This module names that class. The taxonomy is one idea — *the invocation
+ * named something that does not exist* — and {@link Reason} enumerates the five
+ * things that something can be. An enumerated answer over prose is a judgment,
+ * so Jev makes it: {@link classify} asks the `probe/attribution` classifier
+ * whether this exit describes the tree or one of the five, and whether a runner
+ * ran anything at all. Nothing here reads the output itself. The judgment used
+ * to be nine regexes over runner wording, vetoed by two more over runner
+ * tallies, and every phrase they matched is also printed by genuine failures —
+ * a bug that *is* an import error, a test that asserts on a shell message — so
+ * precision cost recall and both cost maintenance.
+ *
+ * A zero exit needs no judgment: it belongs to the tree, whatever it printed.
+ * A non-zero exit, including 126 or 127, needs attribution. A runner can
+ * execute its tests and return either status, so neither proves that the shell
+ * refused to launch the check.
+ *
+ * There is no third path. When the judge does not answer, {@link unjudged}
+ * turns its failure into the caller's typed failure: a guess about whether a
+ * reproduction reproduced is worth less than nothing, and the caller can say
+ * that a result was not judged where it cannot say that a guess was wrong.
+ *
+ * A flow that can tell the difference reports it under the reserved output key
+ * {@link key}. The harness reads that key off an otherwise opaque call result
+ * and refuses to count such a result as evidence that anything about the tree
+ * changed, which is what stops a broken probe from being cited as a
+ * reproduction.
+ *
+ * @since 1.0.0
+ */
+
+import * as Classifier from "@smthrs/model/Classifier"
+import type * as Evaluator from "@smthrs/model/Evaluator"
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import * as Schema from "effect/Schema"
+import { truncateBytes } from "./internal/Text.ts"
+import * as StdError from "./StdError.ts"
+
+/**
+ * The reserved output key a flow reports an invalid probe under.
+ *
+ * Stated as a constant because it is a wire contract with the harness rather
+ * than a shared type: `@smthrs/harness` reads this key off a `Schema.Json`
+ * result and must not depend on the tool library to do it.
+ *
+ * @category identifiers
+ * @since 1.0.0
+ */
+export const key = "invalidProbe"
+
+/**
+ * What the invocation named that does not exist.
+ *
+ * Five members, and the classifier offers exactly these five beside `tree`.
+ *
+ * @category schemas
+ * @since 1.0.0
+ */
+export const Reason = Schema.Literals([
+  "unknown-command",
+  "unknown-test",
+  "unknown-path",
+  "unknown-module",
+  "unknown-environment"
+])
+
+/**
+ * What the invocation named that does not exist.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export type Reason = typeof Reason.Type
+
+/**
+ * One command's failure, attributed to the command rather than to the tree.
+ *
+ * @category schemas
+ * @since 1.0.0
+ */
+export const InvalidProbe = Schema.Struct({
+  reason: Reason.annotate({ description: "Which kind of name the command could not resolve" }),
+  evidence: Schema.String.annotate({
+    description: "What this attribution rests on: the shell's own exit code, or the judge's reading and how sure it is"
+  }),
+  message: Schema.String.annotate({ description: "What the failure does and does not prove, stated for the caller" })
+})
+
+/**
+ * One command's failure, attributed to the command rather than to the tree.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export type InvalidProbe = typeof InvalidProbe.Type
+
+/**
+ * What one command's exit code was attributed to.
+ *
+ * `to` is the tree, or the kind of name the command could not resolve, and
+ * `invalidProbe` is present in exactly the second case, ready to report under
+ * {@link key}. `executed` is the judge's separate reading of whether a runner
+ * ran any test, absent when nothing was asked.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Attribution {
+  readonly to: "tree" | Reason
+  readonly executed?: boolean
+  readonly invalidProbe?: InvalidProbe
+}
+
+/**
+ * The confidence an attribution needs before it outranks the tree.
+ *
+ * Below it the failure is the tree's, which is the reading that leaves the
+ * caller the output it would have had anyway. A missed invalid probe costs a
+ * reader nothing it had; a false one tells it its reproduction proved nothing.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const CONFIDENCE_FLOOR = 0.7
+
+/**
+ * The most output bytes one judgment carries, newest kept.
+ *
+ * A runner prints its refusal last, and the classifier's whole state is held
+ * to 32 KiB.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_OUTPUT_BYTES = 32 * 1024
+
+/** What each reason means, in the second half of one sentence. */
+const named: Record<Reason, string> = {
+  "unknown-command": "the shell could not find the program it was asked to run",
+  "unknown-test": "the test runner could not find the test that was named",
+  "unknown-path": "the file or directory that was named does not exist",
+  "unknown-module": "the module that was imported does not exist here",
+  "unknown-environment": "the runner has no environment by that name"
+}
+
+const explain = (reason: Reason): string =>
+  `This command never ran a check: ${
+    named[reason]
+  }. The non-zero exit describes the command, not the code under test, so it is not a reproduction and reads the same before and after a correct fix. Find the real names, repair the command, and run it again before drawing any conclusion from it.`
+
+const invalid = (reason: Reason, evidence: string): InvalidProbe => ({
+  reason,
+  evidence,
+  message: explain(reason)
+})
+
+/** How the judge's own reading is quoted back to the caller. */
+const judged = (reason: Reason, confidence: number, executed: boolean): string =>
+  `the judge read this output as ${reason} at confidence ${confidence.toFixed(2)}${
+    executed ? ", although the runner also reported that it ran tests" : ""
+  }`
+
+/** Which typed failure each way the judge can fail to answer is. */
+const unjudgedCode: Record<Evaluator.EvaluatorErrorCode, StdError.Code> = {
+  unconfigured: "provider_unavailable",
+  unreachable: "provider_unavailable",
+  refused: "provider_unavailable",
+  empty: "provider_unavailable",
+  timeout: "timeout",
+  invalid_answer: "request_failed",
+  invalid_question: "request_failed"
+}
+
+/**
+ * The standard-flow failure a judge that did not answer is.
+ *
+ * A handler that cannot say whether a failing command failed about itself has
+ * no result to return: reporting the exit code alone is the reading that lost
+ * the thirty-nine frames. So the call fails, the code names which way the
+ * gateway went wrong, and the message says what is missing and how a host
+ * supplies it.
+ *
+ * @category errors
+ * @since 1.0.0
+ */
+export const unjudged = (error: Classifier.ClassifierError): StdError.StdError =>
+  new StdError.StdError({
+    code: unjudgedCode[error.code],
+    message:
+      `This result was not judged: the attribution judge answered ${error.code}. ${error.message} Nothing here says whether the non-zero exit is the code's failure or the command's, and a guess at that is worth less than no answer. Connect a subscription seat for the evaluator, then run the command again.`
+  })
+
+/**
+ * What one failed command's exit code describes: the tree, or a name the
+ * command could not resolve.
+ *
+ * No cell calls this classifier. {@link classify} asks it on behalf of the
+ * `test` flow, which has the command and its output and has no task to judge
+ * them against, so the state is the three facts of the run and the questions
+ * carry every criterion themselves. It is declared here rather than in a
+ * catalog a host binds: a door for the model to ask this would be a door onto
+ * a judgment the flow has already made.
+ *
+ * @category classifiers
+ * @since 1.0.0
+ */
+export const probeAttribution = Classifier.make("probe/attribution", {
+  description:
+    "Judge one failed command: whether it ran the tests it named and failed about the code, or never resolved a name it was given.",
+  state: Schema.Struct({
+    command: Schema.String.annotate({ description: "The command that ran" }),
+    exitCode: Schema.Int.annotate({ description: "Its exit code, which is not zero" }),
+    output: Schema.String.annotate({ description: "Its captured stdout and stderr, newest bytes when clipped" })
+  }),
+  questions: {
+    attribution: Classifier.choice({
+      instructions: "What does this non-zero exit describe?",
+      criteria: {
+        tree: "the command ran the intended tests and the failure is the code's",
+        "unknown-command": "the shell could not find the program it was asked to run",
+        "unknown-test": "the test runner could not find the test that was named",
+        "unknown-path": "the file or directory that was named does not exist",
+        "unknown-module": "the module that was imported does not exist here",
+        "unknown-environment": "the runner has no environment by that name"
+      }
+    }),
+    executed: Classifier.boolean({
+      instructions: "Did a test runner actually run tests?",
+      criteria: {
+        true: "the runner reported its own tally of tests that ran, such as a count of passes or failures",
+        false: "nothing in the output reports that any test was executed"
+      }
+    })
+  }
+})
+
+/**
+ * Attributes one command result to the tree, or to a name the command could
+ * not resolve.
+ *
+ * A zero exit needs no judgment. Every non-zero exit, including 126 and 127,
+ * goes through the attribution classifier: a runner may choose either code
+ * after executing tests. An answer under {@link CONFIDENCE_FLOOR} is the tree's
+ * failure, reported with the `executed` reading beside it.
+ *
+ * Pass the text the caller will actually return, so the judgment is made on
+ * the output the reader can see.
+ *
+ * @category classification
+ * @since 1.0.0
+ */
+export const classify = Effect.fn("Probe.classify")(function*(result: {
+  readonly command: string
+  readonly exitCode: number
+  readonly output: string
+}): Effect.fn.Return<Attribution, Classifier.ClassifierError, Evaluator.Evaluator> {
+  if (result.exitCode === 0) return { to: "tree" }
+  const answers = yield* probeAttribution.evaluate({
+    command: result.command,
+    exitCode: result.exitCode,
+    output: truncateBytes(result.output, MAX_OUTPUT_BYTES, { keep: "tail" }).text
+  })
+  const executed = answers.executed.value
+  const decided = Option.getOrUndefined(Classifier.confident(answers.attribution, CONFIDENCE_FLOOR))
+  return decided === undefined || decided === "tree"
+    ? { to: "tree", executed }
+    : {
+      to: decided,
+      executed,
+      invalidProbe: invalid(decided, judged(decided, answers.attribution.confidence, executed))
+    }
+})