@smthrs/std 0.0.0-stage → 1.0.0-rc.3

package/src/internal/TestReport.ts

@@ -0,0 +1,194 @@
+/**
+ * What a test runner said, read out of what it printed.
+ *
+ * A suite's answer is two facts — how many passed, and which ones failed — and
+ * a runner spends twenty to thirty kilobytes of stdout saying them. Paying a
+ * model to read that is the expensive way to learn a number, and the measured
+ * program shows what it costs: sixteen frames on sphinx-8721 existed because a
+ * pre-existing failure was never isolated, and the failure *names* were in the
+ * output all along.
+ *
+ * The parsers below recognise the report shapes the standard runners print.
+ * Precision is the design constraint, exactly as in `Probe`: a reader claims a
+ * complete report only when the runner printed an authoritative completion
+ * statement and the observed outcomes agree with it. Pytest requires a tally,
+ * TAP requires a plan with every planned outcome, and unittest requires its
+ * `Ran` line plus an `OK` or `FAILED` summary. Incomplete captures still return
+ * the outcomes they contain with `parsed: false`. A wrong failure set is worse
+ * than no failure set, because attribution is built on it.
+ *
+ * @since 1.0.0
+ */
+
+/**
+ * One run's outcome as this module could read it.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Report {
+  readonly passed: number
+  readonly failed: ReadonlyArray<string>
+  readonly reportedFailed: number | undefined
+  readonly parsed: boolean
+}
+
+const unique = (ids: ReadonlyArray<string>): ReadonlyArray<string> => [...new Set(ids)]
+
+const collect = (text: string, pattern: RegExp): ReadonlyArray<string> => {
+  const found: Array<string> = []
+  for (const match of text.matchAll(pattern)) {
+    found.push(match[1]!)
+  }
+  return found
+}
+
+const count = (text: string, pattern: RegExp): number | undefined => {
+  const match = pattern.exec(text)
+  return match === null ? undefined : Number(match[1])
+}
+
+/**
+ * pytest, in every verbosity it is normally run at.
+ *
+ * The short summary (`-rA`, and the default on failure) prints one
+ * `FAILED path::id - reason` line per failure, and the tally line prints the
+ * authoritative counts. Verbose outcome lines preserve observations from an
+ * incomplete capture but cannot establish that the run completed.
+ */
+const pytest = (text: string): Report | undefined => {
+  const summaryFailed = collect(text, /^(?:FAILED|ERROR)[ \t]+(.+?)(?:[ \t]+-[ \t]+.*)?$/gm)
+  const outcomes = [...text.matchAll(/^(.+?)[ \t]+(PASSED|FAILED|ERROR)(?:[ \t]+.*)?$/gm)]
+  const verboseFailed = outcomes.flatMap((match) => match[2] === "FAILED" || match[2] === "ERROR" ? [match[1]!] : [])
+  const verbosePassed = unique(outcomes.flatMap((match) => match[2] === "PASSED" ? [match[1]!] : [])).length
+  const failed = unique([...summaryFailed, ...verboseFailed])
+  // Only a terminal pytest summary can certify completion. Earlier diagnostic
+  // lines (including test stdout) are not runner tallies.
+  const lastLine = text.trimEnd().split("\n").at(-1)!
+  const summary = /^(?:=+[ \t]*)?(\d+ [a-z]+(?:,[ \t]*\d+ [a-z]+)*) in \d+(?:\.\d+)?s(?: \([^)]*\))?[ \t]*(?:=+)?$/
+    .exec(lastLine)?.[1]
+  const tally = summary === undefined ? undefined : count(summary, /\b(\d+) passed\b/)
+  const failures = summary === undefined ? undefined : count(summary, /\b(\d+) failed\b/)
+  const errors = summary === undefined ? undefined : count(summary, /\b(\d+) errors?\b/)
+  const hasTally = tally !== undefined || failures !== undefined || errors !== undefined
+  const reportedFailed = hasTally ? (failures ?? 0) + (errors ?? 0) : undefined
+  if (!hasTally && outcomes.length === 0 && summaryFailed.length === 0) return undefined
+  return {
+    passed: tally ?? verbosePassed,
+    failed,
+    reportedFailed,
+    parsed: hasTally && failed.length === reportedFailed
+  }
+}
+
+/**
+ * unittest, whose failures name a method and a class in the opposite order to
+ * the dotted id everything else uses.
+ */
+const unittest = (text: string): Report | undefined => {
+  const ran = count(text, /^Ran (\d+) tests?\b/m)
+  const outcomes = [...text.matchAll(/^(?:FAIL|ERROR|UNEXPECTED SUCCESS):[ \t]+(\S+)[ \t]+\(([^)\s]+)\)/gm)]
+  if (ran === undefined && outcomes.length === 0) return undefined
+  const failed = unique(
+    outcomes.map((match) => match[2]!.endsWith(`.${match[1]}`) ? match[2]! : `${match[2]}.${match[1]}`)
+  )
+  const summary = /^(OK|FAILED)(?: \(([^)]*)\))?[ \t]*$/m.exec(text)
+  const counts = new Map<string, number>()
+  let known = summary !== null
+  for (const entry of summary?.[2]?.split(", ") ?? []) {
+    const match = /^(failures|errors|skipped|expected failures|unexpected successes)=(\d+)$/.exec(entry)
+    if (match === null || counts.has(match[1]!)) known = false
+    else counts.set(match[1]!, Number(match[2]))
+  }
+  const reportedFailed = known
+    ? (counts.get("failures") ?? 0) + (counts.get("errors") ?? 0) + (counts.get("unexpected successes") ?? 0)
+    : undefined
+  const skipped = counts.get("skipped") ?? 0
+  const expectedFailures = counts.get("expected failures") ?? 0
+  const passed = ran === undefined || reportedFailed === undefined
+    ? 0
+    : Math.max(0, ran - reportedFailed - skipped - expectedFailures)
+  return {
+    passed,
+    failed,
+    reportedFailed,
+    parsed: ran !== undefined && reportedFailed !== undefined && failed.length === reportedFailed &&
+      (summary?.[1] === "OK" ? reportedFailed === 0 : reportedFailed > 0) &&
+      passed + failed.length + skipped + expectedFailures === ran
+  }
+}
+
+/**
+ * TAP, which several ecosystems emit and which names each test on its own line.
+ */
+const tap = (text: string): Report | undefined => {
+  const outcomes = [...text.matchAll(/^(not ok|ok)\b[ \t]*\d*[ \t]*-?[ \t]*(.*)$/gm)].map((match) => {
+    const directive = /[ \t]*#[ \t]*(SKIP|TODO)\b.*$/i.exec(match[2]!)
+    return {
+      ok: match[1] === "ok",
+      id: match[2]!.slice(0, directive?.index).trim(),
+      directive: directive === null ? undefined : directive[1]!.toUpperCase()
+    }
+  })
+  const failures = outcomes.filter((outcome) => !outcome.ok && outcome.directive === undefined)
+  const failed = unique(failures.flatMap((outcome) => outcome.id === "" ? [] : [outcome.id]))
+  const passed = outcomes.filter((outcome) => outcome.ok && outcome.directive !== "SKIP").length
+  const planned = count(text, /^1\.\.(\d+)\b/m)
+  if (outcomes.length === 0 && planned === undefined) return undefined
+  const reportedFailed = failures.length
+  return {
+    passed,
+    failed,
+    reportedFailed,
+    parsed: planned !== undefined && outcomes.length === planned && failed.length === reportedFailed
+  }
+}
+
+/**
+ * Reads one run's report, or says plainly that it could not.
+ *
+ * @category parsing
+ * @since 1.0.0
+ */
+export const parse = (text: string): Report => {
+  const hasTap = /^(?:TAP version \d+|1\.\.\d+|not ok\b|ok\b)/m.test(text)
+  const hasUnittest = /^Ran \d+ tests?\b/m.test(text)
+  const lastLine = text.trimEnd().split("\n").at(-1)!
+  const hasPytest = /^(?:=+[ \t]*)?\d+ [a-z]+(?:,[ \t]*\d+ [a-z]+)* in \d+(?:\.\d+)?s(?: \([^)]*\))?[ \t]*(?:=+)?$/
+    .test(lastLine)
+  // Two runner completion signatures in one capture cannot establish which
+  // protocol owns the outcomes. Never attribute such a report.
+  if ([hasTap, hasUnittest, hasPytest].filter(Boolean).length > 1) {
+    return { passed: 0, failed: [], reportedFailed: undefined, parsed: false }
+  }
+  const readers = hasTap ? [tap] : hasUnittest ? [unittest] : hasPytest ? [pytest] : [unittest, pytest, tap]
+  for (const reader of readers) {
+    const report = reader(text)
+    if (report !== undefined) return report
+  }
+  return { passed: 0, failed: [], reportedFailed: undefined, parsed: false }
+}
+
+/**
+ * How two runs of the same command differ, which is the whole of attribution.
+ *
+ * @category parsing
+ * @since 1.0.0
+ */
+export const attribute = (
+  current: Report,
+  base: Report
+): {
+  readonly introduced: ReadonlyArray<string>
+  readonly preexisting: ReadonlyArray<string>
+  readonly fixed: ReadonlyArray<string>
+} | undefined => {
+  if (!current.parsed || !base.parsed) return undefined
+  const before = new Set(base.failed)
+  const now = new Set(current.failed)
+  return {
+    introduced: current.failed.filter((id) => !before.has(id)),
+    preexisting: current.failed.filter((id) => before.has(id)),
+    fixed: base.failed.filter((id) => !now.has(id))
+  }
+}

package/src/internal/Text.ts

@@ -0,0 +1,170 @@
+/**
+ * Shared text paging, numbering, and byte-budget helpers.
+ *
+ * Every limit here is a display budget, not a policy limit: exceeding one is
+ * always disclosed to the model rather than silently applied.
+ *
+ * @since 1.0.0
+ */
+
+/**
+ * Default number of lines or entries returned by a paged flow.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const DEFAULT_READ_LIMIT = 2_000
+
+/**
+ * Maximum number of directory entries or glob paths returned in one call.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_ENTRIES = 1_000
+
+/**
+ * Maximum number of grep matches returned in one call.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_GREP_MATCHES = 200
+
+/**
+ * Maximum number of Unicode scalar values displayed for a single line.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_LINE_CHARS = 2_000
+
+/**
+ * Maximum UTF-8 byte budget for one rendered text payload.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_OUTPUT_BYTES = 60_000
+
+/**
+ * Maximum UTF-8 byte budget captured per shell output stream.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const MAX_SHELL_OUTPUT_BYTES = 30_000
+
+/**
+ * A byte-budgeted rendering of a text value.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Truncation {
+  readonly text: string
+  readonly truncated: boolean
+  readonly keptBytes: number
+  readonly droppedBytes: number
+}
+
+/**
+ * One page of source lines.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Page {
+  readonly lines: ReadonlyArray<string>
+  readonly startLine: number
+  readonly endLine: number
+  readonly totalLines: number
+}
+
+const encoder = new TextEncoder()
+const fatalDecoder = new TextDecoder("utf-8", { fatal: true })
+
+/**
+ * Trims a string to a UTF-8 byte budget without splitting a code point.
+ *
+ * `keep: "head"` retains the beginning of the value; `keep: "tail"` retains
+ * the end, which is what shell output needs because the interesting part of an
+ * overflowing stream is almost always its last lines.
+ *
+ * @category rendering
+ * @since 1.0.0
+ */
+export const truncateBytes = (
+  text: string,
+  maxBytes: number,
+  options: { readonly keep: "head" | "tail" }
+): Truncation => {
+  const bytes = encoder.encode(text)
+  if (bytes.byteLength <= maxBytes) {
+    return { text, truncated: false, keptBytes: bytes.byteLength, droppedBytes: 0 }
+  }
+  let start = options.keep === "head" ? 0 : bytes.byteLength - maxBytes
+  let end = options.keep === "head" ? maxBytes : bytes.byteLength
+  // TextEncoder produces valid UTF-8. Moving a cut inward reaches a scalar
+  // boundary after at most three continuation bytes, so decoding must succeed.
+  while (true) {
+    try {
+      const decoded = fatalDecoder.decode(bytes.subarray(start, end))
+      const keptBytes = end - start
+      return {
+        text: decoded,
+        truncated: true,
+        keptBytes,
+        droppedBytes: bytes.byteLength - keptBytes
+      }
+    } catch {
+      if (options.keep === "head") end--
+      else start++
+    }
+  }
+}
+
+/**
+ * Splits on LF without counting a terminal LF as a line.
+ * Carriage returns remain part of each line, including the CR in CRLF, so
+ * joining a page with LF preserves its bytes except for the final LF.
+ *
+ * @category parsing
+ * @since 1.0.0
+ */
+export const sourceLines = (text: string): ReadonlyArray<string> => {
+  const lines = text.length === 0 ? [] : text.split("\n")
+  if (text.endsWith("\n")) lines.pop()
+  return lines
+}
+
+/**
+ * Selects a 1-based page of lines from a text value.
+ *
+ * @category paging
+ * @since 1.0.0
+ */
+export const slice = (
+  text: string,
+  options: { readonly offset: number; readonly limit: number }
+): Page => {
+  const lines = sourceLines(text)
+  const totalLines = lines.length
+  const startLine = Math.max(1, options.offset)
+  const endLine = Math.min(totalLines, startLine + Math.max(0, options.limit) - 1)
+  return {
+    lines: lines.slice(startLine - 1, endLine),
+    startLine,
+    endLine,
+    totalLines
+  }
+}
+
+/**
+ * Renders the disclosure appended whenever output was capped.
+ *
+ * @category rendering
+ * @since 1.0.0
+ */
+export const notice = (unit: string, shown: number, total: number): string =>
+  `Showing ${shown} of ${total} ${unit}; output was truncated.`

package/src/internal/Url.ts

@@ -0,0 +1,23 @@
+/**
+ * Shared validation for outbound HTTP URLs.
+ *
+ * @since 1.0.0
+ */
+
+/**
+ * Parses an HTTP or HTTPS URL without admitting credentials or hostless URLs.
+ *
+ * @category parsing
+ * @since 1.0.0
+ */
+export const parseHttpUrl = (value: string, base?: URL): URL | undefined => {
+  if (value.trim() === "") return undefined
+  try {
+    const url = base === undefined ? new URL(value) : new URL(value, base)
+    if (url.protocol !== "http:" && url.protocol !== "https:") return undefined
+    if (url.hostname === "" || url.username !== "" || url.password !== "") return undefined
+    return url
+  } catch {
+    return undefined
+  }
+}

package/src/internal/Walk.ts

@@ -0,0 +1,176 @@
+/**
+ * The directory names a repository walk never descends into.
+ *
+ * `grep` and `glob` walk with the kernel filesystem, and every entry costs a
+ * guarded stat. A walk rooted at "." that descends into `.git` visits tens of
+ * thousands of object files no search wants: one such `grep` held a benchmark
+ * frame for its entire fifteen-minute ceiling. The skip set is the same
+ * convention ripgrep ships — version control, dependency trees, and caches —
+ * and an explicit root inside a skipped directory still walks, because
+ * skipping applies to descent, never to what the caller named.
+ *
+ * @since 1.0.0
+ */
+
+import type * as Path from "@smthrs/kernel/Path"
+import { Effect } from "effect"
+import type * as FileSystem from "effect/FileSystem"
+import type * as StdError from "../StdError.ts"
+import * as Ignore from "./Ignore.ts"
+import * as Prune from "./Prune.ts"
+import { rootFailure } from "./SearchContract.ts"
+
+/**
+ * Directory basenames excluded from recursive descent.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const skippedDirectories: ReadonlySet<string> = new Set(Prune.repositoryDirectories)
+
+/**
+ * How many filesystem questions one directory level asks at a time.
+ *
+ * A metadata call through the layer costs far more in fiber scheduling than in
+ * kernel time, so asking one entry at a time is what makes a walk slow: the
+ * same probe measured 28.8 µs sequentially and 6.0 µs at this width on the
+ * SWE-bench pytest tree. The bound keeps the file-descriptor and thread-pool
+ * pressure of a wide directory predictable.
+ */
+const concurrency = 16
+
+/**
+ * Separates pattern path segments with slashes while preserving POSIX filenames.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const relativeForPattern = (path: Path.Path, from: string, to: string): string => {
+  const relative = path.relative(from, to)
+  return path.sep === "\\" ? relative.replaceAll("\\", "/") : relative
+}
+
+/**
+ * Answers whether each path is a symbolic link, which neither peer follows.
+ *
+ * `FileSystem.stat` resolves links, so the only probe available is `readLink`,
+ * and every probe is one more call in a loop that already makes one per entry.
+ * The walk therefore probes directories, where following a link would duplicate
+ * a subtree or loop forever, and the callers probe the far smaller set of files
+ * they are about to report — batched, never one at a time.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const symbolicLinks = (
+  fileSystem: FileSystem.FileSystem,
+  candidates: ReadonlyArray<string>
+): Effect.Effect<ReadonlyArray<boolean>> =>
+  Effect.forEach(
+    candidates,
+    (candidate) => fileSystem.readLink(candidate).pipe(Effect.as(true), Effect.orElseSucceed(() => false)),
+    { concurrency }
+  )
+
+/**
+ * One walk: admitted files, explicit-root status, and ignore exclusions.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export interface Walked {
+  readonly ignored: boolean
+  readonly explicitFile: boolean
+  readonly files: ReadonlyArray<string>
+}
+
+/**
+ * Lists the files a search under `root` reaches.
+ *
+ * Only the root the caller named is allowed to fail the walk. Every entry
+ * below it that the process cannot inspect — a dangling symlink, a symlink
+ * loop, a directory it may not list — is skipped and the walk continues, which
+ * is what `rg --no-messages` does with the same tree. Turning one of those into
+ * a typed failure would make a whole repository unsearchable because of one
+ * link, and would answer differently from the native peer. Only .gitignore
+ * files within this root are read; ignored directories are never descended.
+ * `included` narrows the empty-result diagnostic without changing descent.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const files = (
+  fileSystem: FileSystem.FileSystem,
+  path: Path.Path,
+  root: string,
+  hidden: boolean,
+  noIgnore = false,
+  included: (relative: string, basename: string) => boolean = () => true
+): Effect.Effect<Walked, StdError.StdError> =>
+  Effect.gen(function*() {
+    const info = yield* fileSystem.stat(root).pipe(Effect.mapError((error) => rootFailure(root, error)))
+    if (info.type === "File") return { explicitFile: true, files: [path.normalize(root)], ignored: false }
+    const files: Array<string> = []
+    let excluded = false
+    const directories: Array<{ directory: string; scopes: ReadonlyArray<Ignore.Scope> }> = [{
+      directory: root,
+      scopes: []
+    }]
+    while (directories.length > 0) {
+      const next = directories.pop()!
+      const { directory } = next
+      let scopes = next.scopes
+      const children: ReadonlyArray<string> = yield* fileSystem.readDirectory(directory).pipe(
+        Effect.catch((error) =>
+          directory === root
+            ? Effect.fail(rootFailure(directory, error))
+            : Effect.succeed<ReadonlyArray<string>>([])
+        )
+      )
+      if (!noIgnore && children.includes(".gitignore")) {
+        const content = yield* fileSystem.readFileString(path.join(directory, ".gitignore")).pipe(
+          Effect.orElseSucceed(() => "")
+        )
+        const scope = Ignore.parse(directory, content)
+        if (scope.rules.length > 0) scopes = [...scopes, scope]
+      }
+      const candidates = children
+        .filter((child) => !skippedDirectories.has(child) && (hidden || !child.startsWith(".")))
+        .map((child) => path.join(directory, child))
+      const entries = yield* Effect.forEach(candidates, (candidate) =>
+        fileSystem.stat(candidate).pipe(
+          Effect.map((candidateInfo): { readonly candidate: string; readonly type: string | undefined } => ({
+            candidate,
+            type: candidateInfo.type
+          })),
+          Effect.orElseSucceed(() => ({ candidate, type: undefined }))
+        ), { concurrency })
+      const nested: Array<string> = []
+      for (const entry of entries) {
+        if (
+          entry.type !== undefined &&
+          Ignore.ignored(
+            scopes,
+            entry.candidate,
+            path.basename(entry.candidate),
+            entry.type === "Directory",
+            (from, to) => relativeForPattern(path, from, to)
+          )
+        ) {
+          if (
+            entry.type === "Directory" ||
+            included(relativeForPattern(path, root, entry.candidate), path.basename(entry.candidate))
+          ) excluded = true
+          continue
+        }
+        if (entry.type === "Directory") nested.push(entry.candidate)
+        else if (entry.type === "File") files.push(path.normalize(entry.candidate))
+      }
+      const links = yield* symbolicLinks(fileSystem, nested)
+      for (let index = 0; index < nested.length; index++) {
+        const candidate = nested[index]!
+        if (links[index] !== true) directories.push({ directory: candidate, scopes })
+      }
+    }
+    return { explicitFile: false, files: files.sort(), ignored: excluded }
+  })