@smthrs/std 0.0.0-stage → 1.0.0-rc.3

package/src/internal/Http.ts

@@ -0,0 +1,209 @@
+/**
+ * Shared HTTP response limits, error mapping, and request deadlines.
+ *
+ * @since 1.0.0
+ */
+
+import type { GrantStore } from "@smthrs/kernel/GrantStore"
+import * as HttpClient from "@smthrs/kernel/HttpClient"
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import * as Stream from "effect/Stream"
+import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
+import * as StdError from "../StdError.ts"
+import { guarded, refusal } from "./HttpNetwork.ts"
+import { MAX_OUTPUT_BYTES, notice, truncateBytes } from "./Text.ts"
+import { parseHttpUrl } from "./Url.ts"
+
+/**
+ * Maximum captured response bytes before decoding.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const MAX_RESPONSE_BYTES = 5 * 1024 * 1024
+
+interface BodyState {
+  readonly chunks: Array<Uint8Array>
+  size: number
+}
+
+/**
+ * Reads a response up to the shared byte limit and finalizes the stream on failure.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const readBounded = <E, R>(
+  stream: Stream.Stream<Uint8Array, E, R>,
+  url: string
+): Effect.Effect<Uint8Array, E | StdError.StdError, R> =>
+  Stream.runFoldEffect(
+    stream,
+    (): BodyState => ({ chunks: [], size: 0 }),
+    (state, chunk) => {
+      const size = state.size + chunk.byteLength
+      if (size > MAX_RESPONSE_BYTES) {
+        return Effect.fail(
+          new StdError.StdError({
+            code: "response_too_large",
+            message: "Response exceeds the 5 MiB limit",
+            path: url
+          })
+        )
+      }
+      state.chunks.push(chunk)
+      state.size = size
+      return Effect.succeed(state)
+    }
+  ).pipe(
+    Effect.map((state) => {
+      const output = new Uint8Array(state.size)
+      let offset = 0
+      for (const chunk of state.chunks) {
+        output.set(chunk, offset)
+        offset += chunk.byteLength
+      }
+      return output
+    })
+  )
+
+/**
+ * Maps transport failures while preserving standard HTTP failure codes.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const requestError = (url: string, error: unknown): StdError.StdError =>
+  error instanceof StdError.StdError ? error : refusal(error) ?? new StdError.StdError({
+    code: "request_failed",
+    message: `Request failed: ${url}${error instanceof Error ? ` (${error.message})` : ""}`
+  })
+
+/**
+ * Looks up a response header without depending on transport normalization.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const header = (headers: Readonly<Record<string, string | undefined>>, name: string): string | undefined => {
+  const lower = name.toLowerCase()
+  return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1]
+}
+
+/**
+ * Seconds; the runtime caps valid overrides at 120.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const Timeout = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0))
+
+/**
+ * Bounds the entire request and body read with one timer. Interruption reaches
+ * HttpClient.execute and response.stream so Effect aborts the transport and
+ * finalizes the body reader before returning the timeout failure.
+ * `timeoutError` preserves a tool's own timeout message while sharing the
+ * validation, duration cap, and interruption behavior.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const withDeadline = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+  url: string,
+  timeout = 30,
+  timeoutError?: (seconds: number) => StdError.StdError
+): Effect.Effect<A, E | StdError.StdError, R> => {
+  if (!Schema.is(Timeout)(timeout)) {
+    return Effect.fail(
+      new StdError.StdError({
+        code: "invalid_input",
+        message: "Timeout must be a finite number greater than zero seconds",
+        path: url
+      })
+    )
+  }
+  const seconds = Math.min(timeout, 120)
+  return Effect.timeoutOrElse(effect, {
+    duration: seconds * 1_000,
+    orElse: () =>
+      Effect.fail(
+        timeoutError?.(seconds) ?? new StdError.StdError({
+          code: "timeout",
+          message: `Request timed out after ${seconds} seconds`,
+          path: url
+        })
+      )
+  })
+}
+
+/**
+ * Keeps the head of rendered response text within the shared output budget and
+ * discloses the cut.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const capOutput = (
+  text: string
+): { readonly text: string; readonly truncated: boolean; readonly notice?: string } => {
+  const rendered = truncateBytes(text, MAX_OUTPUT_BYTES, { keep: "head" })
+  return {
+    text: rendered.text,
+    truncated: rendered.truncated,
+    ...(rendered.truncated
+      ? { notice: notice("bytes", rendered.keptBytes, rendered.keptBytes + rendered.droppedBytes) }
+      : {})
+  }
+}
+
+/** Shared response contract for the GET and POST tools.
+ * @since 1.0.0
+ * @private
+ */
+export const Response = Schema.Struct({
+  status: Schema.Number.annotate({ description: "HTTP status code, including error statuses" }),
+  body: Schema.String.annotate({ description: "Response body as text" }),
+  truncated: Schema.Boolean.annotate({ description: "Whether the body exceeded the display budget" }),
+  notice: Schema.optional(Schema.String.annotate({ description: "Truncation disclosure" }))
+})
+
+/** Validate, execute, and render one bounded HTTP request.
+ * @since 1.0.0
+ * @private
+ */
+export const execute = Effect.fn("Http.execute")(function*(
+  input: {
+    readonly url: string
+    readonly headers?: Readonly<Record<string, string>> | undefined
+    readonly timeout?: number | undefined
+  },
+  makeRequest: (url: string) => HttpClientRequest.HttpClientRequest
+): Effect.fn.Return<typeof Response.Type, StdError.StdError, HttpClient.HttpClient | GrantStore> {
+  const url = parseHttpUrl(input.url)
+  if (url === undefined) {
+    return yield* Effect.fail(
+      new StdError.StdError({
+        code: "invalid_input",
+        message: `URL must use http or https without user information: ${input.url}`,
+        path: input.url
+      })
+    )
+  }
+  const client = guarded(yield* HttpClient.HttpClient)
+  const request = makeRequest(url.toString())
+  const { status, bytes } = yield* withDeadline(
+    Effect.gen(function*() {
+      const response = yield* client.execute(
+        input.headers === undefined ? request : HttpClientRequest.setHeaders(request, input.headers)
+      )
+      const bytes = yield* readBounded(response.stream, input.url)
+      return { status: response.status, bytes }
+    }).pipe(Effect.mapError((error) => requestError(input.url, error))),
+    input.url,
+    input.timeout
+  )
+  const { text, ...disclosure } = capOutput(new TextDecoder().decode(bytes))
+  return { status, body: text, ...disclosure }
+})

package/src/internal/HttpNetwork.ts

@@ -0,0 +1,153 @@
+/**
+ * Destination policy shared by the web tools, including every redirect hop.
+ *
+ * @since 1.0.0
+ */
+
+import * as Capability from "@smthrs/capability/Capability"
+import { GrantStore } from "@smthrs/kernel/GrantStore"
+import {
+  authorizePreflight,
+  Destination,
+  fromHttpClientError,
+  supportsDestinationPinning
+} from "@smthrs/kernel/HttpClient"
+import { Context, Effect, Option } from "effect"
+import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
+import * as HttpClient from "effect/unstable/http/HttpClient"
+import * as HttpClientError from "effect/unstable/http/HttpClientError"
+import { lookup } from "node:dns/promises"
+import { BlockList, isIP } from "node:net"
+import * as StdError from "../StdError.ts"
+import { parseHttpUrl } from "./Url.ts"
+
+/** Resolver seam: all addresses must be public before an unprivileged request.
+ * @since 1.0.0
+ * @private
+ */
+export const ResolveHost = Context.Reference<
+  (hostname: string) => Effect.Effect<ReadonlyArray<string>, unknown>
+>("@smthrs/std/HttpNetwork/ResolveHost", {
+  defaultValue: () => (hostname) =>
+    Effect.tryPromise(async () => (await lookup(hostname, { all: true })).map((a) => a.address))
+})
+
+const blocked = new BlockList()
+for (
+  const [address, prefix] of [
+    ["0.0.0.0", 8],
+    ["10.0.0.0", 8],
+    ["127.0.0.0", 8],
+    ["169.254.0.0", 16],
+    ["172.16.0.0", 12],
+    ["192.168.0.0", 16],
+    ["192.0.0.0", 24],
+    ["100.64.0.0", 10],
+    ["198.18.0.0", 15],
+    ["224.0.0.0", 4],
+    ["240.0.0.0", 4]
+  ] as const
+) blocked.addSubnet(address, prefix, "ipv4")
+// Allow global IPv6 and mapped IPv4 only, excluding transition tunnels.
+const publicV6 = new BlockList()
+publicV6.addSubnet("2000::", 3, "ipv6")
+publicV6.addSubnet("::ffff:0:0", 96, "ipv6")
+blocked.addSubnet("2002::", 16, "ipv6")
+blocked.addSubnet("2001::", 32, "ipv6")
+
+const isPublic = (address: string): boolean => {
+  const family = isIP(address)
+  if (family === 4) return !blocked.check(address, "ipv4")
+  return family === 6 && publicV6.check(address, "ipv6") && !blocked.check(address, "ipv6")
+}
+
+const failure = (code: StdError.Code, message: string, path: string) => new StdError.StdError({ code, message, path })
+
+const check = Effect.fn("HttpNetwork.check")(function*(value: string, client: HttpClient.HttpClient) {
+  const url = parseHttpUrl(value)
+  if (url === undefined) {
+    return yield* Effect.fail(failure("invalid_input", "URL must use http or https without user information", value))
+  }
+  if (!supportsDestinationPinning(client)) {
+    return yield* Effect.fail(failure("unsupported", "HTTP host must support destination pinning", value))
+  }
+  const hostname = url.hostname.replace(/^\[|\]$/g, "")
+  const local = hostname.replace(/\.$/, "").toLowerCase()
+  const literal = isIP(hostname) !== 0
+  const localhost = local === "localhost" || local.endsWith(".localhost")
+  const resolve = yield* ResolveHost
+  // Refuse localhost before DNS unless the origin is explicitly authorized.
+  const privateGrant = Effect.gen(function*() {
+    const store = yield* GrantStore
+    const capability = yield* Effect.try({
+      try: () => Capability.make("net:private", url.origin),
+      catch: () => failure("invalid_input", "Private HTTP origin exceeds the capability resource limit", value)
+    })
+    yield* store.check(capability).pipe(
+      Effect.mapError(() =>
+        failure("permission_denied", "Private HTTP destination requires a net:private grant", value)
+      )
+    )
+  })
+  if (localhost) yield* privateGrant
+  const addresses = literal ? [hostname] : [
+    ...yield* resolve(hostname).pipe(
+      Effect.mapError(() => failure("request_failed", "Could not resolve HTTP destination", value))
+    )
+  ]
+  if (addresses.length === 0 || addresses.some((address) => isIP(address) === 0)) {
+    return yield* Effect.fail(failure("request_failed", "HTTP destination has no valid addresses", value))
+  }
+  if (!localhost && !addresses.every(isPublic)) yield* privateGrant
+  return Object.freeze({ origin: url.origin, addresses: Object.freeze(addresses) })
+})
+
+/** Decorates the supplied client before following redirects through the guard.
+ * The transport must honor manual redirects, as required by the kernel client.
+ * @since 1.0.0
+ * @private
+ */
+export const guarded = (
+  client: HttpClient.HttpClient,
+  follow = true
+): HttpClient.HttpClient.With<HttpClientError.HttpClientError, GrantStore> => {
+  const checked = HttpClient.transform(
+    client,
+    (execute, request) =>
+      authorizePreflight(
+        request,
+        check(request.url, client).pipe(
+          Effect.mapError((cause) =>
+            new HttpClientError.HttpClientError({
+              reason: new HttpClientError.TransportError({ request, cause, description: cause.message })
+            })
+          ),
+          Effect.flatMap((destination) =>
+            Effect.updateContext(execute, (context: Context.Context<never>) =>
+              Context.add(Context.add(context, Destination, destination), FetchHttpClient.RequestInit, {
+                ...Context.getOrUndefined(context, FetchHttpClient.RequestInit),
+                redirect: "manual"
+              }))
+          )
+        )
+      )
+  )
+  return follow ? HttpClient.followRedirects(checked) : checked
+}
+
+/** Recovers the typed refusal carried through Effect's HTTP error channel.
+ * @since 1.0.0
+ * @private
+ */
+export const refusal = (error: unknown): StdError.StdError | undefined => {
+  if (!(error instanceof HttpClientError.HttpClientError) || error.reason._tag !== "TransportError") return undefined
+  if (error.reason.cause instanceof StdError.StdError) return error.reason.cause
+  if (Option.isSome(fromHttpClientError(error))) {
+    return failure(
+      "permission_denied",
+      error.reason.description ?? "HTTP permission was refused",
+      error.reason.request.url
+    )
+  }
+  return undefined
+}

package/src/internal/Ignore.ts

@@ -0,0 +1,135 @@
+/**
+ * Root-scoped gitignore rules, compiled once per visited directory.
+ *
+ * @since 1.0.0
+ */
+
+import { escapeRegex } from "./SearchContract.ts"
+
+interface Rule {
+  readonly expression: RegExp
+  readonly directoryOnly: boolean
+  readonly negated: boolean
+  readonly byPath: boolean
+}
+
+/**
+ * Rules belonging to one visited directory.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export interface Scope {
+  readonly directory: string
+  readonly rules: ReadonlyArray<Rule>
+}
+
+// Like the search glob contract, rg's globset matches UTF-8 bytes, not code points.
+const bytes = (value: string): string =>
+  /[^\x20-\x7e]/.test(value)
+    ? Array.from(new TextEncoder().encode(value), (byte) => String.fromCharCode(byte)).join("")
+    : value
+
+// The caller-glob subset cannot parse ignore files: escapes and classes are
+// intentionally refused there. Keep their richer grammar here, including rg's
+// brace alternatives. Invalid lines are skipped without discarding valid rules.
+const expression = (pattern: string): RegExp => {
+  let source = ""
+  const groups: Array<{ prefix: string; alternatives: Array<string> }> = []
+  for (let index = 0; index < pattern.length; index++) {
+    const character = pattern[index]!
+    if (character === "\\") {
+      if (++index === pattern.length) throw new Error("dangling escape")
+      source += escapeRegex(pattern[index]!)
+    } else if (character === "[") {
+      let end = index + 1
+      const negated = pattern[end] === "!" || pattern[end] === "^"
+      if (negated) end++
+      if (pattern[end] === "]") end++
+      end = pattern.indexOf("]", end)
+      if (end < 0) source += "\\["
+      else {
+        const content = pattern.slice(index + (negated ? 2 : 1), end)
+        source += `[${negated ? "^" : ""}${content.replace(/[\\[\]^]/g, "\\$&")}]`
+        index = end
+      }
+    } else if (character === "{") {
+      groups.push({ prefix: source, alternatives: [] })
+      source = ""
+    } else if (character === "," && groups.length > 0) {
+      groups[groups.length - 1]!.alternatives.push(source)
+      source = ""
+    } else if (character === "}") {
+      const group = groups.pop()
+      if (group === undefined) throw new Error("unopened alternatives")
+      const alternatives = [...group.alternatives, source].filter((part) => part.length > 0)
+      source = group.prefix + (alternatives.length > 0 ? `(?:${alternatives.join("|")})` : "")
+    } else if (
+      character === "*" && pattern[index + 1] === "*" &&
+      (index === 0 || "/{,".includes(pattern[index - 1]!)) &&
+      (pattern[index + 2] === undefined || "/},".includes(pattern[index + 2]!))
+    ) {
+      source += pattern[index + 2] === "/" ? "(?:.*/)?" : ".*"
+      index += pattern[index + 2] === "/" ? 2 : 1
+    } else if (character === "*") source += "[^/]*"
+    else if (character === "?") source += "[^/]"
+    else source += escapeRegex(character)
+  }
+  if (groups.length > 0) throw new Error("unclosed alternatives")
+  return new RegExp(`^(?:${source})$`)
+}
+
+/**
+ * Parses gitignore lines without widening the caller-glob grammar.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const parse = (directory: string, content: string): Scope => {
+  const rules: Array<Rule> = []
+  for (let line of content.replace(/^\uFEFF/, "").split(/\r?\n/)) {
+    if (line.startsWith("#")) continue
+    // Match rg's gitignore parser: an escaped final space preserves whitespace.
+    if (!line.endsWith("\\ ")) line = line.trimEnd()
+    if (line.length === 0) continue
+    const negated = line.startsWith("!")
+    if (negated) line = line.slice(1)
+    const rooted = line.startsWith("/")
+    if (rooted) line = line.slice(1)
+    const directoryOnly = line.endsWith("/")
+    if (directoryOnly) line = line.slice(0, -1).replace(/\\$/, "")
+    const byPath = rooted || line.includes("/")
+    try {
+      rules.push({ expression: expression(bytes(line)), negated, directoryOnly, byPath })
+    } catch {
+      // rg --no-messages also tolerates malformed rules and unreadable files.
+    }
+  }
+  return { directory, rules }
+}
+
+/**
+ * Tests an entry before descent; deeper scopes and later rules take priority.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const ignored = (
+  scopes: ReadonlyArray<Scope>,
+  candidate: string,
+  basename: string,
+  directory: boolean,
+  relative: (from: string, to: string) => string
+): boolean => {
+  // The nearest ignore file wins; within a file, the last matching line wins.
+  for (let scopeIndex = scopes.length - 1; scopeIndex >= 0; scopeIndex--) {
+    const scope = scopes[scopeIndex]!
+    const scoped = bytes(relative(scope.directory, candidate))
+    for (let index = scope.rules.length - 1; index >= 0; index--) {
+      const rule = scope.rules[index]!
+      if (rule.directoryOnly && !directory) continue
+      if (rule.expression.test(rule.byPath ? scoped : bytes(basename))) return !rule.negated
+    }
+  }
+  return false
+}

package/src/internal/LinearRegex.ts

@@ -0,0 +1,173 @@
+/**
+ * Thompson matching for the validated ASCII search grammar. Each state is
+ * visited once per input position; alternatives never backtrack over input.
+ *
+ * @since 1.0.0
+ */
+
+type Term =
+  | { readonly kind: "atom"; readonly source: string }
+  | { readonly kind: "sequence" | "alternative"; readonly terms: ReadonlyArray<Term> }
+  | { readonly kind: "repeat"; readonly term: Term; readonly min: number; readonly max: number }
+
+type State =
+  | { readonly kind: "atom"; readonly expression: RegExp; readonly next: number }
+  | { readonly kind: "start" | "end"; readonly next: number }
+  | { readonly kind: "split"; readonly next: ReadonlyArray<number> }
+  | { readonly kind: "accept" }
+
+/**
+ * Compiles an already syntax-validated pattern. Compilation is bounded to
+ * 8192 states and 128 nested groups, including expanded counted repetitions.
+ * The iterator yields every 4096 state visits so the host can dispatch timers
+ * and interruption even for long lines and expensive linear matches.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const compile = (
+  pattern: string,
+  insensitive: boolean
+): { readonly test: (line: string) => Generator<void, boolean> } => {
+  let offset = 0
+  const parse = (depth: number): Term => {
+    if (depth > 128) throw new Error("patterns must not exceed 128 nested groups")
+    const alternatives: Array<Term> = []
+    let sequence: Array<Term> = []
+    while (offset < pattern.length && pattern[offset] !== ")") {
+      const start = offset
+      const character = pattern[offset++]!
+      if (character === "|") {
+        alternatives.push({ kind: "sequence", terms: sequence })
+        sequence = []
+        continue
+      }
+      let term: Term
+      if (character === "(") {
+        term = parse(depth + 1)
+        offset++
+      } else {
+        if (character === "\\") offset++
+        if (character === "[") {
+          while (pattern[offset] !== "]") {
+            if (pattern[offset++] === "\\") offset++
+          }
+          offset++
+        }
+        term = { kind: "atom", source: pattern.slice(start, offset) }
+      }
+      const quantifier = pattern[offset]
+      if (quantifier === "*" || quantifier === "+" || quantifier === "?" || quantifier === "{") {
+        offset++
+        let min = quantifier === "+" ? 1 : 0
+        let max = quantifier === "?" ? 1 : Infinity
+        if (quantifier === "{") {
+          const end = pattern.indexOf("}", offset)
+          const counts = pattern.slice(offset, end).split(",")
+          min = Number(counts[0])
+          max = counts.length === 1 ? min : counts[1] === "" ? Infinity : Number(counts[1])
+          offset = end + 1
+        }
+        // Greediness affects captures, not whether a line contains a match.
+        if (pattern[offset] === "?") offset++
+        term = { kind: "repeat", term, min, max }
+      }
+      sequence.push(term)
+    }
+    alternatives.push({ kind: "sequence", terms: sequence })
+    return { kind: "alternative", terms: alternatives }
+  }
+  const tree = parse(0)
+  const states: Array<State> = [{ kind: "accept" }]
+  const append = (state: State): number => {
+    if (states.length >= 8192) throw new Error("patterns must not exceed 8192 compiled states")
+    states.push(state)
+    return states.length - 1
+  }
+  const build = (term: Term, next: number): number => {
+    switch (term.kind) {
+      case "atom":
+        if (term.source === "^") return append({ kind: "start", next })
+        if (term.source === "$") return append({ kind: "end", next })
+        return append({
+          kind: "atom",
+          expression: new RegExp(term.source === "." ? "[^\\n]" : term.source, insensitive ? "iu" : "u"),
+          next
+        })
+      case "sequence": {
+        let head = next
+        for (let index = term.terms.length - 1; index >= 0; index--) head = build(term.terms[index]!, head)
+        return head
+      }
+      case "alternative":
+        return append({ kind: "split", next: term.terms.map((child) => build(child, next)) })
+      case "repeat": {
+        let head = next
+        if (term.max === Infinity) {
+          const loop = append({ kind: "split", next: [] })
+          states[loop] = { kind: "split", next: [next, build(term.term, loop)] }
+          head = loop
+        } else {
+          for (let index = term.min; index < term.max; index++) {
+            head = append({ kind: "split", next: [head, build(term.term, head)] })
+          }
+        }
+        for (let index = 0; index < term.min; index++) head = build(term.term, head)
+        return head
+      }
+    }
+  }
+  const start = build(tree, 0)
+  return {
+    *test(line) {
+      let current: Array<number> = []
+      let position = 0
+      let work = 0
+      const seen = new Int32Array(states.length)
+      let generation = 0
+      while (true) {
+        generation++
+        const pending = [...current, start]
+        const consuming: Array<number> = []
+        while (pending.length > 0) {
+          if (++work === 4096) {
+            work = 0
+            yield
+          }
+          const id = pending.pop()!
+          if (seen[id] === generation) continue
+          seen[id] = generation
+          const state = states[id]!
+          switch (state.kind) {
+            case "accept":
+              return true
+            case "atom":
+              consuming.push(id)
+              break
+            case "split":
+              for (const next of state.next) pending.push(next)
+              break
+            case "start":
+              if (position === 0) pending.push(state.next)
+              break
+            case "end":
+              if (position === line.length) pending.push(state.next)
+              break
+          }
+        }
+        if (position === line.length) return false
+        const character = String.fromCodePoint(line.codePointAt(position)!)
+        position += character.length
+        current = []
+        for (const id of consuming) {
+          if (++work === 4096) {
+            work = 0
+            yield
+          }
+          const state = states[id]!
+          if (state.kind === "atom" && state.expression.test(character)) current.push(state.next)
+        }
+      }
+    }
+  }
+}