@smthrs/std 0.0.0-stage → 1.0.0-rc.12

package/src/internal/Html.ts

@@ -0,0 +1,87 @@
+/**
+ * Small, dependency-free HTML renderers for web retrieval.
+ *
+ * @since 1.0.0
+ */
+
+const comments = /<!--[\s\S]*?-->/g
+const entities: Readonly<Record<string, string>> = { amp: "&", lt: "<", gt: ">", quot: "\"", apos: "'", nbsp: " " }
+
+const dropSkipped = (html: string): string => {
+  const lower = html.replace(/[A-Z]/g, (letter) => letter.toLowerCase())
+  const opening = /<(head|script|style|noscript|iframe|object|embed)\b/gi
+  let output = ""
+  let cursor = 0
+  let match: RegExpExecArray | null
+  while ((match = opening.exec(html)) !== null) {
+    output += html.slice(cursor, match.index)
+    const marker = `</${match[1]!.toLowerCase()}`
+    let close = lower.indexOf(marker, opening.lastIndex)
+    while (close !== -1) {
+      let end = close + marker.length
+      while (/\s/.test(lower[end] ?? "")) end++
+      if (lower[end] === ">") {
+        cursor = end + 1
+        opening.lastIndex = cursor
+        break
+      }
+      close = lower.indexOf(marker, close + marker.length)
+    }
+    if (close === -1) return output
+  }
+  return output + html.slice(cursor)
+}
+
+const decode = (value: string): string =>
+  value.replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => {
+    if (entity[0] === "#") {
+      const number = Number.parseInt(
+        entity.slice(entity[1]!.toLowerCase() === "x" ? 2 : 1),
+        entity[1]!.toLowerCase() === "x" ? 16 : 10
+      )
+      return number <= 0x10ffff
+          && !(number >= 0xd800 && number <= 0xdfff)
+        ? String.fromCodePoint(number)
+        : match
+    }
+    const key = entity.toLowerCase()
+    return Object.hasOwn(entities, key) ? entities[key]! : match
+  })
+
+const source = (html: string): string => dropSkipped(html.replace(comments, ""))
+
+/**
+ * Extracts readable text while dropping executable and presentation-only elements.
+ *
+ * @category rendering
+ * @since 1.0.0
+ */
+export const toText = (html: string): string =>
+  decode(source(html).replace(/<(br|\/p|\/div|\/li|\/h[1-6]|\/tr)\b[^>]*>/gi, "\n").replace(/<[^>]*>/g, " "))
+    .replace(/[ \t]+\n/g, "\n").replace(/\n[ \t]+/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim()
+
+/**
+ * Converts a conservative subset of HTML into readable Markdown.
+ *
+ * @category rendering
+ * @since 1.0.0
+ */
+export const toMarkdown = (html: string): string => {
+  const markdown = source(html)
+    .replace(
+      /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi,
+      (_all, level: string, text: string) => `\n\n${"#".repeat(Number(level))} ${toText(text)}\n\n`
+    )
+    .replace(/<(strong|b)\b[^>]*>([\s\S]*?)<\/\1>/gi, "**$2**")
+    .replace(/<(em|i)\b[^>]*>([\s\S]*?)<\/\1>/gi, "*$2*")
+    .replace(
+      /<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi,
+      (_all, href: string, text: string) => `[${toText(text)}](${href})`
+    )
+    .replace(/<li\b[^>]*>/gi, "\n- ").replace(
+      /<\/?(ul|ol|p|div|section|article|main|blockquote|pre|tr)\b[^>]*>/gi,
+      "\n"
+    )
+    .replace(/<br\b[^>]*>/gi, "\n").replace(/<[^>]*>/g, "")
+  return decode(markdown).replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim()
+}

package/src/internal/Http.ts

@@ -0,0 +1,209 @@
+/**
+ * Shared HTTP response limits, error mapping, and request deadlines.
+ *
+ * @since 1.0.0
+ */
+
+import type { GrantStore } from "@smthrs/kernel/GrantStore"
+import * as HttpClient from "@smthrs/kernel/HttpClient"
+import * as Effect from "effect/Effect"
+import * as HttpClientRequest from "effect/http/HttpClientRequest"
+import * as Schema from "effect/Schema"
+import * as Stream from "effect/Stream"
+import * as StdError from "../StdError.ts"
+import { guarded, refusal } from "./HttpNetwork.ts"
+import { MAX_OUTPUT_BYTES, notice, truncateBytes } from "./Text.ts"
+import { parseHttpUrl } from "./Url.ts"
+
+/**
+ * Maximum captured response bytes before decoding.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const MAX_RESPONSE_BYTES = 5 * 1024 * 1024
+
+interface BodyState {
+  readonly chunks: Array<Uint8Array>
+  size: number
+}
+
+/**
+ * Reads a response up to the shared byte limit and finalizes the stream on failure.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const readBounded = <E, R>(
+  stream: Stream.Stream<Uint8Array, E, R>,
+  url: string
+): Effect.Effect<Uint8Array, E | StdError.StdError, R> =>
+  Stream.runFoldEffect(
+    stream,
+    (): BodyState => ({ chunks: [], size: 0 }),
+    (state, chunk) => {
+      const size = state.size + chunk.byteLength
+      if (size > MAX_RESPONSE_BYTES) {
+        return Effect.fail(
+          new StdError.StdError({
+            code: "response_too_large",
+            message: "Response exceeds the 5 MiB limit",
+            path: url
+          })
+        )
+      }
+      state.chunks.push(chunk)
+      state.size = size
+      return Effect.succeed(state)
+    }
+  ).pipe(
+    Effect.map((state) => {
+      const output = new Uint8Array(state.size)
+      let offset = 0
+      for (const chunk of state.chunks) {
+        output.set(chunk, offset)
+        offset += chunk.byteLength
+      }
+      return output
+    })
+  )
+
+/**
+ * Maps transport failures while preserving standard HTTP failure codes.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const requestError = (url: string, error: unknown): StdError.StdError =>
+  error instanceof StdError.StdError ? error : refusal(error) ?? new StdError.StdError({
+    code: "request_failed",
+    message: `Request failed: ${url}${error instanceof Error ? ` (${error.message})` : ""}`
+  })
+
+/**
+ * Looks up a response header without depending on transport normalization.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const header = (headers: Readonly<Record<string, string | undefined>>, name: string): string | undefined => {
+  const lower = name.toLowerCase()
+  return Object.entries(headers).find(([key]) => key.toLowerCase() === lower)?.[1]
+}
+
+/**
+ * Seconds; the runtime caps valid overrides at 120.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const Timeout = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0))
+
+/**
+ * Bounds the entire request and body read with one timer. Interruption reaches
+ * HttpClient.execute and response.stream so Effect aborts the transport and
+ * finalizes the body reader before returning the timeout failure.
+ * `timeoutError` preserves a tool's own timeout message while sharing the
+ * validation, duration cap, and interruption behavior.
+ *
+ * @category http
+ * @since 1.0.0
+ */
+export const withDeadline = <A, E, R>(
+  effect: Effect.Effect<A, E, R>,
+  url: string,
+  timeout = 30,
+  timeoutError?: (seconds: number) => StdError.StdError
+): Effect.Effect<A, E | StdError.StdError, R> => {
+  if (!Schema.is(Timeout)(timeout)) {
+    return Effect.fail(
+      new StdError.StdError({
+        code: "invalid_input",
+        message: "Timeout must be a finite number greater than zero seconds",
+        path: url
+      })
+    )
+  }
+  const seconds = Math.min(timeout, 120)
+  return Effect.timeoutOrElse(effect, {
+    duration: seconds * 1_000,
+    orElse: () =>
+      Effect.fail(
+        timeoutError?.(seconds) ?? new StdError.StdError({
+          code: "timeout",
+          message: `Request timed out after ${seconds} seconds`,
+          path: url
+        })
+      )
+  })
+}
+
+/**
+ * Keeps the head of rendered response text within the shared output budget and
+ * discloses the cut.
+ *
+ * @since 1.0.0
+ * @private
+ */
+export const capOutput = (
+  text: string
+): { readonly text: string; readonly truncated: boolean; readonly notice?: string } => {
+  const rendered = truncateBytes(text, MAX_OUTPUT_BYTES, { keep: "head" })
+  return {
+    text: rendered.text,
+    truncated: rendered.truncated,
+    ...(rendered.truncated
+      ? { notice: notice("bytes", rendered.keptBytes, rendered.keptBytes + rendered.droppedBytes) }
+      : {})
+  }
+}
+
+/** Shared response contract for the GET and POST tools.
+ * @since 1.0.0
+ * @private
+ */
+export const Response = Schema.Struct({
+  status: Schema.Number.annotate({ description: "HTTP status code, including error statuses" }),
+  body: Schema.String.annotate({ description: "Response body as text" }),
+  truncated: Schema.Boolean.annotate({ description: "Whether the body exceeded the display budget" }),
+  notice: Schema.optional(Schema.String.annotate({ description: "Truncation disclosure" }))
+})
+
+/** Validate, execute, and render one bounded HTTP request.
+ * @since 1.0.0
+ * @private
+ */
+export const execute = Effect.fn("Http.execute")(function*(
+  input: {
+    readonly url: string
+    readonly headers?: Readonly<Record<string, string>> | undefined
+    readonly timeout?: number | undefined
+  },
+  makeRequest: (url: string) => HttpClientRequest.HttpClientRequest
+): Effect.fn.Return<typeof Response.Type, StdError.StdError, HttpClient.HttpClient | GrantStore> {
+  const url = parseHttpUrl(input.url)
+  if (url === undefined) {
+    return yield* Effect.fail(
+      new StdError.StdError({
+        code: "invalid_input",
+        message: `URL must use http or https without user information: ${input.url}`,
+        path: input.url
+      })
+    )
+  }
+  const client = guarded(yield* HttpClient.HttpClient)
+  const request = makeRequest(url.toString())
+  const { status, bytes } = yield* withDeadline(
+    Effect.gen(function*() {
+      const response = yield* client.execute(
+        input.headers === undefined ? request : HttpClientRequest.setHeaders(request, input.headers)
+      )
+      const bytes = yield* readBounded(response.stream, input.url)
+      return { status: response.status, bytes }
+    }).pipe(Effect.mapError((error) => requestError(input.url, error))),
+    input.url,
+    input.timeout
+  )
+  const { text, ...disclosure } = capOutput(new TextDecoder().decode(bytes))
+  return { status, body: text, ...disclosure }
+})

package/src/internal/HttpNetwork.ts

@@ -0,0 +1,153 @@
+/**
+ * Destination policy shared by the web tools, including every redirect hop.
+ *
+ * @since 1.0.0
+ */
+
+import * as Capability from "@smthrs/capability/Capability"
+import { GrantStore } from "@smthrs/kernel/GrantStore"
+import {
+  authorizePreflight,
+  Destination,
+  fromHttpClientError,
+  supportsDestinationPinning
+} from "@smthrs/kernel/HttpClient"
+import { Context, Effect, Option } from "effect"
+import * as FetchHttpClient from "effect/http/FetchHttpClient"
+import * as HttpClient from "effect/http/HttpClient"
+import * as HttpClientError from "effect/http/HttpClientError"
+import { lookup } from "node:dns/promises"
+import { BlockList, isIP } from "node:net"
+import * as StdError from "../StdError.ts"
+import { parseHttpUrl } from "./Url.ts"
+
+/** Resolver seam: all addresses must be public before an unprivileged request.
+ * @since 1.0.0
+ * @private
+ */
+export const ResolveHost = Context.Reference<
+  (hostname: string) => Effect.Effect<ReadonlyArray<string>, unknown>
+>("@smthrs/std/HttpNetwork/ResolveHost", {
+  defaultValue: () => (hostname) =>
+    Effect.tryPromise(async () => (await lookup(hostname, { all: true })).map((a) => a.address))
+})
+
+const blocked = new BlockList()
+for (
+  const [address, prefix] of [
+    ["0.0.0.0", 8],
+    ["10.0.0.0", 8],
+    ["127.0.0.0", 8],
+    ["169.254.0.0", 16],
+    ["172.16.0.0", 12],
+    ["192.168.0.0", 16],
+    ["192.0.0.0", 24],
+    ["100.64.0.0", 10],
+    ["198.18.0.0", 15],
+    ["224.0.0.0", 4],
+    ["240.0.0.0", 4]
+  ] as const
+) blocked.addSubnet(address, prefix, "ipv4")
+// Allow global IPv6 and mapped IPv4 only, excluding transition tunnels.
+const publicV6 = new BlockList()
+publicV6.addSubnet("2000::", 3, "ipv6")
+publicV6.addSubnet("::ffff:0:0", 96, "ipv6")
+blocked.addSubnet("2002::", 16, "ipv6")
+blocked.addSubnet("2001::", 32, "ipv6")
+
+const isPublic = (address: string): boolean => {
+  const family = isIP(address)
+  if (family === 4) return !blocked.check(address, "ipv4")
+  return family === 6 && publicV6.check(address, "ipv6") && !blocked.check(address, "ipv6")
+}
+
+const failure = (code: StdError.Code, message: string, path: string) => new StdError.StdError({ code, message, path })
+
+const check = Effect.fn("HttpNetwork.check")(function*(value: string, client: HttpClient.HttpClient) {
+  const url = parseHttpUrl(value)
+  if (url === undefined) {
+    return yield* Effect.fail(failure("invalid_input", "URL must use http or https without user information", value))
+  }
+  if (!supportsDestinationPinning(client)) {
+    return yield* Effect.fail(failure("unsupported", "HTTP host must support destination pinning", value))
+  }
+  const hostname = url.hostname.replace(/^\[|\]$/g, "")
+  const local = hostname.replace(/\.$/, "").toLowerCase()
+  const literal = isIP(hostname) !== 0
+  const localhost = local === "localhost" || local.endsWith(".localhost")
+  const resolve = yield* ResolveHost
+  // Refuse localhost before DNS unless the origin is explicitly authorized.
+  const privateGrant = Effect.gen(function*() {
+    const store = yield* GrantStore
+    const capability = yield* Effect.try({
+      try: () => Capability.make("net:private", url.origin),
+      catch: () => failure("invalid_input", "Private HTTP origin exceeds the capability resource limit", value)
+    })
+    yield* store.check(capability).pipe(
+      Effect.mapError(() =>
+        failure("permission_denied", "Private HTTP destination requires a net:private grant", value)
+      )
+    )
+  })
+  if (localhost) yield* privateGrant
+  const addresses = literal ? [hostname] : [
+    ...yield* resolve(hostname).pipe(
+      Effect.mapError(() => failure("request_failed", "Could not resolve HTTP destination", value))
+    )
+  ]
+  if (addresses.length === 0 || addresses.some((address) => isIP(address) === 0)) {
+    return yield* Effect.fail(failure("request_failed", "HTTP destination has no valid addresses", value))
+  }
+  if (!localhost && !addresses.every(isPublic)) yield* privateGrant
+  return Object.freeze({ origin: url.origin, addresses: Object.freeze(addresses) })
+})
+
+/** Decorates the supplied client before following redirects through the guard.
+ * The transport must honor manual redirects, as required by the kernel client.
+ * @since 1.0.0
+ * @private
+ */
+export const guarded = (
+  client: HttpClient.HttpClient,
+  follow = true
+): HttpClient.HttpClient.With<HttpClientError.HttpClientError, GrantStore> => {
+  const checked = HttpClient.transform(
+    client,
+    (execute, request) =>
+      authorizePreflight(
+        request,
+        check(request.url, client).pipe(
+          Effect.mapError((cause) =>
+            new HttpClientError.HttpClientError({
+              reason: new HttpClientError.TransportError({ request, cause, description: cause.message })
+            })
+          ),
+          Effect.flatMap((destination) =>
+            Effect.updateContext(execute, (context: Context.Context<never>) =>
+              Context.add(Context.add(context, Destination, destination), FetchHttpClient.RequestInit, {
+                ...Context.getOrUndefined(context, FetchHttpClient.RequestInit),
+                redirect: "manual"
+              }))
+          )
+        )
+      )
+  )
+  return follow ? HttpClient.followRedirects(checked) : checked
+}
+
+/** Recovers the typed refusal carried through Effect's HTTP error channel.
+ * @since 1.0.0
+ * @private
+ */
+export const refusal = (error: unknown): StdError.StdError | undefined => {
+  if (!(error instanceof HttpClientError.HttpClientError) || error.reason._tag !== "TransportError") return undefined
+  if (error.reason.cause instanceof StdError.StdError) return error.reason.cause
+  if (Option.isSome(fromHttpClientError(error))) {
+    return failure(
+      "permission_denied",
+      error.reason.description ?? "HTTP permission was refused",
+      error.reason.request.url
+    )
+  }
+  return undefined
+}

package/src/internal/Ignore.ts

@@ -0,0 +1,135 @@
+/**
+ * Root-scoped gitignore rules, compiled once per visited directory.
+ *
+ * @since 1.0.0
+ */
+
+import { escapeRegex } from "./SearchContract.ts"
+
+interface Rule {
+  readonly expression: RegExp
+  readonly directoryOnly: boolean
+  readonly negated: boolean
+  readonly byPath: boolean
+}
+
+/**
+ * Rules belonging to one visited directory.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export interface Scope {
+  readonly directory: string
+  readonly rules: ReadonlyArray<Rule>
+}
+
+// Like the search glob contract, rg's globset matches UTF-8 bytes, not code points.
+const bytes = (value: string): string =>
+  /[^\x20-\x7e]/.test(value)
+    ? Array.from(new TextEncoder().encode(value), (byte) => String.fromCharCode(byte)).join("")
+    : value
+
+// The caller-glob subset cannot parse ignore files: escapes and classes are
+// intentionally refused there. Keep their richer grammar here, including rg's
+// brace alternatives. Invalid lines are skipped without discarding valid rules.
+const expression = (pattern: string): RegExp => {
+  let source = ""
+  const groups: Array<{ prefix: string; alternatives: Array<string> }> = []
+  for (let index = 0; index < pattern.length; index++) {
+    const character = pattern[index]!
+    if (character === "\\") {
+      if (++index === pattern.length) throw new Error("dangling escape")
+      source += escapeRegex(pattern[index]!)
+    } else if (character === "[") {
+      let end = index + 1
+      const negated = pattern[end] === "!" || pattern[end] === "^"
+      if (negated) end++
+      if (pattern[end] === "]") end++
+      end = pattern.indexOf("]", end)
+      if (end < 0) source += "\\["
+      else {
+        const content = pattern.slice(index + (negated ? 2 : 1), end)
+        source += `[${negated ? "^" : ""}${content.replace(/[\\[\]^]/g, "\\$&")}]`
+        index = end
+      }
+    } else if (character === "{") {
+      groups.push({ prefix: source, alternatives: [] })
+      source = ""
+    } else if (character === "," && groups.length > 0) {
+      groups[groups.length - 1]!.alternatives.push(source)
+      source = ""
+    } else if (character === "}") {
+      const group = groups.pop()
+      if (group === undefined) throw new Error("unopened alternatives")
+      const alternatives = [...group.alternatives, source].filter((part) => part.length > 0)
+      source = group.prefix + (alternatives.length > 0 ? `(?:${alternatives.join("|")})` : "")
+    } else if (
+      character === "*" && pattern[index + 1] === "*" &&
+      (index === 0 || "/{,".includes(pattern[index - 1]!)) &&
+      (pattern[index + 2] === undefined || "/},".includes(pattern[index + 2]!))
+    ) {
+      source += pattern[index + 2] === "/" ? "(?:.*/)?" : ".*"
+      index += pattern[index + 2] === "/" ? 2 : 1
+    } else if (character === "*") source += "[^/]*"
+    else if (character === "?") source += "[^/]"
+    else source += escapeRegex(character)
+  }
+  if (groups.length > 0) throw new Error("unclosed alternatives")
+  return new RegExp(`^(?:${source})$`)
+}
+
+/**
+ * Parses gitignore lines without widening the caller-glob grammar.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const parse = (directory: string, content: string): Scope => {
+  const rules: Array<Rule> = []
+  for (let line of content.replace(/^\uFEFF/, "").split(/\r?\n/)) {
+    if (line.startsWith("#")) continue
+    // Match rg's gitignore parser: an escaped final space preserves whitespace.
+    if (!line.endsWith("\\ ")) line = line.trimEnd()
+    if (line.length === 0) continue
+    const negated = line.startsWith("!")
+    if (negated) line = line.slice(1)
+    const rooted = line.startsWith("/")
+    if (rooted) line = line.slice(1)
+    const directoryOnly = line.endsWith("/")
+    if (directoryOnly) line = line.slice(0, -1).replace(/\\$/, "")
+    const byPath = rooted || line.includes("/")
+    try {
+      rules.push({ expression: expression(bytes(line)), negated, directoryOnly, byPath })
+    } catch {
+      // rg --no-messages also tolerates malformed rules and unreadable files.
+    }
+  }
+  return { directory, rules }
+}
+
+/**
+ * Tests an entry before descent; deeper scopes and later rules take priority.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const ignored = (
+  scopes: ReadonlyArray<Scope>,
+  candidate: string,
+  basename: string,
+  directory: boolean,
+  relative: (from: string, to: string) => string
+): boolean => {
+  // The nearest ignore file wins; within a file, the last matching line wins.
+  for (let scopeIndex = scopes.length - 1; scopeIndex >= 0; scopeIndex--) {
+    const scope = scopes[scopeIndex]!
+    const scoped = bytes(relative(scope.directory, candidate))
+    for (let index = scope.rules.length - 1; index >= 0; index--) {
+      const rule = scope.rules[index]!
+      if (rule.directoryOnly && !directory) continue
+      if (rule.expression.test(rule.byPath ? scoped : bytes(basename))) return !rule.negated
+    }
+  }
+  return false
+}