@wrongstack/tools 0.310.1 → 0.313.1

package/dist/fetch.js

@@ -1,401 +1,11 @@
-// src/fetch.ts
-import { FetchError as FetchError2, ToolError, ToolValidationError as ToolValidationError2 } from "@wrongstack/core/types";
-import TurndownService from "turndown";
-
-// src/_fetch-guard.ts
-import * as dns from "node:dns/promises";
-import * as net from "node:net";
-import { isPrivateIPv4, isPrivateIPv6 } from "@wrongstack/core/utils";
-import { FetchError, ToolValidationError } from "@wrongstack/core/types";
-import { Agent, fetch as undiciFetch } from "undici";
-var nativeGlobalFetch = globalThis.fetch;
-var ALLOW_PRIVATE = process.env["WRONGSTACK_FETCH_ALLOW_PRIVATE"] === "1";
-if (ALLOW_PRIVATE && !process.env["CI"]) {
-  console.warn(
-    "[WrongStack] WARNING: WRONGSTACK_FETCH_ALLOW_PRIVATE=1 is active \u2014\n  fetch tool can now access private IPs (10.x, 192.168.x, 169.254.x),\n  cloud metadata endpoints, and plaintext HTTP. Use only on isolated networks."
-  );
-}
-function guardedLookup(hostname, options, callback) {
-  dns.lookup(hostname, { all: true }).then((records) => {
-    const family = options?.family;
-    const byFamily = family === 4 || family === 6 ? records.filter((r) => r.family === family) : records;
-    const list = byFamily.length > 0 ? byFamily : records;
-    if (!ALLOW_PRIVATE) {
-      for (const r of list) {
-        const bad = r.family === 4 ? isPrivateIPv4(r.address) : isPrivateIPv6(r.address);
-        if (bad) {
-          callback(
-            Object.assign(new Error(`fetch: resolved to private address ${r.address}`), {
-              code: "EAI_FAIL"
-            })
-          );
-          return;
-        }
-      }
-    }
-    if (options?.all) {
-      callback(
-        null,
-        list.map((r) => ({ address: r.address, family: r.family }))
-      );
-      return;
-    }
-    const first = list.at(0);
-    if (!first) {
-      callback(
-        Object.assign(new Error(`fetch: no address for ${hostname}`), { code: "ENOTFOUND" })
-      );
-      return;
-    }
-    callback(null, first.address, first.family);
-  }).catch((err) => callback(err));
-}
-var pinnedAgent;
-function getPinnedDispatcher() {
-  if (!pinnedAgent) {
-    pinnedAgent = new Agent({ allowH2: false, connect: { lookup: guardedLookup } });
-  }
-  return pinnedAgent;
-}
-function dispatcherFetch() {
-  return globalThis.fetch === nativeGlobalFetch ? undiciFetch : globalThis.fetch;
-}
-var _beforeExitRegistered = false;
-if (!_beforeExitRegistered) {
-  _beforeExitRegistered = true;
-  process.on("beforeExit", () => {
-    pinnedAgent?.destroy();
-    pinnedAgent = void 0;
-  });
-}
-async function guardedFetch(url, maxRedirects, signal, headers = {
-  "user-agent": "WrongStack/1.0 (+https://wrongstack.com)",
-  accept: "text/html,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1"
-}) {
-  let redirectCount = 0;
-  let currentUrl = url;
-  for (; ; ) {
-    const parsed = new URL(currentUrl);
-    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
-      throw new ToolValidationError({
-        message: `fetch: redirect to unsupported protocol "${parsed.protocol}"`,
-        field: "url"
-      });
-    }
-    if (parsed.protocol === "http:" && !ALLOW_PRIVATE) {
-      throw new ToolValidationError({
-        message: "fetch: redirect to http:// blocked (HTTPS required by default)",
-        field: "url"
-      });
-    }
-    await assertNotPrivate(parsed.hostname);
-    const init = {
-      redirect: "manual",
-      signal,
-      headers,
-      dispatcher: getPinnedDispatcher()
-    };
-    const res = await dispatcherFetch()(currentUrl, init);
-    if (res.status < 300 || res.status > 399) {
-      return res;
-    }
-    try {
-      await res.body?.cancel();
-    } catch {
-    }
-    redirectCount++;
-    if (redirectCount > maxRedirects) {
-      throw new FetchError({
-        message: `fetch: exceeded ${maxRedirects} redirects`,
-        status: res.status,
-        context: { url: currentUrl, maxRedirects, redirectCount }
-      });
-    }
-    const location = res.headers.get("location");
-    if (!location) {
-      throw new FetchError({
-        message: "fetch: redirect status with no location header",
-        status: res.status,
-        context: { url: currentUrl, redirectCount }
-      });
-    }
-    currentUrl = new URL(location, currentUrl).toString();
-  }
-}
-async function assertNotPrivate(hostname) {
-  if (ALLOW_PRIVATE) return;
-  const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
-  if (host === "localhost" || host.endsWith(".localhost")) {
-    throw new ToolValidationError({
-      message: "fetch: blocked localhost target",
-      field: "url"
-    });
-  }
-  const ipVersion = net.isIP(host);
-  if (ipVersion === 4) {
-    if (isPrivateIPv4(host)) {
-      throw new ToolValidationError({
-        message: `fetch: blocked private/loopback address "${host}"`,
-        field: "url"
-      });
-    }
-  } else if (ipVersion === 6) {
-    if (isPrivateIPv6(host)) {
-      throw new ToolValidationError({
-        message: `fetch: blocked private/loopback address "${host}"`,
-        field: "url"
-      });
-    }
-  } else {
-    try {
-      const records = await dns.lookup(host, { all: true });
-      for (const r of records) {
-        const bad = r.family === 4 ? isPrivateIPv4(r.address) : isPrivateIPv6(r.address);
-        if (bad) {
-          throw new ToolValidationError({
-            message: `fetch: resolved to private address ${r.address}`,
-            field: "url"
-          });
-        }
-      }
-    } catch (err) {
-      if (err instanceof ToolValidationError) throw err;
-    }
-  }
-}
-
-// src/_util.ts
-import * as Core from "@wrongstack/core/utils";
-function truncateMiddle(s, max) {
-  if (Buffer.byteLength(s, "utf8") <= max) return s;
-  const half = Math.floor(max / 2);
-  return s.slice(0, half) + `
-\u2026[truncated ${Buffer.byteLength(s, "utf8") - max} bytes from middle]\u2026
-` + s.slice(-half);
-}
-
-// src/fetch.ts
-var TD = new TurndownService({
-  // Use `# Title` for headings, not setext underline style (`Title\n=====`).
-  headingStyle: "atx",
-  // Don't wrap code blocks in <pre> — render them as triple-backtick blocks.
-  codeBlockStyle: "fenced"
-});
-TD.addRule("stripDangerousElements", {
-  filter: ["script", "style", "noscript"],
-  replacement: () => ""
-});
-var PRUNED_BOILERPLATE_TAGS = /* @__PURE__ */ new Set(["nav", "header", "footer", "aside", "svg", "iframe"]);
-TD.remove((node) => PRUNED_BOILERPLATE_TAGS.has(node.nodeName.toLowerCase()));
-var MAX_BYTES = 131072;
-var TIMEOUT_MS = 2e4;
-var combineSignals = (signals) => AbortSignal.any(signals);
-var fetchTool = {
-  name: "fetch",
-  category: "Network",
-  description: "Fetch a URL and return its content. HTML pages are automatically converted to clean markdown. This tool has strong SSRF protections (private IPs, localhost, and cloud metadata endpoints are blocked by default).",
-  usageHint: "Use this when you need external information (documentation, API responses, web pages, etc.).\n\nSecurity notes:\n- Only HTTPS is allowed by default.\n- Internal/private networks are blocked unless explicitly enabled via environment variable.\n- Redirects are followed but re-validated at each hop.\n- Output is capped (128KB by default) to avoid flooding context.\nPrefer this over raw `bash curl` or `bash wget`.",
-  permission: "auto",
-  mutating: false,
-  capabilities: ["net.outbound"],
-  icon: "web",
-  // Trust rules for fetch match on the literal URL — declare it explicitly
-  // so a user can trust `https://api.example.com/*` without accidentally
-  // matching that pattern on any other tool that happens to have a `url`
-  // input field.
-  subjectKey: "url",
-  timeoutMs: TIMEOUT_MS,
-  maxOutputBytes: MAX_BYTES,
-  inputSchema: {
-    type: "object",
-    properties: {
-      url: {
-        type: "string",
-        description: "The target URL (must use https://)."
-      },
-      format: {
-        type: "string",
-        enum: ["markdown", "text", "raw"],
-        description: 'Output format. "markdown" is recommended for HTML pages; for non-HTML content types it falls back to plain text (JSON is pretty-printed).'
-      }
-    },
-    required: ["url"]
-  },
-  async execute(input, ctx, opts) {
-    let final;
-    const executeStream = fetchTool.executeStream;
-    if (!executeStream) {
-      throw new ToolError({
-        message: "fetchTool: stream execution unavailable",
-        code: "TOOL_EXECUTION_FAILED",
-        toolName: "fetch"
-      });
-    }
-    for await (const ev of executeStream(input, ctx, opts)) {
-      if (ev.type === "final") final = ev.output;
-    }
-    if (!final) {
-      throw new ToolError({
-        message: "fetch: stream ended without final event",
-        code: "TOOL_EXECUTION_FAILED",
-        toolName: "fetch"
-      });
-    }
-    return final;
-  },
-  async *executeStream(input, ctx, opts) {
-    if (!input?.url) {
-      throw new ToolValidationError2({
-        message: "fetch: url is required",
-        field: "url"
-      });
-    }
-    const u = new URL(input.url);
-    if (u.username || u.password) {
-      throw new ToolValidationError2({
-        message: "fetch: URLs with embedded credentials (user:pass@host) are not allowed",
-        field: "url"
-      });
-    }
-    if (u.protocol !== "https:" && u.protocol !== "http:") {
-      throw new ToolValidationError2({
-        message: `fetch: unsupported protocol "${u.protocol}"`,
-        field: "url"
-      });
-    }
-    if (u.protocol === "http:" && !ALLOW_PRIVATE) {
-      throw new ToolValidationError2({
-        message: "fetch: http:// blocked (HTTPS required by default)",
-        field: "url"
-      });
-    }
-    await assertNotPrivate(u.hostname);
-    yield { type: "log", text: `GET ${input.url}` };
-    const ctrl = new AbortController();
-    const timer = setTimeout(
-      () => ctrl.abort(
-        new ToolError({
-          message: "fetch timeout",
-          code: "TOOL_TIMEOUT",
-          toolName: "fetch"
-        })
-      ),
-      TIMEOUT_MS
-    );
-    const combined = combineSignals([opts.signal, ctrl.signal]);
-    try {
-      let res;
-      try {
-        res = await guardedFetch(input.url, 5, combined);
-      } catch (err) {
-        if (opts.signal.aborted) throw err;
-        throw describeFetchError(err, input.url, ctrl.signal.aborted);
-      }
-      const ct = res.headers.get("content-type") ?? "application/octet-stream";
-      if (/^image\/|^audio\/|^video\/|application\/octet-stream/.test(ct)) {
-        throw new FetchError2({
-          message: `fetch: refusing to read binary content-type "${ct}"`,
-          status: res.status,
-          context: { url: res.url, contentType: ct }
-        });
-      }
-      yield {
-        type: "log",
-        text: `HTTP ${res.status} ${ct}`,
-        data: { status: res.status, contentType: ct }
-      };
-      const reader = res.body?.getReader();
-      let received = 0;
-      const chunks = [];
-      let pendingBytes = 0;
-      const FLUSH_AT = 4 * 1024;
-      if (reader) {
-        for (; ; ) {
-          const { value, done } = await reader.read();
-          if (done) break;
-          if (!value) continue;
-          received += value.byteLength;
-          pendingBytes += value.byteLength;
-          chunks.push(value);
-          if (pendingBytes >= FLUSH_AT) {
-            const recent = Buffer.from(value).toString("utf-8");
-            yield {
-              type: "partial_output",
-              text: recent,
-              data: { received }
-            };
-            pendingBytes = 0;
-          }
-          if (received > MAX_BYTES) break;
-        }
-      }
-      const text = Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8");
-      const format = input.format ?? (ct.includes("text/html") ? "markdown" : "text");
-      let content;
-      if (format === "raw") content = text;
-      else if (format === "markdown" && ct.includes("text/html")) content = TD.turndown(text);
-      else if (ct.includes("application/json")) content = prettyJson(text);
-      else content = text;
-      yield {
-        type: "final",
-        output: {
-          content: truncateMiddle(content, MAX_BYTES),
-          status: res.status,
-          content_type: ct,
-          url: res.url
-        }
-      };
-      ctx.recordSideEffect?.({
-        toolUseId: `fetch-${Date.now()}`,
-        toolName: "fetch",
-        ts: (/* @__PURE__ */ new Date()).toISOString(),
-        input: { url: input.url, format: input.format },
-        outcome: `HTTP ${res.status} (${ct})`,
-        risk: "network"
-      });
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-};
-function describeFetchError(err, url, timedOut) {
-  if (timedOut) {
-    return new ToolError({
-      message: `fetch: GET ${url} timed out after ${TIMEOUT_MS}ms`,
-      code: "TOOL_TIMEOUT",
-      toolName: "fetch",
-      context: { url, timedOut: true, timeoutMs: TIMEOUT_MS },
-      cause: err
-    });
-  }
-  const parts = [];
-  const seen = /* @__PURE__ */ new Set();
-  let cur = err;
-  while (cur instanceof Error && !seen.has(cur)) {
-    seen.add(cur);
-    const code = cur.code;
-    const label = code ? `${code}: ${cur.message}` : cur.message;
-    if (label && label !== "fetch failed" && !parts.includes(label)) parts.push(label);
-    cur = cur.cause;
-  }
-  const detail = parts.length > 0 ? parts.join(" \u2192 ") : "fetch failed";
-  return new FetchError2({
-    message: `fetch: GET ${url} failed \u2014 ${detail}`,
-    status: 502,
-    context: { url, timedOut: false, transportErrors: parts },
-    // Preserve the original undici / DNS / TLS chain so callers can inspect
-    // it via `err.cause` and structured `instanceof` checks. The flattened
-    // text version stays in the message for human readability.
-    cause: err
-  });
-}
-function prettyJson(s) {
-  try {
-    return JSON.stringify(JSON.parse(s), null, 2);
-  } catch {
-    return s;
-  }
-}
+import {
+  fetchTool
+} from "./chunk-IXMNTNW5.js";
+import {
+  guardedFetch,
+  guardedLookup
+} from "./chunk-NBDZMMG2.js";
+import "./chunk-D33S4RQG.js";
 export {
   fetchTool,
   guardedFetch,