@iowarp/clio-coder 0.4.4 → 0.4.5

package/dist/{web-fetch-V4FKSDAV.js → web-fetch-MQN4FH3X.js}

@@ -1,18 +1,164 @@
 import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
 import {
   webFetchToolSurface
-} from "./chunk-2CU2H6KE.js";
-import "./chunk-5B2AEOW5.js";
+} from "./chunk-BBV5TKE6.js";
+import {
+  WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV
+} from "./chunk-SL2RDHKO.js";
+import {
+  UNTRUSTED_CONTENT_BANNER
+} from "./chunk-WXA2CFJO.js";
+import "./chunk-GH5ZR6JP.js";
 import {
   truncateUtf8
-} from "./chunk-6XLNIQDB.js";
-import "./chunk-4NSRCOYP.js";
+} from "./chunk-MYI32N4R.js";
+import "./chunk-EHOSNVVM.js";
 import {
   init_esm_shims
-} from "./chunk-3R73A4XB.js";
+} from "./chunk-ECBHH7LU.js";
 
 // src/tools/web-fetch.ts
 init_esm_shims();
+
+// src/tools/web-fetch-network.ts
+init_esm_shims();
+import { lookup } from "node:dns/promises";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { BlockList, isIP } from "node:net";
+import { Readable } from "node:stream";
+import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
+var blocked = new BlockList();
+for (const [address, prefix] of [
+  ["0.0.0.0", 8],
+  ["10.0.0.0", 8],
+  ["100.64.0.0", 10],
+  ["127.0.0.0", 8],
+  ["169.254.0.0", 16],
+  ["172.16.0.0", 12],
+  ["192.168.0.0", 16],
+  ["192.0.0.0", 24],
+  ["192.0.2.0", 24],
+  ["198.18.0.0", 15],
+  ["198.51.100.0", 24],
+  ["203.0.113.0", 24],
+  ["224.0.0.0", 4],
+  ["240.0.0.0", 4]
+])
+  blocked.addSubnet(address, prefix, "ipv4");
+for (const [address, prefix] of [
+  ["2001::", 32],
+  ["2001:db8::", 32],
+  ["2002::", 16]
+])
+  blocked.addSubnet(address, prefix, "ipv6");
+var globalV6 = new BlockList();
+globalV6.addSubnet("2000::", 3, "ipv6");
+function isPublicWebAddress(address) {
+  const family = isIP(address);
+  if (family === 4) return !blocked.check(address, "ipv4");
+  return family === 6 && globalV6.check(address, "ipv6") && !blocked.check(address, "ipv6");
+}
+var WebFetchNetworkError = class extends Error {
+  constructor(message) {
+    super(`WEB_FETCH_PRIVATE_NETWORK: ${message}`);
+    this.name = "WebFetchNetworkError";
+  }
+};
+async function requestPinnedWebUrl(url, init, address) {
+  return new Promise((resolve, reject) => {
+    const headers = new Headers(init.headers);
+    headers.delete("host");
+    const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(
+      url,
+      {
+        method: init.method ?? "GET",
+        headers: Object.fromEntries(headers),
+        agent: false,
+        ...init.signal ? { signal: init.signal } : {},
+        lookup: (_hostname, options, callback) => {
+          if (options.all) callback(null, [address]);
+          else callback(null, address.address, address.family);
+        }
+      },
+      (response) => {
+        const responseHeaders = new Headers();
+        for (const [key, value] of Object.entries(response.headers)) {
+          if (value !== void 0) responseHeaders.set(key, Array.isArray(value) ? value.join(", ") : value);
+        }
+        const status = response.statusCode ?? 500;
+        let stream = response;
+        const encoding = responseHeaders.get("content-encoding");
+        const decoder = encoding === "gzip" ? createGunzip() : encoding === "br" ? createBrotliDecompress() : encoding === "deflate" ? createInflate() : null;
+        if (decoder) {
+          response.on("error", (error) => decoder.destroy(error));
+          decoder.on("close", () => response.destroy());
+          stream = response.pipe(decoder);
+          responseHeaders.delete("content-encoding");
+          responseHeaders.delete("content-length");
+        }
+        const noBody = init.method === "HEAD" || [204, 205, 304].includes(status);
+        if (noBody) response.resume();
+        const result = new Response(noBody ? null : Readable.toWeb(stream), {
+          status,
+          statusText: response.statusMessage ?? "",
+          headers: responseHeaders
+        });
+        Object.defineProperty(result, "url", { value: url.toString() });
+        resolve(result);
+      }
+    );
+    request.on("error", reject);
+    request.end(typeof init.body === "string" ? init.body : void 0);
+  });
+}
+async function abortableLookup(pending, signal) {
+  if (!signal) return pending;
+  signal.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+    pending.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
+  });
+}
+async function fetchWebUrl(input, initial = {}, dependencies = {}) {
+  let url = new URL(input);
+  const init = { ...initial, headers: new Headers(initial.headers) };
+  const allowPrivate = dependencies.allowPrivateNetwork ?? process.env[WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV] === "1";
+  for (let hop = 0; hop <= 10; hop++) {
+    init.signal?.throwIfAborted();
+    if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
+      throw new Error("web_fetch: unsupported URL scheme or embedded credentials");
+    const hostname = url.hostname.replace(/^\[|\]$/g, "");
+    const addresses = isIP(hostname) ? [{ address: hostname, family: isIP(hostname) }] : await abortableLookup((dependencies.resolve ?? ((host) => lookup(host, { all: true })))(hostname), init.signal);
+    init.signal?.throwIfAborted();
+    const address = addresses[0];
+    if (address === void 0) throw new Error("web_fetch: DNS returned no addresses");
+    if (!allowPrivate && addresses.some((address2) => !isPublicWebAddress(address2.address)))
+      throw new WebFetchNetworkError(
+        `destination ${url.hostname} is not public; operator opt-in ${WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV}=1 is required`
+      );
+    const response = await (dependencies.request ?? requestPinnedWebUrl)(url, init, address);
+    const location = response.headers.get("location");
+    if (![301, 302, 303, 307, 308].includes(response.status) || location === null) return response;
+    await response.body?.cancel();
+    if (hop === 10) throw new Error("web_fetch: too many redirects");
+    const next = new URL(location, url);
+    if (next.origin !== url.origin) {
+      init.headers = new Headers();
+    }
+    if (response.status === 303 && init.method !== "HEAD" || [301, 302].includes(response.status) && init.method === "POST") {
+      init.method = "GET";
+      delete init.body;
+      init.headers.delete("content-type");
+      init.headers.delete("content-length");
+    }
+    url = next;
+  }
+  throw new Error("web_fetch: too many redirects");
+}
+
+// src/tools/web-fetch.ts
 var DEFAULT_TIMEOUT_MS = 3e4;
 var DEFAULT_MAX_BYTES = 6e5;
 var HARD_MAX_BYTES = 5e6;
@@ -298,7 +444,7 @@ function parseArxivEntry(entry, fallbackId) {
 }
 async function fetchArxivPaperSummary(paperId, init, maxBytes) {
   const metadataUrl = `https://export.arxiv.org/api/query?id_list=${encodeURIComponent(paperId)}`;
-  const response = await fetch(metadataUrl, init);
+  const response = await fetchWebUrl(metadataUrl, init);
   if (response.status < 200 || response.status >= 300) return null;
   const read = await readResponseText(response, Math.min(12e4, maxBytes));
   const entry = /<entry\b[^>]*>([\s\S]*?)<\/entry>/i.exec(read.text)?.[1];
@@ -307,7 +453,7 @@ async function fetchArxivPaperSummary(paperId, init, maxBytes) {
   let alphaxivRead = null;
   const alphaUrl = `https://alphaxiv.org/overview/${encodeURIComponent(paperId)}.md`;
   try {
-    const alpha = await fetch(alphaUrl, init);
+    const alpha = await fetchWebUrl(alphaUrl, init);
     if (alpha.status >= 200 && alpha.status < 300) {
       alphaxivRead = await readResponseText(alpha, Math.min(8e4, Math.max(0, maxBytes - read.bytesRead)));
       if (alphaxivRead.text.trim().length > 0) paper.alphaxivOverview = alphaxivRead.text.trim();
@@ -337,7 +483,8 @@ async function fetchArxivPaperSummary(paperId, init, maxBytes) {
   const bytesRead = read.bytesRead + (alphaxivRead?.bytesRead ?? 0);
   return {
     kind: "ok",
-    output: truncate(lines.join("\n"), maxBytes),
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, maxBytes),
     details: {
       url: `https://arxiv.org/abs/${paper.id}`,
       status: 200,
@@ -352,7 +499,7 @@ function arxivApiUrl(url) {
   return url.hostname === "export.arxiv.org" && url.pathname === "/api/query";
 }
 async function fetchArxivApiSummary(url, init, maxBytes) {
-  const response = await fetch(url, init);
+  const response = await fetchWebUrl(url, init);
   if (response.status < 200 || response.status >= 300) return null;
   const read = await readResponseText(response, Math.min(24e4, maxBytes));
   const entries = Array.from(read.text.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/gi)).map((match, index) => match[1] ? parseArxivEntry(match[1], `entry-${index + 1}`) : null).filter((entry) => entry !== null);
@@ -382,7 +529,8 @@ async function fetchArxivApiSummary(url, init, maxBytes) {
   }
   return {
     kind: "ok",
-    output: truncate(lines.join("\n"), maxBytes),
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, maxBytes),
     details: {
       url: url.toString(),
       status: 200,
@@ -447,7 +595,7 @@ function importantRepoFiles(entries) {
   return selected;
 }
 async function fetchRepoTreeSummary(target, init, maxBytes) {
-  const apiResponse = await fetch(target.apiUrl, init);
+  const apiResponse = await fetchWebUrl(target.apiUrl, init);
   if (apiResponse.status < 200 || apiResponse.status >= 300) return null;
   const apiRead = await readResponseText(apiResponse, Math.min(2e5, maxBytes));
   let entries;
@@ -472,7 +620,7 @@ async function fetchRepoTreeSummary(target, init, maxBytes) {
   for (const file of files) {
     if (remaining <= 2e3) break;
     const rawUrl = `${target.rawBaseUrl}/${file}`;
-    const rawResponse = await fetch(rawUrl, init);
+    const rawResponse = await fetchWebUrl(rawUrl, init);
     if (rawResponse.status < 200 || rawResponse.status >= 300) continue;
     const rawRead = await readResponseText(rawResponse, Math.min(8e4, remaining));
     sections.push("", `--- ${file} ---`, rawRead.text.trim());
@@ -480,7 +628,8 @@ async function fetchRepoTreeSummary(target, init, maxBytes) {
   }
   return {
     kind: "ok",
-    output: truncate(sections.join("\n"), maxBytes),
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${sections.join("\n")}`, maxBytes),
     details: {
       url: target.apiUrl,
       status: 200,
@@ -503,7 +652,8 @@ function formatOutput(args) {
   if (args.extracted.description) lines.push(`Description: ${args.extracted.description}`);
   if (args.extracted.canonical) lines.push(`Canonical: ${args.extracted.canonical}`);
   lines.push("", "Content:", args.extracted.content || "[empty]");
-  return truncate(lines.join("\n"), args.maxBytes);
+  return truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, args.maxBytes);
 }
 var webFetchTool = {
   ...webFetchToolSurface,
@@ -556,10 +706,10 @@ var webFetchTool = {
         method,
         headers,
         signal: controller.signal,
-        redirect: "follow"
+        redirect: "manual"
       };
       if (body !== void 0) init.body = body;
-      if (method === "GET" && body === void 0 && format !== "raw") {
+      if (method === "GET" && body === void 0 && format !== "raw" && Object.keys(userHeaders).length === 0) {
         if (arxivApiUrl(parsed)) {
           const arxivApiSummary = await fetchArxivApiSummary(parsed, init, maxBytes);
           if (arxivApiSummary) return arxivApiSummary;
@@ -575,7 +725,7 @@ var webFetchTool = {
           if (repoSummary) return repoSummary;
         }
       }
-      const response = await fetch(parsed, init);
+      const response = await fetchWebUrl(parsed, init);
       const contentType = headerValue(response, "content-type");
       const readLimit = response.status >= 200 && response.status < 300 ? maxBytes : Math.min(ERROR_PREVIEW_BYTES, maxBytes);
       const read = await readResponseText(response, readLimit);
@@ -584,6 +734,7 @@ var webFetchTool = {
         return {
           kind: "error",
           message: `web_fetch: HTTP ${response.status}: ${response.statusText}${preview ? `
+${UNTRUSTED_CONTENT_BANNER}
 Preview:
 ${truncate(preview, ERROR_PREVIEW_BYTES)}` : ""}`
         };
@@ -634,4 +785,4 @@ ${truncate(preview, ERROR_PREVIEW_BYTES)}` : ""}`
 export {
   webFetchTool
 };
-//# sourceMappingURL=web-fetch-V4FKSDAV.js.map
+//# sourceMappingURL=web-fetch-MQN4FH3X.js.map