@iowarp/clio-coder 0.4.9 → 0.5.0

package/dist/web-fetch-PVZNMBOB.js

@@ -0,0 +1,803 @@
+import { createRequire as __clioCreateRequire } from "node:module"; const require = __clioCreateRequire(import.meta.url);
+import {
+  WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV,
+  webFetchToolSurface,
+  webReadToolSurface
+} from "./chunk-KBUTSJSR.js";
+import {
+  UNTRUSTED_CONTENT_BANNER
+} from "./chunk-WXA2CFJO.js";
+import {
+  truncateUtf8
+} from "./chunk-MYI32N4R.js";
+import "./chunk-V4XJWDHR.js";
+import "./chunk-GH5ZR6JP.js";
+import {
+  init_esm_shims
+} from "./chunk-ECBHH7LU.js";
+
+// src/tools/web-fetch.ts
+init_esm_shims();
+
+// src/tools/web-fetch-network.ts
+init_esm_shims();
+import { lookup } from "node:dns/promises";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { BlockList, isIP } from "node:net";
+import { Readable } from "node:stream";
+import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
+var blocked = new BlockList();
+for (const [address, prefix] of [
+  ["0.0.0.0", 8],
+  ["10.0.0.0", 8],
+  ["100.64.0.0", 10],
+  ["127.0.0.0", 8],
+  ["169.254.0.0", 16],
+  ["172.16.0.0", 12],
+  ["192.168.0.0", 16],
+  ["192.0.0.0", 24],
+  ["192.0.2.0", 24],
+  ["198.18.0.0", 15],
+  ["198.51.100.0", 24],
+  ["203.0.113.0", 24],
+  ["224.0.0.0", 4],
+  ["240.0.0.0", 4]
+])
+  blocked.addSubnet(address, prefix, "ipv4");
+for (const [address, prefix] of [
+  ["2001::", 32],
+  ["2001:db8::", 32],
+  ["2002::", 16]
+])
+  blocked.addSubnet(address, prefix, "ipv6");
+var globalV6 = new BlockList();
+globalV6.addSubnet("2000::", 3, "ipv6");
+function isPublicWebAddress(address) {
+  const family = isIP(address);
+  if (family === 4) return !blocked.check(address, "ipv4");
+  return family === 6 && globalV6.check(address, "ipv6") && !blocked.check(address, "ipv6");
+}
+var WebFetchNetworkError = class extends Error {
+  constructor(message) {
+    super(`WEB_FETCH_PRIVATE_NETWORK: ${message}`);
+    this.name = "WebFetchNetworkError";
+  }
+};
+async function requestPinnedWebUrl(url, init, address) {
+  return new Promise((resolve, reject) => {
+    const headers = new Headers(init.headers);
+    headers.delete("host");
+    const request = (url.protocol === "https:" ? httpsRequest : httpRequest)(
+      url,
+      {
+        method: init.method ?? "GET",
+        headers: Object.fromEntries(headers),
+        agent: false,
+        ...init.signal ? { signal: init.signal } : {},
+        lookup: (_hostname, options, callback) => {
+          if (options.all) callback(null, [address]);
+          else callback(null, address.address, address.family);
+        }
+      },
+      (response) => {
+        const responseHeaders = new Headers();
+        for (const [key, value] of Object.entries(response.headers)) {
+          if (value !== void 0) responseHeaders.set(key, Array.isArray(value) ? value.join(", ") : value);
+        }
+        const status = response.statusCode ?? 500;
+        let stream = response;
+        const encoding = responseHeaders.get("content-encoding");
+        const decoder = encoding === "gzip" ? createGunzip() : encoding === "br" ? createBrotliDecompress() : encoding === "deflate" ? createInflate() : null;
+        if (decoder) {
+          response.on("error", (error) => decoder.destroy(error));
+          decoder.on("close", () => response.destroy());
+          stream = response.pipe(decoder);
+          responseHeaders.delete("content-encoding");
+          responseHeaders.delete("content-length");
+        }
+        const noBody = init.method === "HEAD" || [204, 205, 304].includes(status);
+        if (noBody) response.resume();
+        const result = new Response(noBody ? null : Readable.toWeb(stream), {
+          status,
+          statusText: response.statusMessage ?? "",
+          headers: responseHeaders
+        });
+        Object.defineProperty(result, "url", { value: url.toString() });
+        resolve(result);
+      }
+    );
+    request.on("error", reject);
+    request.end(typeof init.body === "string" ? init.body : void 0);
+  });
+}
+async function abortableLookup(pending, signal) {
+  if (!signal) return pending;
+  signal.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const onAbort = () => reject(signal.reason);
+    signal.addEventListener("abort", onAbort, { once: true });
+    pending.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
+  });
+}
+async function fetchWebUrl(input, initial = {}, dependencies = {}) {
+  let url = new URL(input);
+  const init = { ...initial, headers: new Headers(initial.headers) };
+  const allowPrivate = dependencies.allowPrivateNetwork ?? process.env[WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV] === "1";
+  for (let hop = 0; hop <= 10; hop++) {
+    init.signal?.throwIfAborted();
+    if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
+      throw new Error("web_fetch: unsupported URL scheme or embedded credentials");
+    const hostname = url.hostname.replace(/^\[|\]$/g, "");
+    const addresses = isIP(hostname) ? [{ address: hostname, family: isIP(hostname) }] : await abortableLookup((dependencies.resolve ?? ((host) => lookup(host, { all: true })))(hostname), init.signal);
+    init.signal?.throwIfAborted();
+    const address = addresses[0];
+    if (address === void 0) throw new Error("web_fetch: DNS returned no addresses");
+    if (!allowPrivate && addresses.some((address2) => !isPublicWebAddress(address2.address)))
+      throw new WebFetchNetworkError(
+        `destination ${url.hostname} is not public; operator opt-in ${WEB_FETCH_ALLOW_PRIVATE_NETWORK_ENV}=1 is required`
+      );
+    const response = await (dependencies.request ?? requestPinnedWebUrl)(url, init, address);
+    const location = response.headers.get("location");
+    if (![301, 302, 303, 307, 308].includes(response.status) || location === null) return response;
+    await response.body?.cancel();
+    if (hop === 10) throw new Error("web_fetch: too many redirects");
+    const next = new URL(location, url);
+    if (next.origin !== url.origin) {
+      init.headers = new Headers();
+    }
+    if (response.status === 303 && init.method !== "HEAD" || [301, 302].includes(response.status) && init.method === "POST") {
+      init.method = "GET";
+      delete init.body;
+      init.headers.delete("content-type");
+      init.headers.delete("content-length");
+    }
+    url = next;
+  }
+  throw new Error("web_fetch: too many redirects");
+}
+
+// src/tools/web-fetch.ts
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_MAX_BYTES = 6e5;
+var HARD_MAX_BYTES = 5e6;
+var ERROR_PREVIEW_BYTES = 8e3;
+var TRUNCATION_MARKER = "\n[output truncated]";
+var HTML_SAMPLE_BYTES = 4096;
+var DEFAULT_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36 clio-coder-web-fetch/1.0",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,text/markdown,text/plain;q=0.8,*/*;q=0.5",
+  "Accept-Language": "en-US,en;q=0.9"
+};
+function parseHeaders(raw) {
+  if (raw === void 0 || raw === null) return {};
+  if (typeof raw !== "object" || Array.isArray(raw)) return null;
+  const out = {};
+  for (const [key, value] of Object.entries(raw)) {
+    if (typeof value !== "string") return null;
+    out[key] = value;
+  }
+  return out;
+}
+function truncate(text, maxBytes) {
+  return truncateUtf8(text, maxBytes, TRUNCATION_MARKER);
+}
+function decodeUtf8Prefix(bytes, maxBytes) {
+  let cut = Math.min(maxBytes, bytes.byteLength);
+  while (cut > 0) {
+    const nextByte = bytes[cut];
+    if (nextByte === void 0 || (nextByte & 192) !== 128) break;
+    cut -= 1;
+  }
+  return bytes.subarray(0, cut).toString("utf8");
+}
+async function readResponseText(response, maxBytes) {
+  if (!response.body) return { text: "", bytesRead: 0, truncated: false };
+  const reader = response.body.getReader();
+  const chunks = [];
+  let totalBytes = 0;
+  let truncated = false;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      const chunk = Buffer.from(value);
+      const remaining = maxBytes + 4 - totalBytes;
+      if (remaining > 0) {
+        const kept = chunk.byteLength > remaining ? chunk.subarray(0, remaining) : chunk;
+        chunks.push(kept);
+        totalBytes += kept.byteLength;
+      }
+      if (totalBytes > maxBytes || chunk.byteLength > remaining) {
+        truncated = true;
+        await reader.cancel();
+        break;
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const bytes = Buffer.concat(chunks, totalBytes);
+  if (truncated || bytes.byteLength > maxBytes) {
+    return {
+      text: `${decodeUtf8Prefix(bytes, maxBytes)}${TRUNCATION_MARKER}`,
+      bytesRead: bytes.byteLength,
+      truncated: true
+    };
+  }
+  return { text: bytes.toString("utf8"), bytesRead: bytes.byteLength, truncated: false };
+}
+function headerValue(response, name) {
+  return response.headers.get(name) ?? "";
+}
+function looksLikeHtml(text) {
+  return /<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>]|<article[\s>]|<main[\s>]/i.test(
+    text.slice(0, HTML_SAMPLE_BYTES)
+  );
+}
+function isProbablyBinary(contentType, text) {
+  if (/^(image|audio|video)\//i.test(contentType)) return true;
+  if (/application\/(pdf|zip|octet-stream|x-tar|gzip)/i.test(contentType)) return true;
+  return text.includes("\0");
+}
+function safeFromCodePoint(code, original) {
+  return Number.isInteger(code) && code >= 0 && code <= 1114111 ? String.fromCodePoint(code) : original;
+}
+function decodeEntities(value) {
+  return value.replace(/&#(\d+);/g, (m, code) => safeFromCodePoint(Number(code), m)).replace(/&#x([0-9a-f]+);/gi, (m, code) => safeFromCodePoint(Number.parseInt(code, 16), m)).replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'");
+}
+function stripTags(value) {
+  return decodeEntities(
+    value.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()
+  );
+}
+function stripCodeTags(value) {
+  return decodeEntities(value.replace(/<[^>]+>/g, ""));
+}
+function firstMatch(html, regex) {
+  const match = regex.exec(html);
+  return match?.[1] ? stripTags(match[1]) : void 0;
+}
+function attrValue(tag, attr) {
+  const regex = new RegExp(`${attr}\\s*=\\s*(["'])(.*?)\\1`, "i");
+  const match = regex.exec(tag);
+  return match?.[2] ? decodeEntities(match[2].trim()) : void 0;
+}
+function absolutizeUrl(href, baseUrl) {
+  try {
+    return new URL(href, baseUrl).toString();
+  } catch {
+    return href;
+  }
+}
+function removeBoilerplate(html) {
+  return html.replace(/<!--[\s\S]*?-->/g, " ").replace(/<script\b[\s\S]*?<\/script>/gi, " ").replace(/<style\b[\s\S]*?<\/style>/gi, " ").replace(/<noscript\b[\s\S]*?<\/noscript>/gi, " ").replace(/<template\b[\s\S]*?<\/template>/gi, " ").replace(/<svg\b[\s\S]*?<\/svg>/gi, " ").replace(/<canvas\b[\s\S]*?<\/canvas>/gi, " ").replace(/<iframe\b[\s\S]*?<\/iframe>/gi, " ").replace(/<form\b[\s\S]*?<\/form>/gi, " ");
+}
+function scoreContent(fragment) {
+  const text = stripTags(fragment);
+  const links = (fragment.match(/<a\b/gi) ?? []).length;
+  const paragraphs = (fragment.match(/<p\b|<li\b|<h[1-6]\b/gi) ?? []).length;
+  return text.length + paragraphs * 80 - links * 20;
+}
+function extractMainHtml(html) {
+  const cleaned = removeBoilerplate(html);
+  const candidates = [];
+  for (const regex of [
+    /<main\b[^>]*>([\s\S]*?)<\/main>/gi,
+    /<article\b[^>]*>([\s\S]*?)<\/article>/gi,
+    /<div\b[^>]*(?:id|class)=["'][^"']*(?:content|main|post|article|entry|docs|markdown)[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi
+  ]) {
+    for (const match of cleaned.matchAll(regex)) {
+      if (match[1]) candidates.push(match[1]);
+    }
+  }
+  if (candidates.length === 0) {
+    const body = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(cleaned);
+    return body?.[1] ?? cleaned;
+  }
+  return candidates.sort((a, b) => scoreContent(b) - scoreContent(a))[0] ?? cleaned;
+}
+function htmlToMarkdown(html, baseUrl) {
+  let out = html;
+  const codeBlocks = [];
+  out = out.replace(/<pre\b[^>]*>([\s\S]*?)<\/pre>/gi, (_, code) => {
+    const placeholder = `
+
+@@CLIO_CODE_${codeBlocks.length}@@
+
+`;
+    codeBlocks.push(`
+
+\`\`\`
+${stripCodeTags(code).replace(/^\n+|\n+$/g, "")}
+\`\`\`
+
+`);
+    return placeholder;
+  });
+  out = out.replace(/<br\s*\/?>/gi, "\n");
+  out = out.replace(
+    /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi,
+    (_, level, text) => `
+
+${"#".repeat(Number(level))} ${stripTags(text)}
+
+`
+  );
+  out = out.replace(
+    /<blockquote\b[^>]*>([\s\S]*?)<\/blockquote>/gi,
+    (_, text) => `
+
+> ${stripTags(text)}
+
+`
+  );
+  out = out.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_, text) => `
+- ${stripTags(text)}`);
+  out = out.replace(
+    /<a\b[^>]*href\s*=\s*(["'])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi,
+    (_, _q, href, text) => {
+      const label = stripTags(text);
+      if (!label) return "";
+      const url = absolutizeUrl(decodeEntities(href), baseUrl);
+      return label === url ? url : `[${label}](${url})`;
+    }
+  );
+  out = out.replace(/<img\b[^>]*>/gi, (tag) => {
+    const alt = attrValue(tag, "alt");
+    const src = attrValue(tag, "src");
+    if (!alt && !src) return " ";
+    if (!alt) return " ";
+    return src ? `![${alt}](${absolutizeUrl(src, baseUrl)})` : alt;
+  });
+  out = out.replace(/<\/(p|div|section|article|main|tr|table|ul|ol)>/gi, "\n\n");
+  out = out.replace(/<\/(td|th)>/gi, " | ");
+  out = out.replace(/<[^>]+>/g, " ");
+  out = decodeEntities(out);
+  for (const [index, block] of codeBlocks.entries()) {
+    out = out.replace(`@@CLIO_CODE_${index}@@`, () => block);
+  }
+  return out.split("\n").map((line) => line.replace(/[ \t]+/g, " ").trimEnd()).join("\n").replace(/\n{3,}/g, "\n\n").replace(/^[ \t]+|[ \t]+$/g, "").trim();
+}
+function extractWebFetchContent(rawText, contentType, finalUrl, format) {
+  const wantsRaw = format === "raw";
+  const isHtml = /\bhtml\b/i.test(contentType) || looksLikeHtml(rawText);
+  if (wantsRaw || !isHtml) {
+    return { format: wantsRaw ? "raw" : "text", content: rawText.trim() };
+  }
+  const title = firstMatch(rawText, /<title\b[^>]*>([\s\S]*?)<\/title>/i);
+  const descriptionTag = /<meta\b[^>]*(?:name|property)=["'](?:description|og:description)["'][^>]*>/i.exec(
+    rawText
+  )?.[0];
+  const canonicalTag = /<link\b[^>]*rel=["']canonical["'][^>]*>/i.exec(rawText)?.[0];
+  const description = descriptionTag ? attrValue(descriptionTag, "content") : void 0;
+  const canonical = canonicalTag ? attrValue(canonicalTag, "href") : void 0;
+  const extracted = {
+    format: "markdown",
+    content: htmlToMarkdown(extractMainHtml(rawText), finalUrl)
+  };
+  if (title) extracted.title = title;
+  if (description) extracted.description = description;
+  if (canonical) extracted.canonical = absolutizeUrl(canonical, finalUrl);
+  return extracted;
+}
+function arxivIdFromUrl(url) {
+  if (url.hostname === "arxiv.org" || url.hostname === "www.arxiv.org") {
+    const match = /^\/(?:abs|pdf|html)\/([^/?#]+?)(?:\.pdf)?$/i.exec(url.pathname);
+    return match?.[1]?.replace(/v\d+$/i, "") ?? null;
+  }
+  if (url.hostname === "alphaxiv.org" || url.hostname === "www.alphaxiv.org") {
+    const match = /^\/(?:abs|overview)\/([^/?#]+?)(?:\.md)?$/i.exec(url.pathname);
+    return match?.[1]?.replace(/v\d+$/i, "") ?? null;
+  }
+  if (url.hostname === "ar5iv.labs.arxiv.org") {
+    const match = /^\/html\/([^/?#]+)$/i.exec(url.pathname);
+    return match?.[1]?.replace(/v\d+$/i, "") ?? null;
+  }
+  return null;
+}
+function xmlText(xml, tag) {
+  const match = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, "i").exec(xml);
+  return match?.[1] ? decodeEntities(
+    match[1].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()
+  ) : void 0;
+}
+function xmlTexts(xml, tag) {
+  const out = [];
+  const regex = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, "gi");
+  for (const match of xml.matchAll(regex)) {
+    if (match[1])
+      out.push(
+        decodeEntities(
+          match[1].replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim()
+        )
+      );
+  }
+  return out;
+}
+function arxivCategories(entryXml) {
+  const categories = [];
+  for (const match of entryXml.matchAll(/<category\b[^>]*term=["']([^"']+)["'][^>]*>/gi)) {
+    if (match[1] && !categories.includes(match[1])) categories.push(match[1]);
+  }
+  return categories;
+}
+function parseArxivEntry(entry, fallbackId) {
+  const idUrl = xmlText(entry, "id");
+  const parsedId = idUrl ? /\/abs\/([^/?#]+)/i.exec(idUrl)?.[1]?.replace(/v\d+$/i, "") : void 0;
+  const paper = {
+    id: parsedId ?? fallbackId,
+    authors: xmlTexts(entry, "name"),
+    categories: arxivCategories(entry)
+  };
+  const title = xmlText(entry, "title");
+  const published = xmlText(entry, "published");
+  const updated = xmlText(entry, "updated");
+  const abstract = xmlText(entry, "summary");
+  if (title) paper.title = title;
+  if (published) paper.published = published;
+  if (updated) paper.updated = updated;
+  if (abstract) paper.abstract = abstract;
+  return paper;
+}
+async function fetchArxivPaperSummary(paperId, init, maxBytes) {
+  const metadataUrl = `https://export.arxiv.org/api/query?id_list=${encodeURIComponent(paperId)}`;
+  const response = await fetchWebUrl(metadataUrl, init);
+  if (response.status < 200 || response.status >= 300) return null;
+  const read = await readResponseText(response, Math.min(12e4, maxBytes));
+  const entry = /<entry\b[^>]*>([\s\S]*?)<\/entry>/i.exec(read.text)?.[1];
+  if (!entry) return null;
+  const paper = parseArxivEntry(entry, paperId);
+  let alphaxivRead = null;
+  const alphaUrl = `https://alphaxiv.org/overview/${encodeURIComponent(paperId)}.md`;
+  try {
+    const alpha = await fetchWebUrl(alphaUrl, init);
+    if (alpha.status >= 200 && alpha.status < 300) {
+      alphaxivRead = await readResponseText(alpha, Math.min(8e4, Math.max(0, maxBytes - read.bytesRead)));
+      if (alphaxivRead.text.trim().length > 0) paper.alphaxivOverview = alphaxivRead.text.trim();
+    }
+  } catch {
+  }
+  const lines = [
+    `URL: https://arxiv.org/abs/${paper.id}`,
+    "Status: 200",
+    "Format: arxiv-paper",
+    `Paper ID: ${paper.id}`
+  ];
+  if (paper.title) lines.push(`Title: ${paper.title}`);
+  if (paper.authors.length > 0) lines.push(`Authors: ${paper.authors.join(", ")}`);
+  if (paper.published) lines.push(`Published: ${paper.published}`);
+  if (paper.updated) lines.push(`Updated: ${paper.updated}`);
+  if (paper.categories.length > 0) lines.push(`Categories: ${paper.categories.join(", ")}`);
+  lines.push(
+    "Links:",
+    `- arXiv: https://arxiv.org/abs/${paper.id}`,
+    `- PDF: https://arxiv.org/pdf/${paper.id}`,
+    `- AlphaXiv: https://alphaxiv.org/abs/${paper.id}`,
+    `- ar5iv HTML: https://ar5iv.labs.arxiv.org/html/${paper.id}`
+  );
+  if (paper.abstract) lines.push("", "Abstract:", paper.abstract);
+  if (paper.alphaxivOverview) lines.push("", "AlphaXiv overview:", paper.alphaxivOverview);
+  const bytesRead = read.bytesRead + (alphaxivRead?.bytesRead ?? 0);
+  return {
+    kind: "ok",
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, maxBytes),
+    details: {
+      url: `https://arxiv.org/abs/${paper.id}`,
+      status: 200,
+      contentType: "application/atom+xml",
+      format: "arxiv-paper",
+      bytesRead,
+      truncated: read.truncated || (alphaxivRead?.truncated ?? false)
+    }
+  };
+}
+function arxivApiUrl(url) {
+  return url.hostname === "export.arxiv.org" && url.pathname === "/api/query";
+}
+async function fetchArxivApiSummary(url, init, maxBytes) {
+  const response = await fetchWebUrl(url, init);
+  if (response.status < 200 || response.status >= 300) return null;
+  const read = await readResponseText(response, Math.min(24e4, maxBytes));
+  const entries = Array.from(read.text.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/gi)).map((match, index) => match[1] ? parseArxivEntry(match[1], `entry-${index + 1}`) : null).filter((entry) => entry !== null);
+  if (entries.length === 0) return null;
+  const total = xmlText(read.text, "opensearch:totalResults") ?? String(entries.length);
+  const query = url.searchParams.get("search_query") ?? url.searchParams.get("id_list") ?? "arxiv query";
+  const lines = [
+    `URL: ${url.toString()}`,
+    "Status: 200",
+    "Format: arxiv-search-results",
+    `Query: ${query}`,
+    `Returned: ${entries.length}`,
+    `Total results: ${total}`,
+    "",
+    "Papers:"
+  ];
+  for (const [index, paper] of entries.entries()) {
+    lines.push("", `## ${index + 1}. ${paper.title ?? paper.id}`);
+    lines.push(`Paper ID: ${paper.id}`);
+    if (paper.authors.length > 0)
+      lines.push(`Authors: ${paper.authors.slice(0, 8).join(", ")}${paper.authors.length > 8 ? ", et al." : ""}`);
+    if (paper.published) lines.push(`Published: ${paper.published}`);
+    if (paper.updated) lines.push(`Updated: ${paper.updated}`);
+    if (paper.categories.length > 0) lines.push(`Categories: ${paper.categories.join(", ")}`);
+    lines.push(`Links: https://arxiv.org/abs/${paper.id} | https://arxiv.org/pdf/${paper.id}`);
+    if (paper.abstract) lines.push("Abstract:", truncate(paper.abstract, 1600));
+  }
+  return {
+    kind: "ok",
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, maxBytes),
+    details: {
+      url: url.toString(),
+      status: 200,
+      contentType: "application/atom+xml",
+      format: "arxiv-search-results",
+      bytesRead: read.bytesRead,
+      truncated: read.truncated
+    }
+  };
+}
+function repoTreeTarget(url) {
+  const parts = url.pathname.split("/").filter(Boolean);
+  if (url.hostname === "github.com" && parts.length >= 5 && parts[2] === "tree") {
+    const [owner, repo, , branch, ...pathParts] = parts;
+    const pathPart = pathParts.join("/");
+    return {
+      kind: "github",
+      apiUrl: `https://api.github.com/repos/${owner}/${repo}/contents/${pathPart}?ref=${branch}`,
+      rawBaseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${pathPart}`.replace(/\/$/, ""),
+      label: `${owner}/${repo}/${pathPart}`
+    };
+  }
+  if (url.hostname.includes("gitlab") && parts.length >= 5) {
+    const treeIndex = parts.findIndex((part, index) => part === "tree" && parts[index - 1] === "-");
+    if (treeIndex >= 2) {
+      const project = parts.slice(0, treeIndex - 1).join("/");
+      const branch = parts[treeIndex + 1];
+      if (!branch) return null;
+      const pathPart = parts.slice(treeIndex + 2).join("/");
+      const encodedProject = encodeURIComponent(project);
+      return {
+        kind: "gitlab",
+        apiUrl: `${url.origin}/api/v4/projects/${encodedProject}/repository/tree?path=${encodeURIComponent(pathPart)}&ref=${encodeURIComponent(branch)}&per_page=100`,
+        rawBaseUrl: `${url.origin}/${project}/-/raw/${branch}/${pathPart}`.replace(/\/$/, ""),
+        label: `${project}/${pathPart}`
+      };
+    }
+  }
+  return null;
+}
+function repoEntryName(entry) {
+  if (!entry || typeof entry !== "object") return null;
+  const record = entry;
+  return typeof record.name === "string" ? record.name : null;
+}
+function repoEntryIsFile(entry) {
+  if (!entry || typeof entry !== "object") return false;
+  const record = entry;
+  return record.type === "file" || record.type === "blob";
+}
+function importantRepoFiles(entries) {
+  const priority = [/^README\.md$/i, /^SKILL\.md$/i, /^INSTALL\.md$/i, /^VERIFY\.md$/i, /\.md$/i];
+  const names = entries.filter(repoEntryIsFile).map(repoEntryName).filter((name) => name !== null);
+  const selected = [];
+  for (const pattern of priority) {
+    for (const name of names) {
+      if (selected.includes(name) || !pattern.test(name)) continue;
+      selected.push(name);
+      if (selected.length >= 6) return selected;
+    }
+  }
+  return selected;
+}
+async function fetchRepoTreeSummary(target, init, maxBytes) {
+  const apiResponse = await fetchWebUrl(target.apiUrl, init);
+  if (apiResponse.status < 200 || apiResponse.status >= 300) return null;
+  const apiRead = await readResponseText(apiResponse, Math.min(2e5, maxBytes));
+  let entries;
+  try {
+    const parsed = JSON.parse(apiRead.text);
+    entries = Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return null;
+  }
+  const files = importantRepoFiles(entries);
+  const allNames = entries.map(repoEntryName).filter((name) => name !== null).slice(0, 60);
+  const sections = [
+    `URL: ${target.apiUrl}`,
+    "Status: 200",
+    `Format: repository-${target.kind}-tree-summary`,
+    `Repository path: ${target.label}`,
+    "",
+    "Directory entries:",
+    ...allNames.map((name) => `- ${name}`)
+  ];
+  let remaining = maxBytes - Buffer.byteLength(sections.join("\n"));
+  for (const file of files) {
+    if (remaining <= 2e3) break;
+    const rawUrl = `${target.rawBaseUrl}/${file}`;
+    const rawResponse = await fetchWebUrl(rawUrl, init);
+    if (rawResponse.status < 200 || rawResponse.status >= 300) continue;
+    const rawRead = await readResponseText(rawResponse, Math.min(8e4, remaining));
+    sections.push("", `--- ${file} ---`, rawRead.text.trim());
+    remaining -= Buffer.byteLength(rawRead.text);
+  }
+  return {
+    kind: "ok",
+    output: truncate(`${UNTRUSTED_CONTENT_BANNER}
+${sections.join("\n")}`, maxBytes),
+    details: {
+      url: target.apiUrl,
+      status: 200,
+      contentType: "application/json",
+      format: `repository-${target.kind}-tree-summary`,
+      bytesRead: apiRead.bytesRead,
+      truncated: apiRead.truncated
+    }
+  };
+}
+function formatOutput(args) {
+  const lines = [
+    `URL: ${args.url}`,
+    `Status: ${args.status}`,
+    `Content-Type: ${args.contentType || "unknown"}`,
+    `Format: ${args.extracted.format}`,
+    `Bytes read: ${args.bytesRead}${args.truncated ? " (truncated)" : ""}`
+  ];
+  if (args.extracted.title) lines.push(`Title: ${args.extracted.title}`);
+  if (args.extracted.description) lines.push(`Description: ${args.extracted.description}`);
+  if (args.extracted.canonical) lines.push(`Canonical: ${args.extracted.canonical}`);
+  lines.push("", "Content:", args.extracted.content || "[empty]");
+  return truncate(`${UNTRUSTED_CONTENT_BANNER}
+${lines.join("\n")}`, args.maxBytes);
+}
+var webFetchTool = {
+  ...webFetchToolSurface,
+  async run(args, options) {
+    const urlArg = typeof args.url === "string" ? args.url : null;
+    if (!urlArg) return { kind: "error", message: "web_fetch: missing url argument" };
+    let parsed;
+    try {
+      parsed = new URL(urlArg);
+    } catch {
+      return { kind: "error", message: `web_fetch: invalid url: ${urlArg}` };
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+      return { kind: "error", message: `web_fetch: unsupported scheme ${parsed.protocol} (must be http or https)` };
+    }
+    const method = typeof args.method === "string" && args.method.length > 0 ? args.method.toUpperCase() : "GET";
+    const userHeaders = parseHeaders(args.headers);
+    if (userHeaders === null) {
+      return { kind: "error", message: "web_fetch: headers must be a Record<string,string>" };
+    }
+    const headers = { ...DEFAULT_HEADERS, ...userHeaders };
+    const hasBody = typeof args.body === "string";
+    const body = hasBody ? args.body : void 0;
+    const timeoutMs = typeof args.timeout_ms === "number" && args.timeout_ms > 0 ? args.timeout_ms : DEFAULT_TIMEOUT_MS;
+    const requestedMaxBytes = typeof args.max_bytes === "number" && args.max_bytes > 0 ? args.max_bytes : DEFAULT_MAX_BYTES;
+    const maxBytes = Math.min(Math.floor(requestedMaxBytes), HARD_MAX_BYTES);
+    const format = typeof args.format === "string" ? args.format.toLowerCase() : "auto";
+    if (!["auto", "markdown", "raw"].includes(format)) {
+      return { kind: "error", message: "web_fetch: format must be one of auto, markdown, raw" };
+    }
+    const externalSignal = options?.signal;
+    const controller = new AbortController();
+    let externalAborted = false;
+    let timedOut = false;
+    const onExternalAbort = () => {
+      externalAborted = true;
+      controller.abort();
+    };
+    if (externalSignal?.aborted) {
+      onExternalAbort();
+    } else {
+      externalSignal?.addEventListener("abort", onExternalAbort, { once: true });
+    }
+    const timer = setTimeout(() => {
+      timedOut = true;
+      controller.abort();
+    }, timeoutMs);
+    try {
+      const init = {
+        method,
+        headers,
+        signal: controller.signal,
+        redirect: "manual"
+      };
+      if (body !== void 0) init.body = body;
+      if (method === "GET" && body === void 0 && format !== "raw" && Object.keys(userHeaders).length === 0) {
+        if (arxivApiUrl(parsed)) {
+          const arxivApiSummary = await fetchArxivApiSummary(parsed, init, maxBytes);
+          if (arxivApiSummary) return arxivApiSummary;
+        }
+        const arxivId = arxivIdFromUrl(parsed);
+        if (arxivId) {
+          const arxivSummary = await fetchArxivPaperSummary(arxivId, init, maxBytes);
+          if (arxivSummary) return arxivSummary;
+        }
+        const repoTarget = repoTreeTarget(parsed);
+        if (repoTarget) {
+          const repoSummary = await fetchRepoTreeSummary(repoTarget, init, maxBytes);
+          if (repoSummary) return repoSummary;
+        }
+      }
+      const response = await fetchWebUrl(parsed, init);
+      const contentType = headerValue(response, "content-type");
+      const readLimit = response.status >= 200 && response.status < 300 ? maxBytes : Math.min(ERROR_PREVIEW_BYTES, maxBytes);
+      const read = await readResponseText(response, readLimit);
+      if (response.status < 200 || response.status >= 300) {
+        const preview = extractWebFetchContent(read.text, contentType, response.url || parsed.toString(), "auto").content;
+        return {
+          kind: "error",
+          message: `web_fetch: HTTP ${response.status}: ${response.statusText}${preview ? `
+${UNTRUSTED_CONTENT_BANNER}
+Preview:
+${truncate(preview, ERROR_PREVIEW_BYTES)}` : ""}`
+        };
+      }
+      if (isProbablyBinary(contentType, read.text)) {
+        return { kind: "error", message: `web_fetch: binary or unsupported content type: ${contentType || "unknown"}` };
+      }
+      const finalUrl = response.url || parsed.toString();
+      const extracted = extractWebFetchContent(read.text, contentType, finalUrl, format);
+      return {
+        kind: "ok",
+        output: formatOutput({
+          url: finalUrl,
+          status: response.status,
+          contentType,
+          bytesRead: read.bytesRead,
+          truncated: read.truncated,
+          extracted,
+          maxBytes
+        }),
+        details: {
+          url: finalUrl,
+          status: response.status,
+          contentType,
+          format: extracted.format,
+          bytesRead: read.bytesRead,
+          truncated: read.truncated
+        }
+      };
+    } catch (err) {
+      if (err instanceof Error && err.name === "AbortError") {
+        if (externalAborted) {
+          return { kind: "error", message: "web_fetch: request aborted" };
+        }
+        if (timedOut) {
+          return { kind: "error", message: `web_fetch: timeout after ${timeoutMs}ms` };
+        }
+        return { kind: "error", message: "web_fetch: request aborted" };
+      }
+      const msg = err instanceof Error ? err.message : String(err);
+      return { kind: "error", message: `web_fetch: ${msg}` };
+    } finally {
+      clearTimeout(timer);
+      externalSignal?.removeEventListener("abort", onExternalAbort);
+    }
+  }
+};
+var WEB_READ_ARGUMENT_KEYS = ["url", "timeout_ms", "max_bytes", "format"];
+var webReadTool = {
+  ...webReadToolSurface,
+  async run(args, options) {
+    const projected = {};
+    for (const key of WEB_READ_ARGUMENT_KEYS) {
+      if (args[key] !== void 0) projected[key] = args[key];
+    }
+    const result = await webFetchTool.run(projected, options);
+    if (result.kind === "error") {
+      return { ...result, message: result.message.replace(/^web_fetch:/u, "web_read:") };
+    }
+    return result;
+  }
+};
+export {
+  webFetchTool,
+  webReadTool
+};
+//# sourceMappingURL=web-fetch-PVZNMBOB.js.map