gsd-pi 2.33.1 → 2.34.0-dev.bbb5216

package/dist/resources/extensions/search-the-web/tool-fetch-page.ts

@@ -1,589 +0,0 @@
-/**
- * fetch_page tool — Extract clean markdown from any URL.
- *
- * v3 improvements:
- * - offset parameter for continuation reading (like file read offsets)
- * - selector parameter for Jina's X-Target-Selector (extract specific sections)
- * - Jina failure diagnostics surfaced in details
- * - Content-type awareness (JSON passthrough, PDF detection)
- */
-
-import type { ExtensionAPI } from "@gsd/pi-coding-agent";
-import { truncateHead, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "@gsd/pi-coding-agent";
-import { Text } from "@gsd/pi-tui";
-import { Type } from "@sinclair/typebox";
-
-import { LRUTTLCache } from "./cache.js";
-import { fetchSimple, HttpError } from "./http.js";
-import { extractDomain, isBlockedUrl } from "./url-utils.js";
-import { formatPageContent, type FormatPageOptions } from "./format.js";
-import { getOllamaApiKey } from "./provider.js";
-
-// =============================================================================
-// Cache
-// =============================================================================
-
-interface CachedPage {
-  content: string;
-  title?: string;
-  source: "jina" | "direct";
-}
-
-// Page content cache: max 30 entries, 15-minute TTL
-const pageCache = new LRUTTLCache<CachedPage>({ max: 30, ttlMs: 900_000 });
-pageCache.startPurgeInterval(120_000);
-
-// =============================================================================
-// Jina Reader
-// =============================================================================
-
-/**
- * Fetch page content via Jina Reader API.
- * Returns content + metadata, or throws with a descriptive error.
- */
-async function fetchViaJina(
-  url: string,
-  options: { signal?: AbortSignal; selector?: string } = {}
-): Promise<{ content: string; title?: string }> {
-  const jinaUrl = `https://r.jina.ai/${url}`;
-
-  const headers: Record<string, string> = {
-    "Accept": "text/plain",
-    "X-Return-Format": "markdown",
-    "X-No-Cache": "false",
-  };
-
-  // Use Jina API key if available for higher rate limits
-  const jinaKey = process.env.JINA_API_KEY;
-  if (jinaKey) {
-    headers["Authorization"] = `Bearer ${jinaKey}`;
-  }
-
-  // Target specific CSS selector on the page
-  if (options.selector) {
-    headers["X-Target-Selector"] = options.selector;
-  }
-
-  const response = await fetchSimple(jinaUrl, {
-    method: "GET",
-    headers,
-    signal: options.signal,
-    timeoutMs: 20_000,
-  });
-
-  const text = await response.text();
-
-  // Jina returns markdown with a title line at the top
-  // Format: "Title: <title>\nURL Source: <url>\n\n<content>"
-  let title: string | undefined;
-  let content = text;
-
-  const titleMatch = text.match(/^Title:\s*(.+)\n/);
-  if (titleMatch) {
-    title = titleMatch[1].trim();
-    content = text.replace(/^Title:\s*.+\n/, "");
-  }
-
-  // Strip the URL Source line
-  content = content.replace(/^URL Source:\s*.+\n\n?/, "");
-
-  // Strip Markdown images to save tokens
-  content = content.replace(/!\[([^\]]*)\]\([^)]+\)/g, "");
-
-  // Collapse excessive whitespace
-  content = content.replace(/\n{4,}/g, "\n\n\n");
-
-  return { content: content.trim(), title };
-}
-
-/**
- * Basic fallback: fetch raw HTML and do crude text extraction.
- */
-async function fetchDirectFallback(
-  url: string,
-  signal?: AbortSignal
-): Promise<{ content: string; title?: string; contentType?: string }> {
-  const response = await fetchSimple(url, {
-    method: "GET",
-    headers: {
-      "Accept": "text/html,application/xhtml+xml,application/json,text/plain",
-      "User-Agent": "Mozilla/5.0 (compatible; pi-coding-agent/1.0)",
-    },
-    signal,
-    timeoutMs: 15_000,
-  });
-
-  const contentType = response.headers.get("content-type") || "";
-
-  // JSON passthrough — return formatted JSON directly
-  if (contentType.includes("application/json")) {
-    const text = await response.text();
-    try {
-      const parsed = JSON.parse(text);
-      return {
-        content: "```json\n" + JSON.stringify(parsed, null, 2) + "\n```",
-        title: undefined,
-        contentType: "application/json",
-      };
-    } catch {
-      return { content: text, title: undefined, contentType };
-    }
-  }
-
-  // Plain text passthrough
-  if (contentType.includes("text/plain")) {
-    const text = await response.text();
-    return { content: text, title: undefined, contentType: "text/plain" };
-  }
-
-  // PDF detection — can't extract, but tell the agent
-  if (contentType.includes("application/pdf")) {
-    return {
-      content: "[This URL is a PDF document. Content extraction is not supported for PDFs.]",
-      title: undefined,
-      contentType: "application/pdf",
-    };
-  }
-
-  const html = await response.text();
-
-  // Extract title
-  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-  const title = titleMatch ? titleMatch[1].trim() : undefined;
-
-  // Strip tags, decode entities, collapse whitespace
-  let text = html
-    .replace(/<script[\s\S]*?<\/script>/gi, "")
-    .replace(/<style[\s\S]*?<\/style>/gi, "")
-    .replace(/<nav[\s\S]*?<\/nav>/gi, "")
-    .replace(/<header[\s\S]*?<\/header>/gi, "")
-    .replace(/<footer[\s\S]*?<\/footer>/gi, "")
-    .replace(/<\/?(p|div|br|h[1-6]|li|tr|blockquote|pre|section|article)[^>]*>/gi, "\n")
-    .replace(/<[^>]+>/g, " ")
-    .replace(/&amp;/g, "&")
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&nbsp;/g, " ")
-    .replace(/[ \t]+/g, " ")
-    .replace(/\n[ \t]+/g, "\n")
-    .replace(/\n{3,}/g, "\n\n")
-    .trim();
-
-  return { content: text, title, contentType };
-}
-
-// =============================================================================
-// Ollama Web Fetch
-// =============================================================================
-
-interface OllamaWebFetchResponse {
-  title?: string;
-  content?: string;
-  links?: string[];
-}
-
-/**
- * Fetch page content via Ollama web_fetch API.
- * Returns content + metadata, or throws on failure.
- */
-async function fetchViaOllama(
-  url: string,
-  signal?: AbortSignal,
-): Promise<{ content: string; title?: string }> {
-  const response = await fetchSimple("https://ollama.com/api/web_fetch", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${getOllamaApiKey()}`,
-    },
-    body: JSON.stringify({ url }),
-    signal,
-    timeoutMs: 20_000,
-  });
-
-  const data: OllamaWebFetchResponse = await response.json();
-
-  const content = (data.content || "").trim();
-  const title = data.title?.trim() || undefined;
-
-  return { content, title };
-}
-
-// =============================================================================
-// Smart Truncation
-// =============================================================================
-
-/**
- * Truncate page content to a target character count, trying to break
- * at paragraph boundaries rather than mid-sentence.
- */
-function smartTruncate(
-  content: string,
-  maxChars: number,
-  offset: number = 0
-): { content: string; truncated: boolean; hasMore: boolean; nextOffset?: number } {
-  // Apply offset first
-  const sliced = offset > 0 ? content.slice(offset) : content;
-
-  if (sliced.length <= maxChars) {
-    return { content: sliced, truncated: false, hasMore: false };
-  }
-
-  // Find the last paragraph break before maxChars
-  const window = sliced.slice(0, maxChars);
-  const lastParagraph = window.lastIndexOf("\n\n");
-  const lastSentence = window.lastIndexOf(". ");
-  const lastNewline = window.lastIndexOf("\n");
-
-  // Prefer paragraph > sentence > newline > hard cut
-  let cutPoint = maxChars;
-  if (lastParagraph > maxChars * 0.6) {
-    cutPoint = lastParagraph;
-  } else if (lastSentence > maxChars * 0.6) {
-    cutPoint = lastSentence + 1;
-  } else if (lastNewline > maxChars * 0.6) {
-    cutPoint = lastNewline;
-  }
-
-  const nextOffset = offset + cutPoint;
-  const hasMore = nextOffset < content.length;
-
-  return {
-    content: sliced.slice(0, cutPoint).trim() + "\n\n[... content truncated]",
-    truncated: true,
-    hasMore,
-    nextOffset: hasMore ? nextOffset : undefined,
-  };
-}
-
-// =============================================================================
-// Single page fetch (shared between single and multi modes)
-// =============================================================================
-
-interface FetchPageResult {
-  content: string;
-  title?: string;
-  source: "jina" | "direct";
-  jinaError?: string;
-  contentType?: string;
-  originalChars: number;
-}
-
-async function fetchOnePage(
-  url: string,
-  options: { signal?: AbortSignal; selector?: string }
-): Promise<FetchPageResult> {
-  let pageContent: string;
-  let pageTitle: string | undefined;
-  let source: "jina" | "direct" = "jina";
-  let jinaError: string | undefined;
-  let contentType: string | undefined;
-
-  try {
-    const result = await fetchViaJina(url, options);
-    pageContent = result.content;
-    pageTitle = result.title;
-  } catch (err) {
-    // Capture Jina failure reason for diagnostics
-    jinaError = err instanceof HttpError
-      ? `Jina HTTP ${err.statusCode}`
-      : (err as Error).message ?? String(err);
-
-    // Try Ollama web_fetch as intermediate fallback if API key is available
-    const ollamaKey = getOllamaApiKey();
-    if (ollamaKey) {
-      try {
-        const ollamaResult = await fetchViaOllama(url, options.signal);
-        if (ollamaResult.content && ollamaResult.content.length >= 50) {
-          pageContent = ollamaResult.content;
-          pageTitle = ollamaResult.title;
-          source = "direct";
-          return {
-            content: pageContent,
-            title: pageTitle,
-            source,
-            jinaError,
-            contentType,
-            originalChars: pageContent.length,
-          };
-        }
-      } catch {
-        // Ollama fetch failed too — fall through to direct
-      }
-    }
-
-    source = "direct";
-
-    const result = await fetchDirectFallback(url, options.signal);
-    pageContent = result.content;
-    pageTitle = result.title;
-    contentType = result.contentType;
-  }
-
-  return {
-    content: pageContent,
-    title: pageTitle,
-    source,
-    jinaError,
-    contentType,
-    originalChars: pageContent.length,
-  };
-}
-
-// =============================================================================
-// Details Interface
-// =============================================================================
-
-interface FetchPageDetails {
-  url: string;
-  title?: string;
-  charCount: number;
-  originalChars?: number;
-  truncated: boolean;
-  cached: boolean;
-  source?: "jina" | "direct";
-  jinaError?: string;
-  contentType?: string;
-  hasMore?: boolean;
-  nextOffset?: number;
-  offset?: number;
-  selector?: string;
-  error?: string;
-}
-
-// =============================================================================
-// Tool Registration
-// =============================================================================
-
-export function registerFetchPageTool(pi: ExtensionAPI) {
-  pi.registerTool({
-    name: "fetch_page",
-    label: "Fetch Page",
-    description:
-      "Fetch a web page and extract its content as clean markdown. " +
-      "Use this to read the full content of URLs found via search-the-web. " +
-      "Uses Jina Reader for high-quality markdown extraction. " +
-      "Control the amount of content returned with maxChars (default: 8000, max: 30000).",
-    promptSnippet: "Fetch and extract clean content from a web page URL as markdown",
-    promptGuidelines: [
-      "Use fetch_page to read the content of URLs found via search-the-web when you need more detail than snippets provide.",
-      "Start with the default maxChars (8000). Increase only if the first fetch lacks the detail you need.",
-      "For very long pages, use a smaller maxChars and increase if needed — this saves context tokens.",
-      "The extracted content is already clean markdown — no HTML tags, no navigation, no ads.",
-    ],
-    parameters: Type.Object({
-      url: Type.String({ description: "URL to fetch and extract content from" }),
-      maxChars: Type.Optional(
-        Type.Number({
-          minimum: 1000,
-          maximum: 30000,
-          default: 8000,
-          description: "Maximum characters of content to return (default: 8000, max: 30000). Controls context token usage.",
-        })
-      ),
-      offset: Type.Optional(
-        Type.Number({
-          minimum: 0,
-          description: "Character offset to start reading from (for continuation of truncated pages). Use the nextOffset value from a previous fetch_page result.",
-        })
-      ),
-      selector: Type.Optional(
-        Type.String({
-          description: "CSS selector to extract only a specific section of the page (e.g., 'main', 'article', '.api-docs'). Reduces noise and token usage.",
-        })
-      ),
-    }),
-
-    async execute(toolCallId, params, signal, onUpdate, ctx) {
-      if (signal?.aborted) {
-        return { content: [{ type: "text", text: "Fetch cancelled." }], details: undefined as unknown };
-      }
-
-      const maxChars = params.maxChars ?? 8000;
-      const offset = params.offset ?? 0;
-      const url = params.url.trim();
-
-      // Validate URL
-      try {
-        new URL(url);
-      } catch {
-        return {
-          content: [{ type: "text", text: `Invalid URL: ${url}` }],
-          isError: true,
-          details: { error: "Invalid URL", url } satisfies Partial<FetchPageDetails>,
-        };
-      }
-
-      if (isBlockedUrl(url)) {
-        return {
-          content: [{ type: "text", text: `Blocked URL: requests to private/internal addresses are not allowed.` }],
-          isError: true,
-          details: { error: "SSRF blocked", url } satisfies Partial<FetchPageDetails>,
-        };
-      }
-
-      // ------------------------------------------------------------------
-      // Cache lookup (full content cached, offset/truncation applied after)
-      // ------------------------------------------------------------------
-      const cacheKey = params.selector ? `${url}|sel:${params.selector}` : url;
-      const cached = pageCache.get(cacheKey);
-
-      if (cached) {
-        const trunc = smartTruncate(cached.content, maxChars, offset);
-        const opts: FormatPageOptions = {
-          title: cached.title,
-          charCount: trunc.content.length,
-          truncated: trunc.truncated,
-          originalChars: trunc.truncated ? cached.content.length : undefined,
-          hasMore: trunc.hasMore,
-          nextOffset: trunc.nextOffset,
-        };
-        const output = formatPageContent(url, trunc.content, opts);
-
-        const finalTruncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
-        const details: FetchPageDetails = {
-          url,
-          title: cached.title,
-          charCount: trunc.content.length,
-          originalChars: cached.content.length,
-          truncated: trunc.truncated,
-          cached: true,
-          source: cached.source,
-          hasMore: trunc.hasMore,
-          nextOffset: trunc.nextOffset,
-          offset: offset || undefined,
-        };
-        return {
-          content: [{ type: "text", text: finalTruncation.content }],
-          details,
-        };
-      }
-
-      const domain = extractDomain(url);
-      onUpdate?.({ content: [{ type: "text", text: `Fetching ${domain}...` }], details: undefined as unknown });
-
-      // ------------------------------------------------------------------
-      // Fetch page content
-      // ------------------------------------------------------------------
-      let result: FetchPageResult;
-      try {
-        result = await fetchOnePage(url, { signal, selector: params.selector });
-      } catch (err) {
-        const message = err instanceof HttpError
-          ? `HTTP ${err.statusCode}`
-          : (err as Error).message ?? String(err);
-        return {
-          content: [{ type: "text", text: `Failed to fetch ${domain}: ${message}` }],
-          isError: true,
-          details: { error: message, url } satisfies Partial<FetchPageDetails>,
-        };
-      }
-
-      // Check for empty content
-      if (!result.content || result.content.length < 50) {
-        return {
-          content: [{ type: "text", text: `Page at ${domain} returned no extractable content.` }],
-          details: { url, charCount: 0, source: result.source, cached: false, truncated: false, jinaError: result.jinaError } satisfies FetchPageDetails,
-        };
-      }
-
-      // Cache the full content
-      pageCache.set(cacheKey, { content: result.content, title: result.title, source: result.source });
-
-      // Smart truncate with offset
-      const trunc = smartTruncate(result.content, maxChars, offset);
-
-      const opts: FormatPageOptions = {
-        title: result.title,
-        charCount: trunc.content.length,
-        truncated: trunc.truncated,
-        originalChars: trunc.truncated ? result.originalChars : undefined,
-        hasMore: trunc.hasMore,
-        nextOffset: trunc.nextOffset,
-      };
-
-      const output = formatPageContent(url, trunc.content, opts);
-
-      const finalTruncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
-      let content = finalTruncation.content;
-      if (finalTruncation.truncated) {
-        const tempFile = await (pi as any).writeTempFile(output, { prefix: "fetch-page-" });
-        content += `\n\n[Truncated to fit context. Full content: ${tempFile}]`;
-      }
-
-      const details: FetchPageDetails = {
-        url,
-        title: result.title,
-        charCount: trunc.content.length,
-        originalChars: result.originalChars,
-        truncated: trunc.truncated,
-        cached: false,
-        source: result.source,
-        jinaError: result.jinaError,
-        contentType: result.contentType,
-        hasMore: trunc.hasMore,
-        nextOffset: trunc.nextOffset,
-        offset: offset || undefined,
-        selector: params.selector,
-      };
-
-      return {
-        content: [{ type: "text", text: content }],
-        details,
-      };
-    },
-
-    renderCall(args, theme) {
-      const domain = extractDomain(args.url);
-      let text = theme.fg("toolTitle", theme.bold("fetch_page "));
-      text += theme.fg("accent", domain);
-
-      const meta: string[] = [];
-      if (args.maxChars && args.maxChars !== 8000) meta.push(`max ${(args.maxChars / 1000).toFixed(0)}k`);
-      if (args.offset) meta.push(`offset:${args.offset}`);
-      if (args.selector) meta.push(`sel:"${args.selector}"`);
-      if (meta.length > 0) {
-        text += " " + theme.fg("dim", `(${meta.join(", ")})`);
-      }
-
-      return new Text(text, 0, 0);
-    },
-
-    renderResult(result, { expanded }, theme) {
-      const details = result.details as FetchPageDetails | undefined;
-      if (details?.error) {
-        return new Text(theme.fg("error", `✗ ${details.error}`), 0, 0);
-      }
-
-      const domain = extractDomain(details?.url || "");
-      const title = details?.title ? ` — ${details.title}` : "";
-      const chars = details?.charCount ? `${(details.charCount / 1000).toFixed(1)}k chars` : "";
-      const cacheTag = details?.cached ? theme.fg("dim", " [cached]") : "";
-      const sourceTag = details?.source === "direct" ? theme.fg("dim", " [direct]") : "";
-      const truncTag = details?.truncated && details?.originalChars
-        ? theme.fg("dim", ` [${(details.originalChars / 1000).toFixed(0)}k total]`)
-        : "";
-      const moreTag = details?.hasMore && details?.nextOffset
-        ? theme.fg("accent", ` [more→offset:${details.nextOffset}]`)
-        : "";
-      const jinaTag = details?.jinaError
-        ? theme.fg("warning", ` [jina failed: ${details.jinaError}]`)
-        : "";
-
-      let text = theme.fg("success", `✓ ${domain}${title}`) + ` ${chars}` +
-        cacheTag + sourceTag + truncTag + moreTag + jinaTag;
-
-      if (expanded) {
-        const content = result.content[0];
-        if (content?.type === "text") {
-          const preview = content.text.split("\n").slice(0, 8).join("\n");
-          text += "\n\n" + theme.fg("dim", preview);
-        }
-      }
-
-      return new Text(text, 0, 0);
-    },
-  });
-}