@tuan_son.dinh/gsd 2.6.0

package/src/resources/extensions/search-the-web/tool-search.ts

@@ -0,0 +1,576 @@
+/**
+ * search-the-web tool — Rich web search with full Brave API support.
+ *
+ * v3 improvements:
+ * - Structured error taxonomy (auth_error, rate_limited, network_error, etc.)
+ * - Spellcheck/query correction surfacing
+ * - Latency tracking in details
+ * - more_results_available from Brave response
+ * - Adaptive snippet budget (fewer results = more snippets each)
+ * - Rate limit info in details
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { truncateHead, formatSize, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "@mariozechner/pi-coding-agent";
+import { Text } from "@mariozechner/pi-tui";
+import { Type } from "@sinclair/typebox";
+import { StringEnum } from "@mariozechner/pi-ai";
+
+import { LRUTTLCache } from "./cache";
+import { fetchWithRetryTimed, fetchWithRetry, classifyError, type RateLimitInfo } from "./http";
+import { normalizeQuery, toDedupeKey, detectFreshness } from "./url-utils";
+import { formatSearchResults, type SearchResultFormatted, type FormatSearchOptions } from "./format";
+import { getTavilyApiKey, resolveSearchProvider } from "./provider";
+import { normalizeTavilyResult, mapFreshnessToTavily, type TavilySearchResponse } from "./tavily";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+interface BraveWebResult {
+  title: string;
+  url: string;
+  description: string;
+  age?: string;
+  page_age?: string;
+  language?: string;
+  extra_snippets?: string[];
+  meta_url?: { scheme?: string; netloc?: string; hostname?: string; path?: string };
+  [key: string]: unknown;
+}
+
+interface BraveSummarizerResponse {
+  type?: string;
+  status?: number;
+  title?: string;
+  summary?: Array<{ type: string; data: string }>;
+  enrichments?: unknown;
+  [key: string]: unknown;
+}
+
+interface BraveSearchResponse {
+  query?: {
+    original?: string;
+    altered?: string;
+    show_strict_warning?: boolean;
+    more_results_available?: boolean;
+    spellcheck_off?: boolean;
+  };
+  web?: {
+    results?: BraveWebResult[];
+  };
+  summarizer?: {
+    key?: string;
+  };
+  [key: string]: unknown;
+}
+
+interface CachedSearchResult {
+  results: SearchResultFormatted[];
+  summarizerKey?: string;
+  summaryText?: string;
+  queryCorrected?: boolean;
+  originalQuery?: string;
+  correctedQuery?: string;
+  moreResultsAvailable?: boolean;
+}
+
+/** Structured details returned from the search tool. */
+interface SearchDetails {
+  query: string;
+  effectiveQuery: string;
+  results: SearchResultFormatted[];
+  count: number;
+  cached: boolean;
+  freshness: string;
+  hasSummary: boolean;
+  latencyMs?: number;
+  rateLimit?: RateLimitInfo;
+  queryCorrected?: boolean;
+  originalQuery?: string;
+  correctedQuery?: string;
+  moreResultsAvailable?: boolean;
+  errorKind?: string;
+  error?: string;
+  retryAfterMs?: number;
+  provider?: 'tavily' | 'brave';
+}
+
+// =============================================================================
+// Caches
+// =============================================================================
+
+// Search results: max 100 entries, 10-minute TTL
+const searchCache = new LRUTTLCache<CachedSearchResult>({ max: 100, ttlMs: 600_000 });
+searchCache.startPurgeInterval(60_000);
+
+// Summarizer responses: max 50 entries, 15-minute TTL
+const summarizerCache = new LRUTTLCache<string>({ max: 50, ttlMs: 900_000 });
+
+// =============================================================================
+// Brave API helpers
+// =============================================================================
+
+function getBraveApiKey(): string {
+  return process.env.BRAVE_API_KEY || "";
+}
+
+function braveHeaders(): Record<string, string> {
+  return {
+    "Accept": "application/json",
+    "Accept-Encoding": "gzip",
+    "X-Subscription-Token": getBraveApiKey(),
+  };
+}
+
+/**
+ * Normalize a Brave result into our formatted result type.
+ */
+function normalizeBraveResult(r: BraveWebResult): SearchResultFormatted {
+  return {
+    title: r.title || "(untitled)",
+    url: r.url,
+    description: r.description || "",
+    age: r.age || r.page_age || undefined,
+    extra_snippets: r.extra_snippets || undefined,
+  };
+}
+
+/**
+ * Deduplicate results by URL (first occurrence wins).
+ */
+function deduplicateResults(results: SearchResultFormatted[]): SearchResultFormatted[] {
+  const seen = new Map<string, SearchResultFormatted>();
+  for (const result of results) {
+    const key = toDedupeKey(result.url);
+    if (key !== null && !seen.has(key)) {
+      seen.set(key, result);
+    }
+  }
+  return Array.from(seen.values());
+}
+
+/**
+ * Fetch AI summary from Brave Summarizer API (best-effort, free).
+ */
+async function fetchSummary(
+  summarizerKey: string,
+  signal?: AbortSignal
+): Promise<string | null> {
+  const cached = summarizerCache.get(summarizerKey);
+  if (cached !== undefined) return cached;
+
+  try {
+    const url = `https://api.search.brave.com/res/v1/summarizer/search?key=${encodeURIComponent(summarizerKey)}&entity_info=false`;
+    const response = await fetchWithRetry(url, {
+      method: "GET",
+      headers: braveHeaders(),
+      signal,
+    }, 1);
+
+    const data: BraveSummarizerResponse = await response.json();
+
+    let summaryText = "";
+    if (data.summary && Array.isArray(data.summary)) {
+      summaryText = data.summary
+        .filter((s) => s.type === "token" || s.type === "text")
+        .map((s) => s.data)
+        .join("");
+    }
+
+    if (summaryText) {
+      summarizerCache.set(summarizerKey, summaryText);
+      return summaryText;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// =============================================================================
+// Tavily API execution
+// =============================================================================
+
+/**
+ * Execute a search against the Tavily API.
+ * Returns a CachedSearchResult with normalized, deduplicated results.
+ */
+async function executeTavilySearch(
+  params: { query: string; freshness: string | null; domain?: string; wantSummary: boolean },
+  signal?: AbortSignal
+): Promise<{ results: CachedSearchResult; latencyMs: number; rateLimit?: RateLimitInfo }> {
+  const requestBody: Record<string, unknown> = {
+    query: params.query,
+    max_results: 10,
+    search_depth: "basic",
+  };
+
+  const tavilyTimeRange = mapFreshnessToTavily(params.freshness);
+  if (tavilyTimeRange) {
+    requestBody.time_range = tavilyTimeRange;
+  }
+
+  if (params.domain) {
+    requestBody.include_domains = [params.domain];
+  }
+
+  if (params.wantSummary) {
+    requestBody.include_answer = true;
+  }
+
+  const timed = await fetchWithRetryTimed("https://api.tavily.com/search", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${getTavilyApiKey()}`,
+    },
+    body: JSON.stringify(requestBody),
+    signal,
+  }, 2);
+
+  const data: TavilySearchResponse = await timed.response.json();
+  const normalized = data.results.map(normalizeTavilyResult);
+  const deduplicated = deduplicateResults(normalized);
+
+  return {
+    results: {
+      results: deduplicated,
+      summaryText: data.answer || undefined,
+      queryCorrected: false,
+      moreResultsAvailable: false,
+    },
+    latencyMs: timed.latencyMs,
+    rateLimit: timed.rateLimit,
+  };
+}
+
+// =============================================================================
+// Tool Registration
+// =============================================================================
+
+export function registerSearchTool(pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "search-the-web",
+    label: "Web Search",
+    description:
+      "Search the web using Brave Search API. Returns top results with titles, URLs, descriptions, " +
+      "extra contextual snippets, result ages, and optional AI summary. " +
+      "Supports freshness filtering, domain filtering, and auto-detects recency-sensitive queries.",
+    promptSnippet: "Search the web for information",
+    promptGuidelines: [
+      "Use this tool when the user asks about current events, facts, or external knowledge not in the codebase.",
+      "Always provide the search query to the user in your response.",
+      "Limit to 3-5 results unless more context is needed.",
+      "Use freshness='week' or 'month' for queries about recent events, releases, or updates.",
+      "Use the fetch_page tool to read the full content of promising URLs from search results.",
+    ],
+    parameters: Type.Object({
+      query: Type.String({ description: "Search query (e.g., 'latest AI news')" }),
+      count: Type.Optional(
+        Type.Number({ minimum: 1, maximum: 10, default: 5, description: "Number of results to return (default: 5)" })
+      ),
+      freshness: Type.Optional(
+        StringEnum(["auto", "day", "week", "month", "year"] as const, {
+          description:
+            "Filter by recency. 'auto' (default) detects from query. 'day'=past 24h, 'week'=past 7d, 'month'=past 30d, 'year'=past 365d.",
+        })
+      ),
+      domain: Type.Optional(
+        Type.String({
+          description: "Limit results to a specific domain (e.g., 'stackoverflow.com', 'github.com')",
+        })
+      ),
+      summary: Type.Optional(
+        Type.Boolean({
+          description: "Request an AI-generated summary of the search results (default: false). Adds latency but provides a concise answer.",
+          default: false,
+        })
+      ),
+    }),
+
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      if (signal?.aborted) {
+        return { content: [{ type: "text", text: "Search cancelled." }] };
+      }
+
+      // ------------------------------------------------------------------
+      // Resolve search provider
+      // ------------------------------------------------------------------
+      const provider = resolveSearchProvider();
+      if (!provider) {
+        return {
+          content: [{ type: "text", text: "Web search unavailable: No search API key is set. Use secure_env_collect to set TAVILY_API_KEY or BRAVE_API_KEY." }],
+          isError: true,
+          details: { errorKind: "auth_error", error: "No search API key set" } satisfies Partial<SearchDetails>,
+        };
+      }
+
+      const count = params.count ?? 5;
+      const wantSummary = params.summary ?? false;
+
+      // ------------------------------------------------------------------
+      // Resolve freshness (shared — Brave format, converted for Tavily later)
+      // ------------------------------------------------------------------
+      let freshness: string | null = null;
+      if (params.freshness && params.freshness !== "auto") {
+        const freshnessMap: Record<string, string> = {
+          day: "pd", week: "pw", month: "pm", year: "py",
+        };
+        freshness = freshnessMap[params.freshness] || null;
+      } else {
+        freshness = detectFreshness(params.query);
+      }
+
+      // ------------------------------------------------------------------
+      // Handle domain filter (provider-specific)
+      // ------------------------------------------------------------------
+      let effectiveQuery = params.query;
+      if (provider === "brave" && params.domain) {
+        if (!effectiveQuery.toLowerCase().includes("site:")) {
+          effectiveQuery = `site:${params.domain} ${effectiveQuery}`;
+        }
+      }
+      // Tavily uses include_domains in request body — no query modification
+
+      // ------------------------------------------------------------------
+      // Cache lookup (provider-prefixed key)
+      // ------------------------------------------------------------------
+      const cacheKey = normalizeQuery(effectiveQuery) + `|f:${freshness || ""}|s:${wantSummary}|p:${provider}`;
+      const cached = searchCache.get(cacheKey);
+
+      if (cached) {
+        const limited = cached.results.slice(0, count);
+
+        let summaryText: string | undefined;
+        if (wantSummary) {
+          if (cached.summaryText) {
+            summaryText = cached.summaryText;
+          } else if (cached.summarizerKey) {
+            summaryText = (await fetchSummary(cached.summarizerKey, signal)) ?? undefined;
+          }
+        }
+
+        const formatOpts: FormatSearchOptions = {
+          cached: true,
+          summary: summaryText,
+          queryCorrected: cached.queryCorrected,
+          originalQuery: cached.originalQuery,
+          correctedQuery: cached.correctedQuery,
+          moreResultsAvailable: cached.moreResultsAvailable,
+        };
+
+        const output = formatSearchResults(params.query, limited, formatOpts);
+
+        const truncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+        let content = truncation.content;
+        if (truncation.truncated) {
+          const tempFile = await pi.writeTempFile(output, { prefix: "web-search-" });
+          content += `\n\n[Truncated: ${truncation.outputLines}/${truncation.totalLines} lines (${formatSize(truncation.outputBytes)}/${formatSize(truncation.totalBytes)}). Full results: ${tempFile}]`;
+        }
+
+        const details: SearchDetails = {
+          query: params.query,
+          effectiveQuery,
+          results: limited,
+          count: limited.length,
+          cached: true,
+          freshness: freshness || "none",
+          hasSummary: !!summaryText,
+          queryCorrected: cached.queryCorrected,
+          originalQuery: cached.originalQuery,
+          correctedQuery: cached.correctedQuery,
+          moreResultsAvailable: cached.moreResultsAvailable,
+          provider,
+        };
+
+        return { content: [{ type: "text", text: content }], details };
+      }
+
+      onUpdate?.({ content: [{ type: "text", text: `Searching for "${params.query}"...` }] });
+
+      try {
+        // ------------------------------------------------------------------
+        // Provider-specific fetch
+        // ------------------------------------------------------------------
+        let searchResult: CachedSearchResult;
+        let latencyMs: number | undefined;
+        let rateLimit: RateLimitInfo | undefined;
+
+        if (provider === "tavily") {
+          const tavilyResult = await executeTavilySearch(
+            { query: params.query, freshness, domain: params.domain, wantSummary },
+            signal
+          );
+          searchResult = tavilyResult.results;
+          latencyMs = tavilyResult.latencyMs;
+          rateLimit = tavilyResult.rateLimit;
+        } else {
+          // ================================================================
+          // BRAVE PATH (unchanged API logic)
+          // ================================================================
+          const url = new URL("https://api.search.brave.com/res/v1/web/search");
+          url.searchParams.append("q", effectiveQuery);
+          url.searchParams.append("count", "10"); // Extra for dedup headroom
+          url.searchParams.append("extra_snippets", "true");
+          url.searchParams.append("text_decorations", "false");
+
+          if (freshness) {
+            url.searchParams.append("freshness", freshness);
+          }
+          if (wantSummary) {
+            url.searchParams.append("summary", "1");
+          }
+
+          const timed = await fetchWithRetryTimed(url.toString(), {
+            method: "GET",
+            headers: braveHeaders(),
+            signal,
+          }, 2);
+
+          const data: BraveSearchResponse = await timed.response.json();
+          const rawResults: BraveWebResult[] = data.web?.results ?? [];
+          const summarizerKey: string | undefined = data.summarizer?.key;
+
+          // Extract spellcheck/correction info
+          const queryInfo = data.query;
+          const queryCorrected = !!(queryInfo?.altered && queryInfo.altered !== queryInfo.original);
+          const originalQuery = queryCorrected ? (queryInfo?.original ?? params.query) : undefined;
+          const correctedQuery = queryCorrected ? queryInfo?.altered : undefined;
+          const moreResultsAvailable = queryInfo?.more_results_available ?? false;
+
+          // Normalize, deduplicate
+          const normalized = rawResults.map(normalizeBraveResult);
+          const deduplicated = deduplicateResults(normalized);
+
+          searchResult = {
+            results: deduplicated,
+            summarizerKey,
+            queryCorrected,
+            originalQuery,
+            correctedQuery,
+            moreResultsAvailable,
+          };
+          latencyMs = timed.latencyMs;
+          rateLimit = timed.rateLimit;
+        }
+
+        // ------------------------------------------------------------------
+        // Shared post-fetch: cache, summary, format, return
+        // ------------------------------------------------------------------
+        searchCache.set(cacheKey, searchResult);
+        const results = searchResult.results.slice(0, count);
+
+        let summaryText: string | undefined;
+        if (wantSummary) {
+          if (searchResult.summaryText) {
+            summaryText = searchResult.summaryText;
+          } else if (searchResult.summarizerKey) {
+            summaryText = (await fetchSummary(searchResult.summarizerKey, signal)) ?? undefined;
+          }
+        }
+
+        const formatOpts: FormatSearchOptions = {
+          summary: summaryText,
+          queryCorrected: searchResult.queryCorrected,
+          originalQuery: searchResult.originalQuery,
+          correctedQuery: searchResult.correctedQuery,
+          moreResultsAvailable: searchResult.moreResultsAvailable,
+        };
+
+        const output = formatSearchResults(params.query, results, formatOpts);
+
+        const truncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+        let content = truncation.content;
+
+        if (truncation.truncated) {
+          const tempFile = await pi.writeTempFile(output, { prefix: "web-search-" });
+          content += `\n\n[Truncated: ${truncation.outputLines}/${truncation.totalLines} lines (${formatSize(truncation.outputBytes)}/${formatSize(truncation.totalBytes)}). Full results: ${tempFile}]`;
+        }
+
+        const details: SearchDetails = {
+          query: params.query,
+          effectiveQuery,
+          results,
+          count: results.length,
+          cached: false,
+          freshness: freshness || "none",
+          hasSummary: !!summaryText,
+          latencyMs,
+          rateLimit,
+          queryCorrected: searchResult.queryCorrected,
+          originalQuery: searchResult.originalQuery,
+          correctedQuery: searchResult.correctedQuery,
+          moreResultsAvailable: searchResult.moreResultsAvailable,
+          provider,
+        };
+
+        return { content: [{ type: "text", text: content }], details };
+      } catch (error) {
+        const classified = classifyError(error);
+        return {
+          content: [{ type: "text", text: `Search failed: ${classified.message}` }],
+          details: {
+            errorKind: classified.kind,
+            error: classified.message,
+            retryAfterMs: classified.retryAfterMs,
+            query: params.query,
+            provider,
+          } satisfies Partial<SearchDetails>,
+          isError: true,
+        };
+      }
+    },
+
+    renderCall(args, theme) {
+      let text = theme.fg("toolTitle", theme.bold("search-the-web "));
+      text += theme.fg("muted", `"${args.query}"`);
+
+      const meta: string[] = [];
+      if (args.count && args.count !== 5) meta.push(`${args.count} results`);
+      if (args.freshness && args.freshness !== "auto") meta.push(`freshness:${args.freshness}`);
+      if (args.domain) meta.push(`site:${args.domain}`);
+      if (args.summary) meta.push("+ summary");
+      if (meta.length > 0) {
+        text += " " + theme.fg("dim", `(${meta.join(", ")})`);
+      }
+
+      return new Text(text, 0, 0);
+    },
+
+    renderResult(result, { expanded }, theme) {
+      const details = result.details as SearchDetails | undefined;
+      if (details?.errorKind || details?.error) {
+        const kindTag = details.errorKind ? theme.fg("dim", ` [${details.errorKind}]`) : "";
+        return new Text(theme.fg("error", `✗ ${details.error ?? "Search failed"}`) + kindTag, 0, 0);
+      }
+
+      const providerTag = details?.provider ? theme.fg("dim", ` [${details.provider}]`) : "";
+      const cacheTag = details?.cached ? theme.fg("dim", " [cached]") : "";
+      const freshTag = details?.freshness && details.freshness !== "none"
+        ? theme.fg("dim", ` [${details.freshness}]`)
+        : "";
+      const summaryTag = details?.hasSummary ? theme.fg("dim", " [+summary]") : "";
+      const latencyTag = details?.latencyMs ? theme.fg("dim", ` ${details.latencyMs}ms`) : "";
+      const correctedTag = details?.queryCorrected
+        ? theme.fg("warning", ` [corrected→"${details.correctedQuery}"]`)
+        : "";
+
+      let text = theme.fg("success", `✓ ${details?.count ?? 0} results for "${details?.query}"`) +
+        providerTag + cacheTag + freshTag + summaryTag + latencyTag + correctedTag;
+
+      if (expanded && details?.results) {
+        text += "\n\n";
+        for (const r of details.results.slice(0, 3)) {
+          const age = r.age ? theme.fg("dim", ` (${r.age})`) : "";
+          text += `${theme.bold(r.title)}${age}\n${r.url}\n${r.description}\n\n`;
+        }
+        if (details.results.length > 3) {
+          text += theme.fg("dim", `... and ${details.results.length - 3} more`);
+        }
+      }
+
+      return new Text(text, 0, 0);
+    },
+  });
+}

package/src/resources/extensions/search-the-web/url-utils.ts

@@ -0,0 +1,91 @@
+/**
+ * URL normalization and query utilities.
+ */
+
+/** Normalize a search query into a stable cache key. */
+export function normalizeQuery(query: string): string {
+  return query.trim().toLowerCase().replace(/\s+/g, " ").normalize("NFC");
+}
+
+/**
+ * Canonical URL for deduplication.
+ * Strips fragment, tracking params, lowercases hostname, sorts query params,
+ * strips trailing "/" on root paths.
+ */
+export function toDedupeKey(url: string): string | null {
+  try {
+    const parsed = new URL(url);
+    parsed.hostname = parsed.hostname.toLowerCase();
+    parsed.hash = "";
+
+    const TRACKING_PARAMS = new Set(["fbclid", "gclid"]);
+    const toDelete: string[] = [];
+    for (const key of parsed.searchParams.keys()) {
+      if (key.startsWith("utm_") || TRACKING_PARAMS.has(key)) {
+        toDelete.push(key);
+      }
+    }
+    for (const key of toDelete) parsed.searchParams.delete(key);
+    parsed.searchParams.sort();
+
+    let canonical = parsed.toString();
+    if (parsed.pathname === "/" && !parsed.search) {
+      canonical = canonical.replace(/\/$/, "");
+    }
+    return canonical;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract a clean domain from a URL for display.
+ * "https://docs.python.org/3/library/asyncio.html" → "docs.python.org"
+ */
+export function extractDomain(url: string): string {
+  try {
+    return new URL(url).hostname.replace(/^www\./, "");
+  } catch {
+    return url;
+  }
+}
+
+/**
+ * Detect if a query likely wants fresh/recent results.
+ * Returns a suggested Brave freshness parameter or null.
+ */
+export function detectFreshness(query: string): string | null {
+  const q = query.toLowerCase();
+
+  // Explicit year references for current/recent years
+  const currentYear = new Date().getFullYear();
+  for (let y = currentYear; y >= currentYear - 1; y--) {
+    if (q.includes(String(y))) return "py"; // past year
+  }
+
+  // Recency keywords
+  const recentPatterns = [
+    /\b(latest|newest|recent|new|just released|just launched)\b/,
+    /\b(today|yesterday|this week|this month)\b/,
+    /\b(breaking|update|announcement|release notes?)\b/,
+    /\b(what('?s| is) new)\b/,
+  ];
+  for (const pattern of recentPatterns) {
+    if (pattern.test(q)) return "pm"; // past month
+  }
+
+  return null;
+}
+
+/**
+ * Detect if a query targets specific domains.
+ * Returns extracted domains or null.
+ */
+export function detectDomainHints(query: string): string[] | null {
+  // Match "site:example.com" patterns
+  const siteMatches = query.match(/site:(\S+)/gi);
+  if (siteMatches) {
+    return siteMatches.map((m) => m.replace(/^site:/i, ""));
+  }
+  return null;
+}