@wrongstack/tools 0.310.1 → 0.313.1

package/dist/chunk-4I4LWFOO.js

@@ -0,0 +1,431 @@
+import {
+  guardedFetch
+} from "./chunk-NBDZMMG2.js";
+
+// src/search.ts
+import { FetchError, ToolValidationError } from "@wrongstack/core/types";
+import { expectDefined } from "@wrongstack/core/utils";
+import { toErrorMessage } from "@wrongstack/core/utils";
+var DEFAULT_NUM = 10;
+var MAX_RESULTS = 50;
+var MAX_SNIPPET_CHARS = 300;
+var TIMEOUT_MS = 15e3;
+var CACHE_TTL_MS = 3e5;
+var CACHE_MAX_ENTRIES = 200;
+var cache = /* @__PURE__ */ new Map();
+var searchTool = {
+  name: "search",
+  category: "Search",
+  description: "Perform a web search and return results with title, URL, and snippet. Use this when you need up-to-date external information that is not in the local codebase. Results are cached (5 min TTL) and deduplicated by URL. google and bing are best-effort HTML scrapes that fall back to duckduckgo when they return nothing usable.",
+  usageHint: "Good for: API documentation, error messages, library usage examples, current best practices.\n\n- Prefer specific queries over very broad ones.\n- Results go through the guarded fetch system (same protections as the `fetch` tool).\n- Supports duckduckgo (default), google, and bing sources.\n- Set `skip_cache: true` to force a fresh search.\n- This is often better than the model trying to recall outdated knowledge.",
+  permission: "auto",
+  mutating: false,
+  capabilities: ["net.outbound"],
+  icon: "search",
+  timeoutMs: TIMEOUT_MS,
+  inputSchema: {
+    type: "object",
+    properties: {
+      query: { type: "string", description: "Search query" },
+      num_results: {
+        type: "integer",
+        description: "Number of results (1-50, default 10)",
+        minimum: 1,
+        maximum: MAX_RESULTS
+      },
+      source: {
+        type: "string",
+        enum: ["duckduckgo", "google", "bing"],
+        description: "Search engine to use (default: duckduckgo)"
+      },
+      skip_cache: {
+        type: "boolean",
+        description: "Skip the in-memory cache and force a fresh search (default: false)"
+      }
+    },
+    required: ["query"]
+  },
+  async execute(input, ctx, opts) {
+    let final;
+    const executeStream = searchTool.executeStream;
+    if (!executeStream) throw new Error("searchTool: stream execution unavailable");
+    for await (const ev of executeStream(input, ctx, opts)) {
+      if (ev.type === "final") final = ev.output;
+    }
+    if (!final) throw new Error("search: stream ended without final event");
+    return final;
+  },
+  async *executeStream(input, _ctx, opts) {
+    if (!input?.query || input.query.trim() === "") {
+      throw new ToolValidationError({
+        message: "search: query is required and must be a non-empty string",
+        field: "query"
+      });
+    }
+    const num = Math.max(1, Math.min(input.num_results ?? DEFAULT_NUM, MAX_RESULTS));
+    const source = input.source ?? "duckduckgo";
+    const skipCache = input.skip_cache ?? false;
+    const cacheKey = `${source}:${input.query}`;
+    if (!skipCache) {
+      const entry = cache.get(cacheKey);
+      if (entry && Date.now() - entry.timestamp < CACHE_TTL_MS) {
+        const results = entry.results.map((r) => ({
+          title: r.title,
+          url: r.url,
+          snippet: r.snippet
+        }));
+        yield {
+          type: "log",
+          text: `Cache hit for "${input.query}" (${source})`,
+          data: { source, query: input.query, cached: true }
+        };
+        yield {
+          type: "partial_output",
+          text: `${results.length} cached results from ${entry.source}`,
+          data: { count: results.length, cached: true, source: entry.source }
+        };
+        yield {
+          type: "final",
+          output: {
+            query: input.query,
+            results: results.slice(0, num),
+            source: entry.source,
+            truncated: results.length > num,
+            cached: true
+          }
+        };
+        return;
+      }
+    }
+    yield {
+      type: "log",
+      text: `Querying ${source} for "${input.query}"\u2026`,
+      data: { source, query: input.query, cached: false }
+    };
+    let engine;
+    let effectiveSource = source;
+    switch (source) {
+      case "duckduckgo":
+        engine = await duckduckgoSearch(input.query, opts.signal);
+        break;
+      case "google":
+        engine = await googleSearch(input.query, opts.signal);
+        break;
+      case "bing":
+        engine = await bingSearch(input.query, opts.signal);
+        break;
+      default:
+        throw new ToolValidationError({
+          message: `search: unknown source "${source}"`,
+          field: "source"
+        });
+    }
+    let ranked = rankSearchResults(engine.results, input.query);
+    let engineError = engine.error;
+    if (source !== "duckduckgo" && shouldFallbackToDuckDuckGo(ranked, input.query)) {
+      yield {
+        type: "log",
+        text: `${source} returned no relevant static results; falling back to duckduckgo`,
+        data: { source, fallback: "duckduckgo", query: input.query }
+      };
+      const fallback = await duckduckgoSearch(input.query, opts.signal);
+      ranked = rankSearchResults(fallback.results, input.query);
+      engineError = fallback.error;
+      effectiveSource = "duckduckgo";
+    }
+    const finalResults = ranked.slice(0, num);
+    if (!engineError) {
+      cache.set(cacheKey, { results: ranked, source: effectiveSource, timestamp: Date.now() });
+      pruneCacheEntries();
+    }
+    yield {
+      type: "partial_output",
+      text: engineError ? `search failed: ${engineError}` : `${finalResults.length} results from ${effectiveSource}`,
+      data: { count: finalResults.length, cached: false, source: effectiveSource }
+    };
+    yield {
+      type: "final",
+      output: {
+        query: input.query,
+        results: finalResults.map((r) => ({
+          title: r.title,
+          url: r.url,
+          snippet: r.snippet
+        })),
+        source: effectiveSource,
+        truncated: ranked.length > num,
+        cached: false,
+        ...engineError ? { error: engineError } : {}
+      }
+    };
+  }
+};
+function pruneCacheEntries() {
+  const cutoff = Date.now() - CACHE_TTL_MS * 2;
+  for (const [key, entry] of cache.entries()) {
+    if (entry.timestamp < cutoff) cache.delete(key);
+  }
+  while (cache.size > CACHE_MAX_ENTRIES) {
+    const oldestKey = cache.keys().next().value;
+    if (oldestKey === void 0) break;
+    cache.delete(oldestKey);
+  }
+}
+function __clearSearchCache() {
+  cache.clear();
+}
+function rankSearchResults(results, query) {
+  const seenUrls = /* @__PURE__ */ new Set();
+  const deduped = [];
+  for (const r of results) {
+    const noQuery = r.url.split("?")[0] ?? r.url;
+    const normalized = noQuery.split("#")[0] ?? r.url;
+    if (!seenUrls.has(normalized) && r.url.startsWith("http")) {
+      seenUrls.add(normalized);
+      deduped.push(r);
+    }
+  }
+  return scoreResults(deduped, query);
+}
+function scoreResults(results, query) {
+  const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length > 0);
+  return results.map((r) => {
+    const titleLower = r.title.toLowerCase();
+    const snippetLower = r.snippet.toLowerCase();
+    let score = r.score;
+    for (const term of terms) {
+      if (titleLower.includes(term)) score += 2;
+      if (snippetLower.includes(term)) score += 1;
+    }
+    return { ...r, score };
+  }).sort((a, b) => b.score - a.score);
+}
+function shouldFallbackToDuckDuckGo(results, query) {
+  if (results.length === 0) return true;
+  const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length >= 3);
+  if (terms.length === 0) return false;
+  return !results.some((r) => {
+    const haystack = `${r.title} ${r.url} ${r.snippet}`.toLowerCase();
+    return terms.some((term) => haystack.includes(term));
+  });
+}
+async function duckduckgoSearch(query, signal) {
+  const encoded = encodeURIComponent(query);
+  const url = `https://lite.duckduckgo.com/lite/?q=${encoded}&kd=-1&kl=wt-wt`;
+  try {
+    const response = await fetchWithTimeout(url, signal, TIMEOUT_MS);
+    const html = await response.text();
+    return { results: parseDuckDuckGo(html, MAX_RESULTS) };
+  } catch (err) {
+    console.log(
+      JSON.stringify({ level: "debug", event: "search_failed", query, error: toErrorMessage(err) })
+    );
+    return { results: [], error: `duckduckgo unreachable: ${toErrorMessage(err)}` };
+  }
+}
+function takeFrom(iter, max) {
+  const out = [];
+  for (const item of iter) {
+    if (out.length >= max) break;
+    out.push(item);
+  }
+  return out;
+}
+function parseDuckDuckGo(html, num) {
+  const results = [];
+  const linkRegex = /<a\b([^>]*\bclass=(["'])[^"']*\bresult-link\b[^"']*\2[^>]*)>([\s\S]*?)<\/a>/gi;
+  const snippetRegex = /<([a-z0-9]+)\b([^>]*\bclass=(["'])[^"']*\bresult-snippet\b[^"']*\3[^>]*)>([\s\S]*?)<\/\1>/gi;
+  const linkMatches = takeFrom(
+    [...html.matchAll(linkRegex)].map((m) => {
+      const attrs = expectDefined(m[1]);
+      const href = getHtmlAttr(attrs, "href");
+      const title = stripTags(expectDefined(m[3]));
+      return href && title ? { url: normalizeDuckDuckGoUrl(href), title } : void 0;
+    }).filter((m) => m !== void 0),
+    num
+  );
+  const snippetMatches = takeFrom(
+    [...html.matchAll(snippetRegex)].filter((m) => m[4]).map((m) => stripTags(expectDefined(m[4]))),
+    num
+  );
+  for (let i = 0; i < linkMatches.length && i < num; i++) {
+    const entry = linkMatches[i];
+    if (entry) {
+      results.push({
+        title: entry.title ?? "",
+        url: entry.url ?? "",
+        snippet: capSnippet(snippetMatches[i] ?? ""),
+        score: 1
+      });
+    }
+  }
+  return results;
+}
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function getHtmlAttr(attrs, name) {
+  const escaped = escapeRegex(name);
+  const quoted = new RegExp(`\\b${escaped}\\s*=\\s*(["'])(.*?)\\1`, "i").exec(attrs);
+  if (quoted?.[2]) return decodeHtmlEntities(quoted[2]);
+  const unquoted = new RegExp(`\\b${escaped}\\s*=\\s*([^\\s>]+)`, "i").exec(attrs);
+  return unquoted?.[1] ? decodeHtmlEntities(unquoted[1]) : void 0;
+}
+function normalizeDuckDuckGoUrl(raw) {
+  if (raw.startsWith("//")) return `https:${raw}`;
+  if (!raw.startsWith("/")) return raw;
+  if (!raw.startsWith("/l/")) return raw;
+  try {
+    const url = new URL(raw, "https://duckduckgo.com");
+    const uddg = url.searchParams.get("uddg");
+    return uddg?.startsWith("http") ? uddg : url.toString();
+  } catch {
+    return raw;
+  }
+}
+async function googleSearch(query, signal) {
+  const encoded = encodeURIComponent(query);
+  const url = `https://www.google.com/search?q=${encoded}&hl=en`;
+  try {
+    const html = await fetchWithTimeout(url, signal, TIMEOUT_MS).then((r) => r.text());
+    return { results: parseGoogleResults(html, MAX_RESULTS) };
+  } catch (err) {
+    return { results: [], error: `google unreachable: ${toErrorMessage(err)}` };
+  }
+}
+function parseGoogleResults(html, num) {
+  const results = [];
+  const titleRegex = /<h3[^>]*class="[^"]*DKV84"[^>]*>([^<]+)<\/h3>/gi;
+  const urlRegex = /<cite[^>]*>([^<]+)<\/cite>/gi;
+  const snippetRegex = /<span[^>]*class="[^"]*aXCZ0b[^>]*>([^<]+)<\/span>/gi;
+  const titles = takeFrom(
+    [...html.matchAll(titleRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1]))),
+    num
+  );
+  const urls = takeFrom(
+    [...html.matchAll(urlRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1])).replace(/^\*(https?:\/\/[^\s]+).*$/, "$1")).filter((u) => u.startsWith("http")),
+    num
+  );
+  const snippets = takeFrom(
+    [...html.matchAll(snippetRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1]))),
+    num
+  );
+  for (let i = 0; i < Math.min(titles.length, num); i++) {
+    results.push({
+      title: titles[i] ?? "",
+      url: urls[i] ?? "",
+      snippet: capSnippet(snippets[i] ?? ""),
+      score: 1
+    });
+  }
+  return results;
+}
+async function bingSearch(query, signal) {
+  const encoded = encodeURIComponent(query);
+  const url = `https://www.bing.com/search?q=${encoded}`;
+  try {
+    const html = await fetchWithTimeout(url, signal, TIMEOUT_MS).then((r) => r.text());
+    return { results: parseBingResults(html, MAX_RESULTS) };
+  } catch (err) {
+    return { results: [], error: `bing unreachable: ${toErrorMessage(err)}` };
+  }
+}
+function parseBingResults(html, num) {
+  const results = [];
+  const blocks = [
+    ...html.matchAll(
+      /<li\b[^>]*class=(["'])[^"']*\bb_algo\b[^"']*\1[^>]*>([\s\S]*?)(?=<li\b[^>]*class=(["'])[^"']*\bb_algo\b[^"']*\3|<\/ol>)/gi
+    )
+  ].map((m) => expectDefined(m[2]));
+  const candidates = blocks.length > 0 ? blocks : [html];
+  const entries = takeFrom(
+    candidates.flatMap((block) => {
+      const titleMatch = /<h2[^>]*>\s*<a\b([^>]*)>([\s\S]*?)<\/a>\s*<\/h2>/i.exec(block);
+      if (!titleMatch) return [];
+      const href = getHtmlAttr(expectDefined(titleMatch[1]), "href");
+      const title = stripTags(expectDefined(titleMatch[2]));
+      if (!href || !title) return [];
+      const snippetMatch = /<p\b[^>]*class=(["'])[^"']*\b(?:b_paractl|b_lineclamp\d*)\b[^"']*\1[^>]*>([\s\S]*?)<\/p>/i.exec(
+        block
+      ) ?? /<p\b[^>]*>([\s\S]*?)<\/p>/i.exec(block);
+      const snippet = snippetMatch ? capSnippet(stripTags(expectDefined(snippetMatch.at(-1)))) : "";
+      return [{ url: normalizeBingUrl(href), title, snippet, score: 1 }];
+    }),
+    num
+  );
+  for (let i = 0; i < entries.length; i++) {
+    const entry = entries[i];
+    if (entry) {
+      results.push({
+        title: entry.title ?? "",
+        url: entry.url ?? "",
+        snippet: entry.snippet ?? "",
+        score: 1
+      });
+    }
+  }
+  return results;
+}
+function normalizeBingUrl(raw) {
+  try {
+    const url = new URL(raw);
+    if (url.hostname.endsWith("bing.com") && url.pathname.startsWith("/ck/")) {
+      const encoded = url.searchParams.get("u");
+      const decoded = decodeBingTarget(encoded);
+      if (decoded?.startsWith("http")) return decoded;
+    }
+  } catch {
+  }
+  return raw;
+}
+function decodeBingTarget(encoded) {
+  if (!encoded) return void 0;
+  const payload = encoded.startsWith("a1") ? encoded.slice(2) : encoded;
+  try {
+    const padded = payload.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(payload.length / 4) * 4, "=");
+    return Buffer.from(padded, "base64").toString("utf8");
+  } catch {
+    return void 0;
+  }
+}
+async function fetchWithTimeout(url, signal, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  const fetchSignal = anySignal(signal, controller.signal);
+  try {
+    const res = await guardedFetch(url, 5, fetchSignal, {
+      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    });
+    clearTimeout(timer);
+    return res;
+  } catch (e) {
+    clearTimeout(timer);
+    if (e instanceof FetchError) {
+      throw e;
+    }
+    throw new FetchError({
+      message: `search: failed to fetch ${url}`,
+      status: 0,
+      context: { url },
+      cause: e
+    });
+  }
+}
+function anySignal(...signals) {
+  return AbortSignal.any(signals);
+}
+function stripTags(html) {
+  return decodeHtmlEntities(html.replace(/<[^>]+>/g, "")).trim();
+}
+function capSnippet(snippet) {
+  return snippet.length > MAX_SNIPPET_CHARS ? `${snippet.slice(0, MAX_SNIPPET_CHARS - 1)}\u2026` : snippet;
+}
+function decodeHtmlEntities(text) {
+  return text.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
+}
+
+export {
+  searchTool,
+  __clearSearchCache
+};
+//# sourceMappingURL=chunk-4I4LWFOO.js.map