@wrongstack/tools 0.313.0 → 0.313.1

package/dist/search.js

@@ -1,588 +1,8 @@
-// src/search.ts
-import { FetchError as FetchError2, ToolValidationError as ToolValidationError2 } from "@wrongstack/core/types";
-import { expectDefined } from "@wrongstack/core/utils";
-
-// src/_fetch-guard.ts
-import * as dns from "node:dns/promises";
-import * as net from "node:net";
-import { isPrivateIPv4, isPrivateIPv6 } from "@wrongstack/core/utils";
-import { FetchError, ToolValidationError } from "@wrongstack/core/types";
-import { Agent, fetch as undiciFetch } from "undici";
-var nativeGlobalFetch = globalThis.fetch;
-var ALLOW_PRIVATE = process.env["WRONGSTACK_FETCH_ALLOW_PRIVATE"] === "1";
-if (ALLOW_PRIVATE && !process.env["CI"]) {
-  console.warn(
-    "[WrongStack] WARNING: WRONGSTACK_FETCH_ALLOW_PRIVATE=1 is active \u2014\n  fetch tool can now access private IPs (10.x, 192.168.x, 169.254.x),\n  cloud metadata endpoints, and plaintext HTTP. Use only on isolated networks."
-  );
-}
-function guardedLookup(hostname, options, callback) {
-  dns.lookup(hostname, { all: true }).then((records) => {
-    const family = options?.family;
-    const byFamily = family === 4 || family === 6 ? records.filter((r) => r.family === family) : records;
-    const list = byFamily.length > 0 ? byFamily : records;
-    if (!ALLOW_PRIVATE) {
-      for (const r of list) {
-        const bad = r.family === 4 ? isPrivateIPv4(r.address) : isPrivateIPv6(r.address);
-        if (bad) {
-          callback(
-            Object.assign(new Error(`fetch: resolved to private address ${r.address}`), {
-              code: "EAI_FAIL"
-            })
-          );
-          return;
-        }
-      }
-    }
-    if (options?.all) {
-      callback(
-        null,
-        list.map((r) => ({ address: r.address, family: r.family }))
-      );
-      return;
-    }
-    const first = list.at(0);
-    if (!first) {
-      callback(
-        Object.assign(new Error(`fetch: no address for ${hostname}`), { code: "ENOTFOUND" })
-      );
-      return;
-    }
-    callback(null, first.address, first.family);
-  }).catch((err) => callback(err));
-}
-var pinnedAgent;
-function getPinnedDispatcher() {
-  if (!pinnedAgent) {
-    pinnedAgent = new Agent({ allowH2: false, connect: { lookup: guardedLookup } });
-  }
-  return pinnedAgent;
-}
-function dispatcherFetch() {
-  return globalThis.fetch === nativeGlobalFetch ? undiciFetch : globalThis.fetch;
-}
-var _beforeExitRegistered = false;
-if (!_beforeExitRegistered) {
-  _beforeExitRegistered = true;
-  process.on("beforeExit", () => {
-    pinnedAgent?.destroy();
-    pinnedAgent = void 0;
-  });
-}
-async function guardedFetch(url, maxRedirects, signal, headers = {
-  "user-agent": "WrongStack/1.0 (+https://wrongstack.com)",
-  accept: "text/html,application/json;q=0.9,text/plain;q=0.8,*/*;q=0.1"
-}) {
-  let redirectCount = 0;
-  let currentUrl = url;
-  for (; ; ) {
-    const parsed = new URL(currentUrl);
-    if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
-      throw new ToolValidationError({
-        message: `fetch: redirect to unsupported protocol "${parsed.protocol}"`,
-        field: "url"
-      });
-    }
-    if (parsed.protocol === "http:" && !ALLOW_PRIVATE) {
-      throw new ToolValidationError({
-        message: "fetch: redirect to http:// blocked (HTTPS required by default)",
-        field: "url"
-      });
-    }
-    await assertNotPrivate(parsed.hostname);
-    const init = {
-      redirect: "manual",
-      signal,
-      headers,
-      dispatcher: getPinnedDispatcher()
-    };
-    const res = await dispatcherFetch()(currentUrl, init);
-    if (res.status < 300 || res.status > 399) {
-      return res;
-    }
-    try {
-      await res.body?.cancel();
-    } catch {
-    }
-    redirectCount++;
-    if (redirectCount > maxRedirects) {
-      throw new FetchError({
-        message: `fetch: exceeded ${maxRedirects} redirects`,
-        status: res.status,
-        context: { url: currentUrl, maxRedirects, redirectCount }
-      });
-    }
-    const location = res.headers.get("location");
-    if (!location) {
-      throw new FetchError({
-        message: "fetch: redirect status with no location header",
-        status: res.status,
-        context: { url: currentUrl, redirectCount }
-      });
-    }
-    currentUrl = new URL(location, currentUrl).toString();
-  }
-}
-async function assertNotPrivate(hostname) {
-  if (ALLOW_PRIVATE) return;
-  const host = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
-  if (host === "localhost" || host.endsWith(".localhost")) {
-    throw new ToolValidationError({
-      message: "fetch: blocked localhost target",
-      field: "url"
-    });
-  }
-  const ipVersion = net.isIP(host);
-  if (ipVersion === 4) {
-    if (isPrivateIPv4(host)) {
-      throw new ToolValidationError({
-        message: `fetch: blocked private/loopback address "${host}"`,
-        field: "url"
-      });
-    }
-  } else if (ipVersion === 6) {
-    if (isPrivateIPv6(host)) {
-      throw new ToolValidationError({
-        message: `fetch: blocked private/loopback address "${host}"`,
-        field: "url"
-      });
-    }
-  } else {
-    try {
-      const records = await dns.lookup(host, { all: true });
-      for (const r of records) {
-        const bad = r.family === 4 ? isPrivateIPv4(r.address) : isPrivateIPv6(r.address);
-        if (bad) {
-          throw new ToolValidationError({
-            message: `fetch: resolved to private address ${r.address}`,
-            field: "url"
-          });
-        }
-      }
-    } catch (err) {
-      if (err instanceof ToolValidationError) throw err;
-    }
-  }
-}
-
-// src/search.ts
-import { toErrorMessage } from "@wrongstack/core/utils";
-var DEFAULT_NUM = 10;
-var MAX_RESULTS = 50;
-var MAX_SNIPPET_CHARS = 300;
-var TIMEOUT_MS = 15e3;
-var CACHE_TTL_MS = 3e5;
-var CACHE_MAX_ENTRIES = 200;
-var cache = /* @__PURE__ */ new Map();
-var searchTool = {
-  name: "search",
-  category: "Search",
-  description: "Perform a web search and return results with title, URL, and snippet. Use this when you need up-to-date external information that is not in the local codebase. Results are cached (5 min TTL) and deduplicated by URL. google and bing are best-effort HTML scrapes that fall back to duckduckgo when they return nothing usable.",
-  usageHint: "Good for: API documentation, error messages, library usage examples, current best practices.\n\n- Prefer specific queries over very broad ones.\n- Results go through the guarded fetch system (same protections as the `fetch` tool).\n- Supports duckduckgo (default), google, and bing sources.\n- Set `skip_cache: true` to force a fresh search.\n- This is often better than the model trying to recall outdated knowledge.",
-  permission: "auto",
-  mutating: false,
-  capabilities: ["net.outbound"],
-  icon: "search",
-  timeoutMs: TIMEOUT_MS,
-  inputSchema: {
-    type: "object",
-    properties: {
-      query: { type: "string", description: "Search query" },
-      num_results: {
-        type: "integer",
-        description: "Number of results (1-50, default 10)",
-        minimum: 1,
-        maximum: MAX_RESULTS
-      },
-      source: {
-        type: "string",
-        enum: ["duckduckgo", "google", "bing"],
-        description: "Search engine to use (default: duckduckgo)"
-      },
-      skip_cache: {
-        type: "boolean",
-        description: "Skip the in-memory cache and force a fresh search (default: false)"
-      }
-    },
-    required: ["query"]
-  },
-  async execute(input, ctx, opts) {
-    let final;
-    const executeStream = searchTool.executeStream;
-    if (!executeStream) throw new Error("searchTool: stream execution unavailable");
-    for await (const ev of executeStream(input, ctx, opts)) {
-      if (ev.type === "final") final = ev.output;
-    }
-    if (!final) throw new Error("search: stream ended without final event");
-    return final;
-  },
-  async *executeStream(input, _ctx, opts) {
-    if (!input?.query || input.query.trim() === "") {
-      throw new ToolValidationError2({
-        message: "search: query is required and must be a non-empty string",
-        field: "query"
-      });
-    }
-    const num = Math.max(1, Math.min(input.num_results ?? DEFAULT_NUM, MAX_RESULTS));
-    const source = input.source ?? "duckduckgo";
-    const skipCache = input.skip_cache ?? false;
-    const cacheKey = `${source}:${input.query}`;
-    if (!skipCache) {
-      const entry = cache.get(cacheKey);
-      if (entry && Date.now() - entry.timestamp < CACHE_TTL_MS) {
-        const results = entry.results.map((r) => ({
-          title: r.title,
-          url: r.url,
-          snippet: r.snippet
-        }));
-        yield {
-          type: "log",
-          text: `Cache hit for "${input.query}" (${source})`,
-          data: { source, query: input.query, cached: true }
-        };
-        yield {
-          type: "partial_output",
-          text: `${results.length} cached results from ${entry.source}`,
-          data: { count: results.length, cached: true, source: entry.source }
-        };
-        yield {
-          type: "final",
-          output: {
-            query: input.query,
-            results: results.slice(0, num),
-            source: entry.source,
-            truncated: results.length > num,
-            cached: true
-          }
-        };
-        return;
-      }
-    }
-    yield {
-      type: "log",
-      text: `Querying ${source} for "${input.query}"\u2026`,
-      data: { source, query: input.query, cached: false }
-    };
-    let engine;
-    let effectiveSource = source;
-    switch (source) {
-      case "duckduckgo":
-        engine = await duckduckgoSearch(input.query, opts.signal);
-        break;
-      case "google":
-        engine = await googleSearch(input.query, opts.signal);
-        break;
-      case "bing":
-        engine = await bingSearch(input.query, opts.signal);
-        break;
-      default:
-        throw new ToolValidationError2({
-          message: `search: unknown source "${source}"`,
-          field: "source"
-        });
-    }
-    let ranked = rankSearchResults(engine.results, input.query);
-    let engineError = engine.error;
-    if (source !== "duckduckgo" && shouldFallbackToDuckDuckGo(ranked, input.query)) {
-      yield {
-        type: "log",
-        text: `${source} returned no relevant static results; falling back to duckduckgo`,
-        data: { source, fallback: "duckduckgo", query: input.query }
-      };
-      const fallback = await duckduckgoSearch(input.query, opts.signal);
-      ranked = rankSearchResults(fallback.results, input.query);
-      engineError = fallback.error;
-      effectiveSource = "duckduckgo";
-    }
-    const finalResults = ranked.slice(0, num);
-    if (!engineError) {
-      cache.set(cacheKey, { results: ranked, source: effectiveSource, timestamp: Date.now() });
-      pruneCacheEntries();
-    }
-    yield {
-      type: "partial_output",
-      text: engineError ? `search failed: ${engineError}` : `${finalResults.length} results from ${effectiveSource}`,
-      data: { count: finalResults.length, cached: false, source: effectiveSource }
-    };
-    yield {
-      type: "final",
-      output: {
-        query: input.query,
-        results: finalResults.map((r) => ({
-          title: r.title,
-          url: r.url,
-          snippet: r.snippet
-        })),
-        source: effectiveSource,
-        truncated: ranked.length > num,
-        cached: false,
-        ...engineError ? { error: engineError } : {}
-      }
-    };
-  }
-};
-function pruneCacheEntries() {
-  const cutoff = Date.now() - CACHE_TTL_MS * 2;
-  for (const [key, entry] of cache.entries()) {
-    if (entry.timestamp < cutoff) cache.delete(key);
-  }
-  while (cache.size > CACHE_MAX_ENTRIES) {
-    const oldestKey = cache.keys().next().value;
-    if (oldestKey === void 0) break;
-    cache.delete(oldestKey);
-  }
-}
-function __clearSearchCache() {
-  cache.clear();
-}
-function rankSearchResults(results, query) {
-  const seenUrls = /* @__PURE__ */ new Set();
-  const deduped = [];
-  for (const r of results) {
-    const noQuery = r.url.split("?")[0] ?? r.url;
-    const normalized = noQuery.split("#")[0] ?? r.url;
-    if (!seenUrls.has(normalized) && r.url.startsWith("http")) {
-      seenUrls.add(normalized);
-      deduped.push(r);
-    }
-  }
-  return scoreResults(deduped, query);
-}
-function scoreResults(results, query) {
-  const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length > 0);
-  return results.map((r) => {
-    const titleLower = r.title.toLowerCase();
-    const snippetLower = r.snippet.toLowerCase();
-    let score = r.score;
-    for (const term of terms) {
-      if (titleLower.includes(term)) score += 2;
-      if (snippetLower.includes(term)) score += 1;
-    }
-    return { ...r, score };
-  }).sort((a, b) => b.score - a.score);
-}
-function shouldFallbackToDuckDuckGo(results, query) {
-  if (results.length === 0) return true;
-  const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length >= 3);
-  if (terms.length === 0) return false;
-  return !results.some((r) => {
-    const haystack = `${r.title} ${r.url} ${r.snippet}`.toLowerCase();
-    return terms.some((term) => haystack.includes(term));
-  });
-}
-async function duckduckgoSearch(query, signal) {
-  const encoded = encodeURIComponent(query);
-  const url = `https://lite.duckduckgo.com/lite/?q=${encoded}&kd=-1&kl=wt-wt`;
-  try {
-    const response = await fetchWithTimeout(url, signal, TIMEOUT_MS);
-    const html = await response.text();
-    return { results: parseDuckDuckGo(html, MAX_RESULTS) };
-  } catch (err) {
-    console.log(
-      JSON.stringify({ level: "debug", event: "search_failed", query, error: toErrorMessage(err) })
-    );
-    return { results: [], error: `duckduckgo unreachable: ${toErrorMessage(err)}` };
-  }
-}
-function takeFrom(iter, max) {
-  const out = [];
-  for (const item of iter) {
-    if (out.length >= max) break;
-    out.push(item);
-  }
-  return out;
-}
-function parseDuckDuckGo(html, num) {
-  const results = [];
-  const linkRegex = /<a\b([^>]*\bclass=(["'])[^"']*\bresult-link\b[^"']*\2[^>]*)>([\s\S]*?)<\/a>/gi;
-  const snippetRegex = /<([a-z0-9]+)\b([^>]*\bclass=(["'])[^"']*\bresult-snippet\b[^"']*\3[^>]*)>([\s\S]*?)<\/\1>/gi;
-  const linkMatches = takeFrom(
-    [...html.matchAll(linkRegex)].map((m) => {
-      const attrs = expectDefined(m[1]);
-      const href = getHtmlAttr(attrs, "href");
-      const title = stripTags(expectDefined(m[3]));
-      return href && title ? { url: normalizeDuckDuckGoUrl(href), title } : void 0;
-    }).filter((m) => m !== void 0),
-    num
-  );
-  const snippetMatches = takeFrom(
-    [...html.matchAll(snippetRegex)].filter((m) => m[4]).map((m) => stripTags(expectDefined(m[4]))),
-    num
-  );
-  for (let i = 0; i < linkMatches.length && i < num; i++) {
-    const entry = linkMatches[i];
-    if (entry) {
-      results.push({
-        title: entry.title ?? "",
-        url: entry.url ?? "",
-        snippet: capSnippet(snippetMatches[i] ?? ""),
-        score: 1
-      });
-    }
-  }
-  return results;
-}
-function escapeRegex(s) {
-  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-function getHtmlAttr(attrs, name) {
-  const escaped = escapeRegex(name);
-  const quoted = new RegExp(`\\b${escaped}\\s*=\\s*(["'])(.*?)\\1`, "i").exec(attrs);
-  if (quoted?.[2]) return decodeHtmlEntities(quoted[2]);
-  const unquoted = new RegExp(`\\b${escaped}\\s*=\\s*([^\\s>]+)`, "i").exec(attrs);
-  return unquoted?.[1] ? decodeHtmlEntities(unquoted[1]) : void 0;
-}
-function normalizeDuckDuckGoUrl(raw) {
-  if (raw.startsWith("//")) return `https:${raw}`;
-  if (!raw.startsWith("/")) return raw;
-  if (!raw.startsWith("/l/")) return raw;
-  try {
-    const url = new URL(raw, "https://duckduckgo.com");
-    const uddg = url.searchParams.get("uddg");
-    return uddg?.startsWith("http") ? uddg : url.toString();
-  } catch {
-    return raw;
-  }
-}
-async function googleSearch(query, signal) {
-  const encoded = encodeURIComponent(query);
-  const url = `https://www.google.com/search?q=${encoded}&hl=en`;
-  try {
-    const html = await fetchWithTimeout(url, signal, TIMEOUT_MS).then((r) => r.text());
-    return { results: parseGoogleResults(html, MAX_RESULTS) };
-  } catch (err) {
-    return { results: [], error: `google unreachable: ${toErrorMessage(err)}` };
-  }
-}
-function parseGoogleResults(html, num) {
-  const results = [];
-  const titleRegex = /<h3[^>]*class="[^"]*DKV84"[^>]*>([^<]+)<\/h3>/gi;
-  const urlRegex = /<cite[^>]*>([^<]+)<\/cite>/gi;
-  const snippetRegex = /<span[^>]*class="[^"]*aXCZ0b[^>]*>([^<]+)<\/span>/gi;
-  const titles = takeFrom(
-    [...html.matchAll(titleRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1]))),
-    num
-  );
-  const urls = takeFrom(
-    [...html.matchAll(urlRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1])).replace(/^\*(https?:\/\/[^\s]+).*$/, "$1")).filter((u) => u.startsWith("http")),
-    num
-  );
-  const snippets = takeFrom(
-    [...html.matchAll(snippetRegex)].filter((m) => m[1]).map((m) => stripTags(expectDefined(m[1]))),
-    num
-  );
-  for (let i = 0; i < Math.min(titles.length, num); i++) {
-    results.push({
-      title: titles[i] ?? "",
-      url: urls[i] ?? "",
-      snippet: capSnippet(snippets[i] ?? ""),
-      score: 1
-    });
-  }
-  return results;
-}
-async function bingSearch(query, signal) {
-  const encoded = encodeURIComponent(query);
-  const url = `https://www.bing.com/search?q=${encoded}`;
-  try {
-    const html = await fetchWithTimeout(url, signal, TIMEOUT_MS).then((r) => r.text());
-    return { results: parseBingResults(html, MAX_RESULTS) };
-  } catch (err) {
-    return { results: [], error: `bing unreachable: ${toErrorMessage(err)}` };
-  }
-}
-function parseBingResults(html, num) {
-  const results = [];
-  const blocks = [
-    ...html.matchAll(
-      /<li\b[^>]*class=(["'])[^"']*\bb_algo\b[^"']*\1[^>]*>([\s\S]*?)(?=<li\b[^>]*class=(["'])[^"']*\bb_algo\b[^"']*\3|<\/ol>)/gi
-    )
-  ].map((m) => expectDefined(m[2]));
-  const candidates = blocks.length > 0 ? blocks : [html];
-  const entries = takeFrom(
-    candidates.flatMap((block) => {
-      const titleMatch = /<h2[^>]*>\s*<a\b([^>]*)>([\s\S]*?)<\/a>\s*<\/h2>/i.exec(block);
-      if (!titleMatch) return [];
-      const href = getHtmlAttr(expectDefined(titleMatch[1]), "href");
-      const title = stripTags(expectDefined(titleMatch[2]));
-      if (!href || !title) return [];
-      const snippetMatch = /<p\b[^>]*class=(["'])[^"']*\b(?:b_paractl|b_lineclamp\d*)\b[^"']*\1[^>]*>([\s\S]*?)<\/p>/i.exec(
-        block
-      ) ?? /<p\b[^>]*>([\s\S]*?)<\/p>/i.exec(block);
-      const snippet = snippetMatch ? capSnippet(stripTags(expectDefined(snippetMatch.at(-1)))) : "";
-      return [{ url: normalizeBingUrl(href), title, snippet, score: 1 }];
-    }),
-    num
-  );
-  for (let i = 0; i < entries.length; i++) {
-    const entry = entries[i];
-    if (entry) {
-      results.push({
-        title: entry.title ?? "",
-        url: entry.url ?? "",
-        snippet: entry.snippet ?? "",
-        score: 1
-      });
-    }
-  }
-  return results;
-}
-function normalizeBingUrl(raw) {
-  try {
-    const url = new URL(raw);
-    if (url.hostname.endsWith("bing.com") && url.pathname.startsWith("/ck/")) {
-      const encoded = url.searchParams.get("u");
-      const decoded = decodeBingTarget(encoded);
-      if (decoded?.startsWith("http")) return decoded;
-    }
-  } catch {
-  }
-  return raw;
-}
-function decodeBingTarget(encoded) {
-  if (!encoded) return void 0;
-  const payload = encoded.startsWith("a1") ? encoded.slice(2) : encoded;
-  try {
-    const padded = payload.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(payload.length / 4) * 4, "=");
-    return Buffer.from(padded, "base64").toString("utf8");
-  } catch {
-    return void 0;
-  }
-}
-async function fetchWithTimeout(url, signal, timeoutMs) {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeoutMs);
-  const fetchSignal = anySignal(signal, controller.signal);
-  try {
-    const res = await guardedFetch(url, 5, fetchSignal, {
-      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-    });
-    clearTimeout(timer);
-    return res;
-  } catch (e) {
-    clearTimeout(timer);
-    if (e instanceof FetchError2) {
-      throw e;
-    }
-    throw new FetchError2({
-      message: `search: failed to fetch ${url}`,
-      status: 0,
-      context: { url },
-      cause: e
-    });
-  }
-}
-function anySignal(...signals) {
-  return AbortSignal.any(signals);
-}
-function stripTags(html) {
-  return decodeHtmlEntities(html.replace(/<[^>]+>/g, "")).trim();
-}
-function capSnippet(snippet) {
-  return snippet.length > MAX_SNIPPET_CHARS ? `${snippet.slice(0, MAX_SNIPPET_CHARS - 1)}\u2026` : snippet;
-}
-function decodeHtmlEntities(text) {
-  return text.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'");
-}
+import {
+  __clearSearchCache,
+  searchTool
+} from "./chunk-4I4LWFOO.js";
+import "./chunk-NBDZMMG2.js";
 export {
   __clearSearchCache,
   searchTool