mcp-castor 2026.3.0

package/src/harness/services/web_service.js

@@ -0,0 +1,828 @@
+/**
+ * Castor Web & Research Service
+ *
+ * Provides:
+ * - Live web search via DuckDuckGo without API keys (web_search)
+ * - Autonomous webpage fetching & extraction via Mozilla Readability + Turndown (web_fetch)
+ * - Safe handling of JSON, plaintext, HTML articles, and general webpages
+ * - Native in-process V8 execution with zero external subprocess/stdio overhead
+ */
+
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+import { PDFParse } from "pdf-parse";
+import { search, SafeSearchType } from "duck-duck-scrape";
+import { getSearchConfig, MAX_FETCH_CHARS } from "../../config.js";
+import {
+  ensureSearxngRunning,
+  markSearxngActive,
+  registerShutdown,
+} from "./searxng_lifecycle.js";
+import { createRequire } from "node:module";
+
+// Single source of truth for the harness version: read from package.json
+// (same createRequire idiom as index.js and mcp_bridge.js).
+const require = createRequire(import.meta.url);
+const PKG_VERSION = require("../../../package.json").version;
+
+const DEFAULT_USER_AGENT =
+  `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Castor/${PKG_VERSION}`;
+
+const DEFAULT_FETCH_TIMEOUT_MS = 20_000;
+const DEFAULT_SEARCH_TIMEOUT_MS = 15_000;
+
+const NOISE_JSON_KEYS = new Set([
+  "avatar_url",
+  "gravatar_id",
+  "node_id",
+  "followers_url",
+  "following_url",
+  "gists_url",
+  "starred_url",
+  "subscriptions_url",
+  "organizations_url",
+  "repos_url",
+  "events_url",
+  "received_events_url",
+  "site_admin",
+  "user_view_type",
+  "reactions",
+]);
+
+function pruneJsonPayload(val, depth = 0) {
+  if (depth > 8) return val;
+  if (Array.isArray(val)) {
+    const maxItems = 30;
+    const pruned = val.slice(0, maxItems).map((item) => pruneJsonPayload(item, depth + 1));
+    if (val.length > maxItems) {
+      pruned.push({ _notice: `...[${val.length - maxItems} additional items omitted for context efficiency]...` });
+    }
+    return pruned;
+  }
+  if (val && typeof val === "object") {
+    const res = {};
+    for (const [k, v] of Object.entries(val)) {
+      if (NOISE_JSON_KEYS.has(k)) continue;
+      res[k] = pruneJsonPayload(v, depth + 1);
+    }
+    return res;
+  }
+  return val;
+}
+
+// Leaky-bucket serialization queue for DuckDuckGo unauthenticated requests
+let lastDdgRequestTime = 0;
+const DDG_MIN_INTERVAL_MS = 1500;
+let ddgQueue = Promise.resolve();
+
+async function throttleDdgRequest() {
+  const current = ddgQueue;
+  let resolveNext;
+  ddgQueue = new Promise((resolve) => {
+    resolveNext = resolve;
+  });
+  await current;
+  try {
+    const now = Date.now();
+    const elapsed = now - lastDdgRequestTime;
+    if (elapsed < DDG_MIN_INTERVAL_MS) {
+      await new Promise((r) => setTimeout(r, DDG_MIN_INTERVAL_MS - elapsed));
+    }
+    lastDdgRequestTime = Date.now();
+  } finally {
+    resolveNext();
+  }
+}
+
+function isDocQuery(q) {
+  return /\b(docs?|documentation|api|library|package|sdk|crate|module|import|framework|reference|guide)\b/i.test(q);
+}
+
+/**
+ * Removes common tracking query parameters (utm_*, gclid, fbclid, mc_eid,
+ * msclkid, ref, source) from a URL string, preserving all other parameters.
+ *
+ * @param {string} urlStr
+ * @returns {string}
+ */
+function stripTrackingParams(urlStr) {
+  if (!urlStr) return urlStr;
+  try {
+    const u = new URL(urlStr);
+    for (const key of [...u.searchParams.keys()]) {
+      if (
+        key.toLowerCase().startsWith("utm_") ||
+        ["gclid", "fbclid", "mc_eid", "msclkid", "ref", "source"].includes(key.toLowerCase())
+      ) {
+        u.searchParams.delete(key);
+      }
+    }
+    return u.toString();
+  } catch {
+    return urlStr;
+  }
+}
+
+/**
+ * Normalizes a URL for deduplication: strips tracking parameters, lowercases
+ * the scheme and host, drops the fragment, and removes a trailing slash.
+ *
+ * @param {string} urlStr
+ * @returns {string}
+ */
+function normalizeUrl(urlStr) {
+  const stripped = stripTrackingParams(urlStr);
+  try {
+    const u = new URL(stripped);
+    u.hash = "";
+    u.hostname = u.hostname.toLowerCase();
+    u.protocol = u.protocol.toLowerCase();
+    let s = u.toString();
+    if (s.endsWith("/")) s = s.slice(0, -1);
+    return s;
+  } catch {
+    return stripped;
+  }
+}
+
+/**
+ * Strips inline HTML tags and decodes common HTML entities in a string.
+ *
+ * @param {string} s
+ * @returns {string}
+ */
+function sanitizeSnippet(s) {
+  if (!s) return s;
+  return s
+    .replace(/<[^>]+>/g, "")
+    .replace(/&#x27;/gi, "'")
+    .replace(/&#39;/g, "'")
+    .replace(/&quot;/g, '"')
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&amp;/g, "&");
+}
+
+// Media types that must never be decoded as UTF-8 text.
+const BINARY_MEDIA_PREFIXES = ["image/", "audio/", "video/"];
+const BINARY_MEDIA_TYPES = new Set([
+  "application/zip",
+  "application/octet-stream",
+  "application/gzip",
+  "application/x-gzip",
+  "application/x-tar",
+  "application/x-7z-compressed",
+  "application/x-rar-compressed",
+  "application/wasm",
+  "application/x-bzip2",
+  "application/x-xz",
+  "font/woff",
+  "font/woff2",
+  "font/ttf",
+  "font/otf",
+]);
+
+const PDF_MAGIC = Buffer.from("%PDF-");
+
+/**
+ * Returns true when the leading bytes of a buffer contain a NUL byte,
+ * a strong signal of binary (non-text) content.
+ *
+ * @param {Buffer} buf
+ * @returns {boolean}
+ */
+function hasNulByte(buf) {
+  const limit = Math.min(buf.length, 8192);
+  for (let i = 0; i < limit; i++) {
+    if (buf[i] === 0) return true;
+  }
+  return false;
+}
+
+/**
+ * Classifies an HTTP response body into a handling category before any text
+ * decoding, using the Content-Type header and leading magic bytes.
+ *
+ * @param {string} contentType Lowercased Content-Type header value.
+ * @param {Buffer} buf Full response body.
+ * @returns {"json"|"text"|"html"|"pdf"|"binary"}
+ */
+function classifyResponse(contentType, buf) {
+  const base = contentType.split(";")[0].trim().toLowerCase();
+
+  // PDF: explicit type or %PDF- magic (catches mislabeled/octet-stream PDFs).
+  if (base === "application/pdf" || buf.subarray(0, 5).equals(PDF_MAGIC)) {
+    return "pdf";
+  }
+  if (base === "application/json" || base.endsWith("+json")) {
+    return "json";
+  }
+  if (base === "text/html" || base === "application/xhtml+xml") {
+    return "html";
+  }
+  if (base.startsWith("text/")) {
+    return "text";
+  }
+  if (
+    BINARY_MEDIA_TYPES.has(base) ||
+    BINARY_MEDIA_PREFIXES.some((p) => base.startsWith(p))
+  ) {
+    return "binary";
+  }
+  // Unknown/absent Content-Type: sniff the leading bytes.
+  if (hasNulByte(buf)) {
+    return "binary";
+  }
+  return "html";
+}
+
+/**
+ * Extracts plain text from a PDF buffer using pdf-parse (pdfjs-dist based).
+ *
+ * @param {Buffer} buf Raw PDF bytes.
+ * @returns {Promise<string>} Concatenated per-page text.
+ */
+async function extractPdfText(buf) {
+  const parser = new PDFParse({ data: new Uint8Array(buf) });
+  try {
+    const result = await parser.getText();
+    return result.pages.map((p) => p.text).join("\n\n").trim();
+  } finally {
+    await parser.destroy();
+  }
+}
+
+/**
+ * Truncates text to at most `maxChars` characters, cutting at a clean
+ * boundary (a paragraph `\n\n`, else a line `\n`) when one exists within the
+ * last 500 characters of the window, and closing any open markdown code
+ * fence. Appends a concise marker reporting the shown and total character
+ * counts. Returns the input unchanged when it already fits.
+ *
+ * @param {string} text
+ * @param {number} maxChars
+ * @returns {string}
+ */
+function smartTruncate(text, maxChars) {
+  if (text.length <= maxChars) return text;
+
+  const window = text.slice(0, maxChars);
+  const searchStart = Math.max(0, maxChars - 500);
+
+  let cut = window.lastIndexOf("\n\n");
+  if (cut < searchStart) cut = window.lastIndexOf("\n");
+  if (cut < searchStart) cut = maxChars;
+
+  let out = window.slice(0, cut);
+
+  // Close an open markdown code fence so the document stays well-formed.
+  const fenceCount = (out.match(/^```/gm) || []).length;
+  if (fenceCount % 2 === 1) {
+    out = out.replace(/\s+$/, "") + "\n```";
+  }
+
+  return out + `\n\n[Content truncated: showing ${out.length} of ${text.length} chars]`;
+}
+
+export class WebService {
+  constructor(options = {}) {
+    this.userAgent = options.userAgent || DEFAULT_USER_AGENT;
+    this.fetchTimeoutMs = options.fetchTimeoutMs || DEFAULT_FETCH_TIMEOUT_MS;
+    this.searchTimeoutMs = options.searchTimeoutMs || DEFAULT_SEARCH_TIMEOUT_MS;
+    this._initTurndown();
+  }
+
+  _initTurndown() {
+    this.turndown = new TurndownService({
+      headingStyle: "atx",
+      hr: "---",
+      bulletListMarker: "-",
+      codeBlockStyle: "fenced",
+    });
+
+    // Remove script, style, noscript, svg, canvas, iframe from markdown conversion
+    this.turndown.remove(["script", "style", "noscript", "svg", "canvas", "iframe"]);
+  }
+
+  /**
+   * Search via Brave Search API.
+   */
+  async _searchBrave(query, limit, apiKey) {
+    if (!apiKey) throw new Error("MissingBraveApiKey");
+    const u = new URL("https://api.search.brave.com/res/v1/web/search");
+    u.searchParams.set("q", query);
+    u.searchParams.set("count", String(limit));
+    const res = await globalThis.fetch(u.toString(), {
+      headers: {
+        "X-Subscription-Token": apiKey,
+        Accept: "application/json",
+        "User-Agent": this.userAgent,
+      },
+      signal: AbortSignal.timeout(this.searchTimeoutMs),
+    });
+    if (!res.ok) {
+      throw new Error(`BraveSearchError: HTTP ${res.status} ${res.statusText}`);
+    }
+    const data = await res.json();
+    const items = data.web?.results || [];
+    return items.slice(0, limit).map((r) => ({
+      title: sanitizeSnippet(r.title || ""),
+      url: stripTrackingParams(r.url || ""),
+      snippet: sanitizeSnippet(r.description || ""),
+    }));
+  }
+
+  /**
+   * Search via Tavily Search API.
+   */
+  async _searchTavily(query, limit, apiKey) {
+    if (!apiKey) throw new Error("MissingTavilyApiKey");
+    const res = await globalThis.fetch("https://api.tavily.com/search", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": this.userAgent,
+      },
+      body: JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: limit,
+        include_raw_content: false,
+      }),
+      signal: AbortSignal.timeout(this.searchTimeoutMs),
+    });
+    if (!res.ok) {
+      throw new Error(`TavilySearchError: HTTP ${res.status} ${res.statusText}`);
+    }
+    const data = await res.json();
+    const items = data.results || [];
+    return items.slice(0, limit).map((r) => ({
+      title: r.title || "",
+      url: stripTrackingParams(r.url || ""),
+      snippet: r.content || "",
+    }));
+  }
+
+  /**
+   * Search via Upstash Context7 API (Library & Framework Documentation).
+   */
+  async _searchContext7(query, limit, apiKey) {
+    if (!apiKey) throw new Error("MissingContext7ApiKey");
+    const u = new URL("https://context7.com/api/v3/search");
+    u.searchParams.set("query", query);
+    u.searchParams.set("type", "json");
+    const res = await globalThis.fetch(u.toString(), {
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "User-Agent": this.userAgent,
+      },
+      signal: AbortSignal.timeout(this.searchTimeoutMs),
+    });
+    if (!res.ok) {
+      throw new Error(`Context7SearchError: HTTP ${res.status} ${res.statusText}`);
+    }
+    const data = await res.json();
+    const items = data.results || data.snippets || (Array.isArray(data) ? data : []);
+    return items.slice(0, limit).map((r) => ({
+      title: r.title || r.library || "Context7 Documentation",
+      url: stripTrackingParams(r.url || r.source || "https://context7.com"),
+      snippet: r.content || r.snippet || r.text || "",
+    }));
+  }
+
+  /**
+   * Search via SearXNG JSON instance.
+   * For local (loopback) instances, ensures the Docker container is running
+   * before issuing the request, and marks activity for the idle-stop watchdog.
+   */
+  async _searchSearxng(query, limit, baseUrl) {
+    if (!baseUrl) throw new Error("MissingSearxngUrl");
+    await ensureSearxngRunning(baseUrl);
+    const u = new URL(baseUrl.replace(/\/+$/, "") + "/search");
+    u.searchParams.set("q", query);
+    u.searchParams.set("format", "json");
+    const res = await globalThis.fetch(u.toString(), {
+      headers: {
+        "User-Agent": this.userAgent,
+      },
+      signal: AbortSignal.timeout(this.searchTimeoutMs),
+    });
+    if (!res.ok) {
+      throw new Error(`SearxngSearchError: HTTP ${res.status} ${res.statusText}`);
+    }
+    const data = await res.json();
+    markSearxngActive();
+    const items = data.results || [];
+    return items.slice(0, limit).map((r) => ({
+      title: r.title || "",
+      url: stripTrackingParams(r.url || ""),
+      snippet: r.content || "",
+    }));
+  }
+
+  /**
+   * Performs web search using DuckDuckGo (HTML endpoint first, then duck-duck-scrape).
+   */
+  async _searchHtml(query, limit, safeSearch) {
+    const res = await globalThis.fetch("https://html.duckduckgo.com/html/", {
+      method: "POST",
+      headers: {
+        "User-Agent": this.userAgent,
+        "Content-Type": "application/x-www-form-urlencoded",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      },
+      body: `q=${encodeURIComponent(query)}&kp=${safeSearch === "off" ? "-2" : safeSearch === "strict" ? "1" : "-1"}`,
+      signal: AbortSignal.timeout(this.searchTimeoutMs),
+    });
+
+    if (!res.ok) return [];
+
+    const html = await res.text();
+    const dom = new JSDOM(html);
+    const document = dom.window.document;
+    const entries = document.querySelectorAll(".result");
+    const results = [];
+
+    for (const el of entries) {
+      if (results.length >= limit) break;
+      const titleEl = el.querySelector(".result__title a");
+      const snippetEl = el.querySelector(".result__snippet");
+      if (!titleEl) continue;
+
+      let link = titleEl.href || "";
+      try {
+        const u = new URL(link, "https://html.duckduckgo.com");
+        const uddg = u.searchParams.get("uddg");
+        if (uddg) link = decodeURIComponent(uddg);
+      } catch {
+        /* keep raw link */
+      }
+
+      const title = (titleEl.textContent || "").trim();
+      const snippet = snippetEl ? (snippetEl.textContent || "").trim() : "";
+      if (title && link) {
+        results.push({ title, url: stripTrackingParams(link), snippet });
+      }
+    }
+
+    return results;
+  }
+
+  async _searchDuckDuckGo(query, limit, safe_search) {
+    await throttleDdgRequest();
+
+    try {
+      const htmlResults = await this._searchHtml(query, limit, safe_search);
+      if (htmlResults.length > 0) {
+        return htmlResults;
+      }
+    } catch {
+      /* fall through to duck-duck-scrape */
+    }
+
+    let ddgSafeSearch = SafeSearchType.MODERATE;
+    if (safe_search === "strict") ddgSafeSearch = SafeSearchType.STRICT;
+    else if (safe_search === "off") ddgSafeSearch = SafeSearchType.OFF;
+
+    const res = await search(query, {
+      safeSearch: ddgSafeSearch,
+    });
+
+    if (res.noResults || !Array.isArray(res.results)) {
+      return [];
+    }
+
+    return res.results.slice(0, limit).map((r) => ({
+      title: r.title || "",
+      url: stripTrackingParams(r.url || ""),
+      snippet: r.description || r.rawDescription || "",
+    }));
+  }
+
+  /**
+   * Performs multi-provider web search with automatic failover.
+   *
+   * @param {object} params
+   * @param {string} params.query Search terms or phrase
+   * @param {number} [params.max_results=5] Max results to return
+   * @param {"strict"|"moderate"|"off"} [params.safe_search="moderate"] SafeSearch level
+   * @param {string} [params.provider="auto"] Provider override ('auto', 'brave', 'tavily', 'context7', 'searxng', 'duckduckgo')
+   * @returns {Promise<{ query: string, count: number, provider: string, results: Array<{ title: string, url: string, snippet: string }> }>}
+   */
+  async search({ query, max_results = 5, safe_search = "moderate", provider }) {
+    if (!query || typeof query !== "string" || !query.trim()) {
+      throw new Error("InvalidQueryError: Search query must be a non-empty string");
+    }
+
+    const trimmedQuery = query.trim();
+    const limit = Math.min(Math.max(1, max_results), 25);
+    const cfg = getSearchConfig();
+    const chosenProvider = provider || cfg.provider || "auto";
+
+    // Build candidate provider chain
+    const chain = [];
+    if (chosenProvider !== "auto") {
+      chain.push(chosenProvider);
+    } else {
+      // Doc queries: Context7 -> Tavily -> SearXNG -> Brave -> DuckDuckGo
+      // General queries: SearXNG -> Tavily -> Brave -> DuckDuckGo
+      if (isDocQuery(trimmedQuery)) {
+        if (cfg.context7_api_key) chain.push("context7");
+        if (cfg.tavily_api_key) chain.push("tavily");
+        if (cfg.searxng_url) chain.push("searxng");
+        if (cfg.brave_api_key) chain.push("brave");
+      } else {
+        if (cfg.searxng_url) chain.push("searxng");
+        if (cfg.tavily_api_key) chain.push("tavily");
+        if (cfg.brave_api_key) chain.push("brave");
+      }
+      chain.push("duckduckgo");
+    }
+
+    let lastError = null;
+    let anySucceeded = false;
+    for (const p of chain) {
+      try {
+        let results = [];
+        if (p === "brave") results = await this._searchBrave(trimmedQuery, limit, cfg.brave_api_key);
+        else if (p === "tavily") results = await this._searchTavily(trimmedQuery, limit, cfg.tavily_api_key);
+        else if (p === "context7") results = await this._searchContext7(trimmedQuery, limit, cfg.context7_api_key);
+        else if (p === "searxng") results = await this._searchSearxng(trimmedQuery, limit, cfg.searxng_url);
+        else if (p === "duckduckgo") results = await this._searchDuckDuckGo(trimmedQuery, limit, safe_search);
+
+        anySucceeded = true;
+        if (results && results.length > 0) {
+          const seen = new Set();
+          const deduped = results.filter((r) => {
+            const key = normalizeUrl(r.url);
+            if (seen.has(key)) return false;
+            seen.add(key);
+            return true;
+          });
+          return {
+            query: trimmedQuery,
+            count: deduped.length,
+            provider: p,
+            results: deduped,
+          };
+        }
+      } catch (err) {
+        lastError = err;
+        // On explicit provider request, fail fast
+        if (chosenProvider !== "auto") throw err;
+      }
+    }
+
+    // Surface error if all providers in the auto chain failed.
+    if (!anySucceeded && lastError) {
+      return {
+        isError: true,
+        text: `SearchError: all ${chain.length} provider(s) in the auto chain failed. Last error: ${lastError.message}`,
+      };
+    }
+
+    return {
+      query: trimmedQuery,
+      count: 0,
+      provider: chain[chain.length - 1],
+      results: [],
+    };
+  }
+
+
+  /**
+   * Fetches a webpage or API endpoint, extracting clean Markdown.
+   *
+   * @param {object} params
+   * @param {string} params.url HTTP/HTTPS URL
+   * @param {boolean} [params.extract_article=true] Whether to use Mozilla Readability
+   * @param {number} [params.timeout_ms=20000] Request timeout
+   * @param {number} [params.max_chars=60000] Maximum response characters
+   * @returns {Promise<object>}
+   */
+  async fetch({ url, extract_article = true, timeout_ms = this.fetchTimeoutMs, max_chars = MAX_FETCH_CHARS }) {
+    if (!url || typeof url !== "string") {
+      throw new Error("InvalidUrlError: URL must be a string");
+    }
+
+    let parsedUrl;
+    try {
+      parsedUrl = new URL(url.trim());
+    } catch {
+      throw new Error(`InvalidUrlError: Malformed URL '${url}'`);
+    }
+
+    if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
+      throw new Error(`InvalidUrlError: Only HTTP and HTTPS URLs are supported (got '${parsedUrl.protocol}')`);
+    }
+
+    const headers = {
+      "User-Agent": this.userAgent,
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,text/plain;q=0.7,*/*;q=0.5",
+      "Accept-Language": "en-US,en;q=0.9",
+    };
+    if (parsedUrl.hostname === "api.github.com" && process.env.GITHUB_TOKEN) {
+      headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
+    }
+
+    const response = await globalThis.fetch(parsedUrl.href, {
+      method: "GET",
+      headers,
+      signal: AbortSignal.timeout(timeout_ms),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HttpError: GET '${url}' failed with status ${response.status} ${response.statusText}`);
+    }
+
+    // Read the body once as a Buffer so the content can be classified before
+    // any text decoding (prevents binary bodies from being mangled as UTF-8).
+    const body = Buffer.from(await response.arrayBuffer());
+    const contentType = (response.headers.get("content-type") || "").toLowerCase();
+    const category = classifyResponse(contentType, body);
+
+    // 1. PDF: extract plain text in-process (never decode as UTF-8).
+    if (category === "pdf") {
+      const text = await extractPdfText(body);
+      const truncated = smartTruncate(text, max_chars);
+      return {
+        url: parsedUrl.href,
+        status: response.status,
+        content_type: "application/pdf",
+        markdown: truncated,
+        length: text.length,
+      };
+    }
+
+    // 2. Binary media: return a structured stub, never a UTF-8 decode.
+    if (category === "binary") {
+      return {
+        url: parsedUrl.href,
+        status: response.status,
+        content_type: contentType || "application/octet-stream",
+        binary: true,
+        byte_length: body.length,
+        markdown:
+          `[Binary content: ${contentType || "unknown type"}, ${body.length} bytes. ` +
+          "Not rendered as text; use a dedicated tool to inspect this file.",
+        length: body.length,
+      };
+    }
+
+    // 3. JSON handling (with automatic token distillation)
+    if (category === "json") {
+      const data = JSON.parse(body.toString("utf-8"));
+      const distilled = pruneJsonPayload(data);
+      const jsonStr = JSON.stringify(distilled, null, 2);
+      const truncated = smartTruncate(jsonStr, max_chars);
+      return {
+        url: parsedUrl.href,
+        status: response.status,
+        content_type: "application/json",
+        markdown: "```json\n" + truncated + "\n```",
+        length: jsonStr.length,
+      };
+    }
+
+    // 4. Plaintext / Markdown handling
+    if (category === "text") {
+      const text = body.toString("utf-8");
+      const truncated = smartTruncate(text, max_chars);
+      return {
+        url: parsedUrl.href,
+        status: response.status,
+        content_type: contentType.includes("markdown") ? "text/markdown" : "text/plain",
+        markdown: truncated,
+        length: text.length,
+      };
+    }
+
+    // 5. HTML handling: Mozilla Readability + Turndown
+    const rawHtml = body.toString("utf-8");
+    const dom = new JSDOM(rawHtml, { url: parsedUrl.href });
+    const document = dom.window.document;
+
+    // Remove scripts, styles, iframes from DOM before extraction
+    const unwanted = document.querySelectorAll("script, style, noscript, iframe");
+    unwanted.forEach((el) => el.remove());
+
+    if (extract_article) {
+      try {
+        const reader = new Readability(document);
+        const article = reader.parse();
+        if (article && article.content) {
+          let markdown = this.turndown.turndown(article.content);
+          const fullLength = markdown.length;
+          markdown = smartTruncate(markdown, max_chars);
+          return {
+            url: parsedUrl.href,
+            status: response.status,
+            content_type: "article",
+            title: article.title || document.title || "",
+            byline: article.byline || null,
+            excerpt: article.excerpt || null,
+            siteName: article.siteName || null,
+            markdown: markdown.trim(),
+            length: fullLength,
+          };
+        }
+      } catch {
+        // If readability parsing threw, fall through to full body conversion
+      }
+    }
+
+    // Fall back to entire body HTML -> Markdown
+    const bodyHtml = document.body ? document.body.innerHTML : rawHtml;
+    let markdown = this.turndown.turndown(bodyHtml);
+    const fullLength = markdown.length;
+    markdown = smartTruncate(markdown, max_chars);
+
+    return {
+      url: parsedUrl.href,
+      status: response.status,
+      content_type: "page",
+      title: document.title || "",
+      markdown: markdown.trim(),
+      length: fullLength,
+    };
+  }
+}
+
+/**
+ * Castor Plugin to mount WebService into Context.
+ */
+export function webPlugin(ctx, options = {}) {
+  const web = new WebService(options);
+  ctx.provide("web", web);
+  registerShutdown();
+
+  ctx.registerTool("web_search", {
+    description:
+      "Search the live web with automatic multi-provider routing (Brave, Tavily, Context7, SearXNG, DuckDuckGo). " +
+      "Returns top matching results with titles, URLs, and text snippets. " +
+      "Use this to find current documentation, technical solutions, and library APIs. " +
+      "Set provider to 'context7' to specifically search framework and library documentation.",
+    parameters: {
+      type: "object",
+      properties: {
+        query: {
+          type: "string",
+          description: "Search query keywords or phrase",
+        },
+        max_results: {
+          type: "integer",
+          description: "Maximum number of search results to return (default 5, max 25)",
+          default: 5,
+        },
+        safe_search: {
+          type: "string",
+          enum: ["strict", "moderate", "off"],
+          description: "SafeSearch filtering level (default 'moderate')",
+          default: "moderate",
+        },
+        provider: {
+          type: "string",
+          enum: ["auto", "brave", "tavily", "context7", "searxng", "duckduckgo"],
+          description: "Search provider override (default 'auto'). Use 'context7' for framework and library documentation.",
+          default: "auto",
+        },
+      },
+      required: ["query"],
+    },
+    execute: (args) => web.search(args),
+  });
+
+  ctx.registerTool("web_fetch", {
+    description:
+      "Fetches web page or API content at a URL and converts it into clean Markdown using Mozilla Readability and Turndown. " +
+      "Extracts main article text, headers, and code blocks while stripping navigation, scripts, and ads. " +
+      "Also supports JSON and plain text endpoints.",
+    parameters: {
+      type: "object",
+      properties: {
+        url: {
+          type: "string",
+          description: "HTTP or HTTPS URL to fetch",
+        },
+        extract_article: {
+          type: "boolean",
+          description: "When true (default), use Mozilla Readability to extract main article body. When false, convert entire page body to markdown.",
+          default: true,
+        },
+        timeout_ms: {
+          type: "integer",
+          description: "Request timeout in milliseconds (default 20000)",
+          default: 20000,
+        },
+        max_chars: {
+          type: "integer",
+          description: "Maximum characters of markdown to return (default 60000)",
+          default: 60000,
+        },
+      },
+      required: ["url"],
+    },
+    execute: (args) => web.fetch(args),
+  });
+}