@zhafron/mcp-web-search 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 tickernelz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,140 @@
+# mcp-web-search
+
+MCP server: web search, Wikipedia summaries, and URL content extraction. No API keys required.
+
+Version: 1.0.0
+
+## Features
+
+- search_web - Two-tier web search (DuckDuckGo HTML / Puppeteer/Bing)
+- fetch_url - Extract content from URLs with semantic truncation
+- summarize_url - Fetch and summarize URL content
+- wiki_get - Wikipedia summary by language
+- wiki_multi - Wikipedia summaries in multiple languages
+
+## Requirements
+
+- Node.js 18+
+- Windows/macOS/Linux
+- Chrome/Chromium (for deep search mode)
+
+## Installation
+
+```bash
+npm install
+```
+
+## Chrome Installation
+
+| OS | Command |
+|----|---------|
+| Ubuntu/Debian | sudo apt install chromium-browser |
+| Fedora | sudo dnf install chromium |
+| Arch | sudo pacman -S chromium |
+| macOS | brew install --cask google-chrome |
+
+Custom path: `export CHROME_PATH=/path/to/chrome`
+
+## Commands
+
+```bash
+npm run dev    # Development
+npm run build  # Build
+npm run start  # Production
+npm test       # Run tests
+npm run format # Format code
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| USER_AGENT | mcp-web-search/1.0 | User agent string |
+| HTTP_TIMEOUT | 15000 | Request timeout (ms) |
+| MAX_RESULTS | 10 | Default search limit |
+| LANG_DEFAULT | en | Default language |
+| MAX_BYTES | 20971520 | Max download size |
+| CHROME_PATH | auto-detect | Chrome executable path |
+
+SSRF Protection: Blocks localhost, 127.0.0.1, ::1, .local domains.
+
+## Tool Reference
+
+### search_web
+
+Two-tier web search.
+
+Input: `{ q: string, limit?: number, lang?: string, mode?: "fast"|"deep"|"auto" }`
+
+Output: `{ items: Array<{ title, url, snippet?, source }>, modeUsed, enginesUsed, escalated }`
+
+Example: `{ "q": "Node.js LTS", "mode": "fast", "limit": 5 }`
+
+### fetch_url
+
+Extract content with intelligent truncation.
+
+Input: `{ url: string, mode?: "compact"|"standard"|"full", max_length?: number, format?: "markdown"|"text"|"html" }`
+
+| Mode | Characters | Tokens | Use Case |
+|------|------------|--------|----------|
+| compact | ~3000 | ~750 | Quick summaries |
+| standard | ~8000 | ~2000 | Balanced (default) |
+| full | unlimited | - | Full content |
+
+max_length: Exact character limit (1000-100000), overrides mode.
+
+format: Output format (markdown, text, html). Default: markdown.
+
+Truncation: Semantic chunking prioritizes headings, code blocks, conclusions.
+
+Output: `{ markdown?, text?, format, url, title?, truncated?, original_length?, truncation_ratio? }`
+
+Examples:
+- `{ "url": "https://example.com", "mode": "compact" }`
+- `{ "url": "https://example.com", "format": "text" }`
+- `{ "url": "https://example.com", "format": "markdown" }`
+- `{ "url": "https://example.com", "max_length": 5000 }`
+
+### summarize_url
+
+Fetch and summarize URL content.
+
+Input: `{ url: string }`
+
+### wiki_get
+
+Wikipedia summary by language.
+
+Input: `{ title: string, lang?: string }`
+
+Output: `{ lang, title, url, description?, extract?, thumbnailUrl? }`
+
+### wiki_multi
+
+Wikipedia summaries in multiple languages.
+
+Input: `{ term: string, baseLang?: string, langs?: string[] }`
+
+## Quick Examples
+
+```
+search_web: { "q": "App Intents", "mode": "deep", "limit": 5 }
+fetch_url: { "url": "https://example.com", "mode": "compact" }
+summarize_url: { "url": "https://python.org/pep-8" }
+wiki_get: { "title": "Lambda calculus", "lang": "en" }
+wiki_multi: { "term": "AI", "langs": ["en", "es", "fr"] }
+```
+
+## Troubleshooting
+
+| Issue | Solution |
+|-------|----------|
+| Chrome not found | Install Chrome or set CHROME_PATH |
+| CAPTCHA/blocks | Reduce frequency, use fast mode |
+| Timeout | Increase HTTP_TIMEOUT, check MAX_BYTES |
+| Blocked URL | SSRF protection, public URLs only |
+
+## License
+
+MIT

package/dist/src/chrome.js

@@ -0,0 +1,66 @@
+import { existsSync } from "fs";
+import { platform } from "os";
+export class ChromeNotFoundError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ChromeNotFoundError";
+    }
+}
+function getDefaultChromePaths() {
+    const plat = platform();
+    if (plat === "win32") {
+        return [
+            process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
+            process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
+            process.env["PROGRAMFILES(X86)"] + "\\Google\\Chrome\\Application\\chrome.exe",
+            process.env.LOCALAPPDATA + "\\Chromium\\Application\\chrome.exe",
+            process.env.PROGRAMFILES + "\\Chromium\\Application\\chrome.exe",
+            process.env["PROGRAMFILES(X86)"] + "\\Chromium\\Application\\chrome.exe"
+        ].filter(Boolean);
+    }
+    if (plat === "darwin") {
+        return [
+            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+            "/Applications/Chromium.app/Contents/MacOS/Chromium",
+            process.env.HOME + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+            process.env.HOME + "/Applications/Chromium.app/Contents/MacOS/Chromium"
+        ].filter(Boolean);
+    }
+    return [
+        "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
+        "/usr/bin/chromium",
+        "/usr/bin/chromium-browser",
+        "/snap/bin/chromium",
+        "/usr/local/bin/chrome",
+        "/usr/local/bin/chromium"
+    ];
+}
+export function findChrome() {
+    if (process.env.CHROME_PATH) {
+        if (existsSync(process.env.CHROME_PATH)) {
+            return process.env.CHROME_PATH;
+        }
+        throw new ChromeNotFoundError(`Chrome not found at CHROME_PATH: ${process.env.CHROME_PATH}`);
+    }
+    const paths = getDefaultChromePaths();
+    for (const path of paths) {
+        if (existsSync(path)) {
+            return path;
+        }
+    }
+    const plat = platform();
+    let installInstructions = "";
+    if (plat === "win32") {
+        installInstructions = "Download from: https://www.google.com/chrome/";
+    }
+    else if (plat === "darwin") {
+        installInstructions =
+            "Install via: brew install --cask google-chrome\nOr download from: https://www.google.com/chrome/";
+    }
+    else {
+        installInstructions =
+            "Install via:\n  Ubuntu/Debian: sudo apt install chromium-browser\n  Fedora: sudo dnf install chromium\n  Arch: sudo pacman -S chromium\nOr download from: https://www.google.com/chrome/";
+    }
+    throw new ChromeNotFoundError(`Chrome/Chromium not found on system.\n\n${installInstructions}\n\nAlternatively, set CHROME_PATH environment variable to your Chrome executable.`);
+}

package/dist/src/engines.js

@@ -0,0 +1,164 @@
+import { JSDOM } from "jsdom";
+import puppeteer from "puppeteer-core";
+import { findChrome } from "./chrome.js";
+function uaHeaders(lang = process.env.LANG_DEFAULT || "en") {
+    const ua = process.env.USER_AGENT || "mcp-web-search/1.0";
+    const acceptLang = lang === "en" ? "en-US,en;q=0.9" : `${lang};q=0.9,en;q=0.8`;
+    return { "User-Agent": ua, "Accept-Language": acceptLang };
+}
+function toMs(env, def) {
+    const n = Number(env);
+    return Number.isFinite(n) && n > 0 ? n : def;
+}
+async function fetchWithTimeout(input, init = {}, timeoutMs = 15000) {
+    const controller = new AbortController();
+    const t = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(input, { ...init, signal: controller.signal });
+    }
+    finally {
+        clearTimeout(t);
+    }
+}
+const HTTP_TIMEOUT = toMs(process.env.HTTP_TIMEOUT, 15000);
+function decodeDuckDuckGoRedirect(href) {
+    try {
+        const u = new URL(href, "https://duckduckgo.com/");
+        if (u.hostname === "duckduckgo.com" && u.pathname.startsWith("/l/")) {
+            const real = u.searchParams.get("uddg");
+            if (real)
+                return decodeURIComponent(real);
+        }
+        return u.toString();
+    }
+    catch {
+        return href;
+    }
+}
+async function ddgHtmlSearch(q, limit, lang) {
+    const url = new URL("https://html.duckduckgo.com/html/");
+    url.searchParams.set("q", q);
+    const res = await fetchWithTimeout(url, { headers: uaHeaders(lang) }, HTTP_TIMEOUT);
+    if (!res.ok)
+        throw new Error(`DuckDuckGo HTML ${res.status}`);
+    const html = await res.text();
+    const dom = new JSDOM(html, { url: "https://duckduckgo.com/?q=" + encodeURIComponent(q) });
+    const doc = dom.window.document;
+    const anchors = Array.from(doc.querySelectorAll("a.result__a"));
+    const snippets = Array.from(doc.querySelectorAll(".result__snippet"));
+    const items = [];
+    for (let i = 0; i < anchors.length && items.length < limit; i++) {
+        const a = anchors[i];
+        const title = (a.textContent || "").trim();
+        const href = decodeDuckDuckGoRedirect(a.getAttribute("href") || "");
+        if (!title || !href)
+            continue;
+        const sn = (snippets[i]?.textContent || "").trim() || undefined;
+        try {
+            const u = new URL(href);
+            items.push({ title, url: u.toString(), snippet: sn, source: "ddg_html" });
+        }
+        catch { }
+    }
+    return items;
+}
+async function bingPuppeteerSearch(q, limit, lang) {
+    const chromePath = findChrome();
+    const browser = await puppeteer.launch({
+        executablePath: chromePath,
+        headless: true,
+        args: [
+            "--no-sandbox",
+            "--disable-setuid-sandbox",
+            "--disable-dev-shm-usage",
+            "--disable-accelerated-2d-canvas",
+            "--no-first-run",
+            "--no-zygote",
+            "--disable-gpu"
+        ]
+    });
+    try {
+        const page = await browser.newPage();
+        await page.setUserAgent((process.env.USER_AGENT || "mcp-web-search/1.0") + " Puppeteer");
+        await page.setExtraHTTPHeaders({
+            "Accept-Language": lang === "en" ? "en-US,en;q=0.9" : `${lang};q=0.9,en;q=0.8`
+        });
+        const url = new URL("https://www.bing.com/search");
+        url.searchParams.set("q", q);
+        if (lang)
+            url.searchParams.set("setlang", lang);
+        await page.goto(url.toString(), { waitUntil: "domcontentloaded", timeout: 30000 });
+        const results = await page.evaluate(maxResults => {
+            const items = [];
+            const cards = document.querySelectorAll("li.b_algo");
+            for (const card of Array.from(cards)) {
+                const anchor = card.querySelector("h2 a");
+                if (!anchor)
+                    continue;
+                const title = anchor.textContent?.trim() || "";
+                const href = anchor.getAttribute("href");
+                if (!href || !title)
+                    continue;
+                let snippet = "";
+                const captionP = card.querySelector("div.b_caption p");
+                if (captionP) {
+                    snippet = captionP.textContent?.trim() || "";
+                }
+                else {
+                    const snippetDiv = card.querySelector("div.b_snippet");
+                    if (snippetDiv) {
+                        snippet = snippetDiv.textContent?.trim() || "";
+                    }
+                }
+                try {
+                    new URL(href);
+                    items.push({ title, url: href, snippet: snippet || undefined });
+                }
+                catch { }
+                if (items.length >= maxResults)
+                    break;
+            }
+            return items;
+        }, limit);
+        return results.map(r => ({ ...r, source: "bing_puppeteer" }));
+    }
+    finally {
+        await browser.close();
+    }
+}
+export async function runTwoTierSearch(opts) {
+    const { q } = opts;
+    const limit = Math.max(1, Math.min(Number(opts.limit ?? (Number(process.env.MAX_RESULTS) || 10)), 50));
+    const lang = opts.lang ?? (process.env.LANG_DEFAULT || "en");
+    const mode = opts.mode ?? "auto";
+    const enginesUsed = [];
+    const diagnostics = {};
+    if (mode === "fast") {
+        const fast = await ddgHtmlSearch(q, limit, lang);
+        enginesUsed.push("ddg_html");
+        diagnostics["fastCount"] = fast.length;
+        return { items: fast, modeUsed: "fast", enginesUsed, escalated: false, diagnostics };
+    }
+    if (mode === "deep") {
+        const deep = await bingPuppeteerSearch(q, limit, lang);
+        enginesUsed.push("bing_puppeteer");
+        diagnostics["deepCount"] = deep.length;
+        return { items: deep, modeUsed: "deep", enginesUsed, escalated: false, diagnostics };
+    }
+    const fast = await ddgHtmlSearch(q, limit, lang);
+    enginesUsed.push("ddg_html");
+    diagnostics["fastCount"] = fast.length;
+    if (fast.length < Math.min(3, limit)) {
+        const deep = await bingPuppeteerSearch(q, limit, lang);
+        enginesUsed.push("bing_puppeteer");
+        diagnostics["deepCount"] = deep.length;
+        return {
+            items: [...fast, ...deep].slice(0, limit),
+            modeUsed: "auto",
+            enginesUsed,
+            escalated: true,
+            diagnostics
+        };
+    }
+    return { items: fast, modeUsed: "auto", enginesUsed, escalated: false, diagnostics };
+}

package/dist/src/extract.js

@@ -0,0 +1,187 @@
+import { JSDOM } from "jsdom";
+import { Readability } from "@mozilla/readability";
+import { extractWithReadabilityAlt } from "./extractors/readability-alt.js";
+import { htmlToMarkdown } from "./extractors/markdown.js";
+import { applySmartTruncation } from "./extractors/truncation.js";
+function uaHeaders() {
+    const ua = process.env.USER_AGENT || "mcp-web-search/1.0";
+    const lang = process.env.LANG_DEFAULT || "en";
+    const accept = lang === "en" ? "en-US,en;q=0.9" : `${lang};q=0.9,en;q=0.8`;
+    return { "User-Agent": ua, "Accept-Language": accept };
+}
+function toMs(env, def) {
+    const n = Number(env);
+    return Number.isFinite(n) && n > 0 ? n : def;
+}
+async function fetchWithTimeout(input, init = {}, timeoutMs = 15000) {
+    const controller = new AbortController();
+    const t = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(input, { ...init, signal: controller.signal });
+    }
+    finally {
+        clearTimeout(t);
+    }
+}
+const HTTP_TIMEOUT = toMs(process.env.HTTP_TIMEOUT, 15000);
+const MAX_BYTES = toMs(process.env.MAX_BYTES, 20 * 1024 * 1024);
+function isBlockedHost(hostname) {
+    const lower = hostname.toLowerCase();
+    if (lower === "localhost" || lower === "127.0.0.1" || lower === "::1")
+        return true;
+    if (lower.endsWith(".local") || lower.endsWith(".localhost"))
+        return true;
+    return false;
+}
+function fallbackExtraction(html, url) {
+    try {
+        const dom = new JSDOM(html, { url });
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        if (article) {
+            return {
+                title: article.title ?? undefined,
+                byline: article.byline ?? undefined,
+                siteName: article.siteName ?? undefined,
+                text: article.textContent ?? ""
+            };
+        }
+        const text = dom.window.document.body.textContent || "";
+        return { text, title: dom.window.document.title };
+    }
+    catch {
+        return { text: "" };
+    }
+}
+export async function fetchAndExtract(url, options) {
+    const u = new URL(url);
+    if (isBlockedHost(u.hostname)) {
+        throw new Error("Blocked localhost/private URL");
+    }
+    const res = await fetchWithTimeout(u.toString(), { redirect: "follow", headers: uaHeaders() }, HTTP_TIMEOUT);
+    if (!res.ok)
+        throw new Error(`Fetch ${res.status} for ${url}`);
+    const lenHeader = res.headers.get("content-length");
+    const len = Number(lenHeader || "0");
+    if (len > 0 && len > MAX_BYTES)
+        throw new Error(`Content too large: ${len} bytes`);
+    const ct = res.headers.get("content-type") || "";
+    const buf = Buffer.from(await res.arrayBuffer());
+    if (buf.byteLength > MAX_BYTES)
+        throw new Error(`Content too large (downloaded)`);
+    if (ct.includes("application/pdf") || u.pathname.toLowerCase().endsWith(".pdf")) {
+        const pdfParse = (await import("pdf-parse")).default;
+        const data = await pdfParse(buf);
+        const text = data.text || "";
+        const truncationResult = applySmartTruncation(text, "text", options);
+        return {
+            text: truncationResult.content,
+            url,
+            title: data.info?.Title,
+            length: data.numpages,
+            format: "text",
+            truncated: truncationResult.truncated,
+            original_length: truncationResult.original_length,
+            truncation_ratio: truncationResult.truncated
+                ? truncationResult.final_length / truncationResult.original_length
+                : undefined
+        };
+    }
+    const html = buf.toString("utf8");
+    const extracted = extractWithReadabilityAlt(html, url);
+    const requestedFormat = options?.format || "markdown";
+    const shouldReturnMarkdown = requestedFormat === "markdown";
+    const shouldReturnText = requestedFormat === "text";
+    const shouldReturnHtml = requestedFormat === "html";
+    if (extracted && extracted.textContent && extracted.textContent.length > 0) {
+        const markdown = htmlToMarkdown(extracted.content);
+        if (shouldReturnMarkdown && markdown) {
+            const truncationResult = applySmartTruncation(markdown, "markdown", options);
+            return {
+                title: extracted.title || undefined,
+                markdown: truncationResult.content,
+                url,
+                length: extracted.length,
+                format: "markdown",
+                truncated: truncationResult.truncated,
+                original_length: truncationResult.original_length,
+                truncation_ratio: truncationResult.truncated
+                    ? truncationResult.final_length / truncationResult.original_length
+                    : undefined
+            };
+        }
+        if (shouldReturnText) {
+            const truncationResult = applySmartTruncation(extracted.textContent, "text", options);
+            return {
+                title: extracted.title || undefined,
+                text: truncationResult.content,
+                url,
+                length: extracted.length,
+                format: "text",
+                truncated: truncationResult.truncated,
+                original_length: truncationResult.original_length,
+                truncation_ratio: truncationResult.truncated
+                    ? truncationResult.final_length / truncationResult.original_length
+                    : undefined
+            };
+        }
+        if (shouldReturnHtml && extracted.content) {
+            const truncationResult = applySmartTruncation(extracted.content, "markdown", options);
+            return {
+                title: extracted.title || undefined,
+                markdown: truncationResult.content,
+                url,
+                length: extracted.length,
+                format: "markdown",
+                truncated: truncationResult.truncated,
+                original_length: truncationResult.original_length,
+                truncation_ratio: truncationResult.truncated
+                    ? truncationResult.final_length / truncationResult.original_length
+                    : undefined
+            };
+        }
+        if (markdown) {
+            const truncationResult = applySmartTruncation(markdown, "markdown", options);
+            return {
+                title: extracted.title || undefined,
+                markdown: truncationResult.content,
+                url,
+                length: extracted.length,
+                format: "markdown",
+                truncated: truncationResult.truncated,
+                original_length: truncationResult.original_length,
+                truncation_ratio: truncationResult.truncated
+                    ? truncationResult.final_length / truncationResult.original_length
+                    : undefined
+            };
+        }
+        const truncationResult = applySmartTruncation(extracted.textContent, "text", options);
+        return {
+            title: extracted.title || undefined,
+            text: truncationResult.content,
+            url,
+            length: extracted.length,
+            format: "text",
+            truncated: truncationResult.truncated,
+            original_length: truncationResult.original_length,
+            truncation_ratio: truncationResult.truncated
+                ? truncationResult.final_length / truncationResult.original_length
+                : undefined
+        };
+    }
+    const fallback = fallbackExtraction(html, url);
+    const truncationResult = applySmartTruncation(fallback.text, "text", options);
+    return {
+        title: fallback.title,
+        byline: fallback.byline,
+        siteName: fallback.siteName,
+        text: truncationResult.content,
+        url,
+        format: "text",
+        truncated: truncationResult.truncated,
+        original_length: truncationResult.original_length,
+        truncation_ratio: truncationResult.truncated
+            ? truncationResult.final_length / truncationResult.original_length
+            : undefined
+    };
+}

package/dist/src/extractors/markdown.js

@@ -0,0 +1,40 @@
+import TurndownService from "turndown";
+const turndownService = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+    emDelimiter: "*",
+    strongDelimiter: "**",
+    linkStyle: "inlined"
+});
+turndownService.addRule("removeEmptyElements", {
+    filter: (node) => {
+        return node.textContent?.trim() === "" && !["IMG", "BR", "HR"].includes(node.nodeName);
+    },
+    replacement: () => ""
+});
+turndownService.addRule("preserveCodeBlocks", {
+    filter: ["pre", "code"],
+    replacement: (content, node) => {
+        if (node.nodeName === "PRE") {
+            const code = node.querySelector("code");
+            const lang = code?.className.match(/language-(\w+)/)?.[1] || "";
+            return `\n\`\`\`${lang}\n${content}\n\`\`\`\n`;
+        }
+        return `\`${content}\``;
+    }
+});
+export function htmlToMarkdown(html) {
+    try {
+        if (!html || html.trim().length === 0)
+            return null;
+        const markdown = turndownService.turndown(html);
+        if (!markdown || markdown.trim().length === 0)
+            return null;
+        return markdown.trim();
+    }
+    catch (error) {
+        console.error("Markdown conversion failed:", error);
+        return null;
+    }
+}

package/dist/src/extractors/readability-alt.js

@@ -0,0 +1,110 @@
+import { JSDOM } from "jsdom";
+import { DEFAULT_CONFIG } from "./types.js";
+function scoreNode(node, config) {
+    let score = 0;
+    const text = node.textContent || "";
+    const textLength = text.trim().length;
+    if (textLength < config.minTextLength)
+        return 0;
+    score += textLength * 0.1;
+    const tagName = node.tagName.toLowerCase();
+    const boost = config.tagBoosts[tagName] || 1.0;
+    score *= boost;
+    const className = node.className || "";
+    const id = node.id || "";
+    if (config.ignoreClasses.test(className) || config.ignoreClasses.test(id)) {
+        return 0;
+    }
+    const density = textLength / (node.children.length + 1);
+    score += density * 0.5;
+    const pCount = node.querySelectorAll("p").length;
+    score += pCount * 5;
+    return score;
+}
+function cleanNode(node) {
+    const toRemove = ["script", "style", "noscript", "iframe", "object", "embed"];
+    toRemove.forEach(tag => {
+        node.querySelectorAll(tag).forEach(el => el.remove());
+    });
+    node.querySelectorAll("*").forEach(el => {
+        const className = el.className || "";
+        const id = el.id || "";
+        if (/ads|advertisement|sponsor|promo/i.test(className) || /ads|advertisement/i.test(id)) {
+            el.remove();
+        }
+    });
+}
+function extractTextContent(node) {
+    let text = "";
+    function traverse(el) {
+        if (el.nodeType === 3) {
+            const content = el.textContent?.trim();
+            if (content)
+                text += content + " ";
+        }
+        else if (el.nodeType === 1) {
+            const element = el;
+            if (["P", "DIV", "BR", "H1", "H2", "H3", "H4", "H5", "H6"].includes(element.tagName)) {
+                text += "\n";
+            }
+            element.childNodes.forEach(child => traverse(child));
+        }
+    }
+    traverse(node);
+    return text.replace(/\s+/g, " ").trim();
+}
+export function extractWithReadabilityAlt(html, url, config = DEFAULT_CONFIG) {
+    try {
+        const dom = new JSDOM(html, { url });
+        const doc = dom.window.document;
+        const title = doc.title || "";
+        const candidates = Array.from(doc.querySelectorAll("article, section, main, div, [role='main']"));
+        if (candidates.length === 0) {
+            const body = doc.body;
+            if (!body)
+                return null;
+            cleanNode(body);
+            const textContent = extractTextContent(body);
+            return {
+                title,
+                textContent,
+                content: body.innerHTML,
+                length: textContent.length
+            };
+        }
+        let bestNode = null;
+        let bestScore = 0;
+        for (const candidate of candidates) {
+            const score = scoreNode(candidate, config);
+            if (score > bestScore) {
+                bestScore = score;
+                bestNode = candidate;
+            }
+        }
+        if (!bestNode || bestScore === 0) {
+            const body = doc.body;
+            if (!body)
+                return null;
+            cleanNode(body);
+            const textContent = extractTextContent(body);
+            return {
+                title,
+                textContent,
+                content: body.innerHTML,
+                length: textContent.length
+            };
+        }
+        cleanNode(bestNode);
+        const textContent = extractTextContent(bestNode);
+        return {
+            title,
+            textContent,
+            content: bestNode.innerHTML,
+            length: textContent.length
+        };
+    }
+    catch (error) {
+        console.error("Content extraction failed:", error);
+        return null;
+    }
+}

package/dist/src/extractors/truncation.js

@@ -0,0 +1,246 @@
+const MODE_LIMITS = {
+    compact: 3000,
+    standard: 8000,
+    full: Infinity
+};
+const KEYWORDS = [
+    "summary",
+    "conclusion",
+    "important",
+    "overview",
+    "introduction",
+    "key",
+    "main",
+    "abstract"
+];
+export function applySmartTruncation(content, format, options) {
+    const mode = options?.mode || "standard";
+    const maxLength = options?.max_length || MODE_LIMITS[mode] || MODE_LIMITS.standard;
+    if (maxLength === Infinity || content.length <= maxLength) {
+        return {
+            content,
+            truncated: false,
+            original_length: content.length,
+            final_length: content.length
+        };
+    }
+    if (format === "markdown") {
+        return truncateMarkdown(content, maxLength);
+    }
+    else {
+        return truncateText(content, maxLength);
+    }
+}
+export function truncateMarkdown(content, maxLength) {
+    const chunks = parseMarkdownChunks(content);
+    if (chunks.length === 0) {
+        return {
+            content: balancedTruncate(content, maxLength),
+            truncated: true,
+            original_length: content.length,
+            final_length: Math.min(content.length, maxLength)
+        };
+    }
+    chunks.forEach((chunk, idx) => {
+        chunk.score = scoreChunk(chunk, chunks.length);
+        chunk.position = idx;
+    });
+    const selected = selectChunks(chunks, maxLength);
+    const assembled = assembleChunks(selected);
+    return {
+        content: assembled,
+        truncated: true,
+        original_length: content.length,
+        final_length: assembled.length,
+        chunks_selected: selected.length,
+        chunks_total: chunks.length
+    };
+}
+export function truncateText(content, maxLength) {
+    const chunks = parseSentences(content);
+    if (chunks.length === 0) {
+        return {
+            content: balancedTruncate(content, maxLength),
+            truncated: true,
+            original_length: content.length,
+            final_length: Math.min(content.length, maxLength)
+        };
+    }
+    chunks.forEach((chunk, idx) => {
+        chunk.score = scoreChunk(chunk, chunks.length);
+        chunk.position = idx;
+    });
+    const selected = selectChunks(chunks, maxLength);
+    const assembled = assembleChunks(selected);
+    return {
+        content: assembled,
+        truncated: true,
+        original_length: content.length,
+        final_length: assembled.length,
+        chunks_selected: selected.length,
+        chunks_total: chunks.length
+    };
+}
+export function parseMarkdownChunks(markdown) {
+    const chunks = [];
+    const lines = markdown.split("\n");
+    let currentChunk = [];
+    let currentType = "paragraph";
+    let inCodeBlock = false;
+    const flushChunk = () => {
+        if (currentChunk.length > 0) {
+            const content = currentChunk.join("\n").trim();
+            if (content) {
+                chunks.push({
+                    content,
+                    type: currentType,
+                    position: 0,
+                    score: 0,
+                    length: content.length
+                });
+            }
+            currentChunk = [];
+        }
+    };
+    for (const line of lines) {
+        if (line.startsWith("```")) {
+            if (inCodeBlock) {
+                currentChunk.push(line);
+                flushChunk();
+                inCodeBlock = false;
+                currentType = "paragraph";
+            }
+            else {
+                flushChunk();
+                inCodeBlock = true;
+                currentType = "code";
+                currentChunk.push(line);
+            }
+            continue;
+        }
+        if (inCodeBlock) {
+            currentChunk.push(line);
+            continue;
+        }
+        if (line.match(/^#{1,6}\s/)) {
+            flushChunk();
+            currentType = "heading";
+            currentChunk.push(line);
+            flushChunk();
+            currentType = "paragraph";
+        }
+        else if (line.match(/^[\s]*[-*+]\s/) || line.match(/^[\s]*\d+\.\s/)) {
+            if (currentType !== "list") {
+                flushChunk();
+                currentType = "list";
+            }
+            currentChunk.push(line);
+        }
+        else if (line.trim() === "") {
+            if (currentType === "list") {
+                flushChunk();
+                currentType = "paragraph";
+            }
+        }
+        else {
+            if (currentType === "list") {
+                flushChunk();
+                currentType = "paragraph";
+            }
+            currentChunk.push(line);
+        }
+    }
+    flushChunk();
+    return chunks;
+}
+export function parseSentences(text) {
+    const sentences = text.match(/[^.!?]+[.!?]+/g) || [];
+    return sentences.map(sentence => ({
+        content: sentence.trim(),
+        type: "text",
+        position: 0,
+        score: 0,
+        length: sentence.trim().length
+    }));
+}
+export function scoreChunk(chunk, totalChunks) {
+    let score = 0;
+    if (chunk.type === "heading") {
+        score += 20;
+    }
+    else if (chunk.type === "code") {
+        score += 10;
+    }
+    else if (chunk.type === "list") {
+        score += 5;
+    }
+    const positionRatio = chunk.position / Math.max(totalChunks - 1, 1);
+    if (positionRatio <= 0.15) {
+        score += 15;
+    }
+    else if (positionRatio >= 0.85) {
+        score += 12;
+    }
+    if (chunk.length >= 100 && chunk.length <= 1000) {
+        score += 5;
+    }
+    else if (chunk.length > 300) {
+        score += 3;
+    }
+    const lowerContent = chunk.content.toLowerCase();
+    for (const keyword of KEYWORDS) {
+        if (lowerContent.includes(keyword)) {
+            score += 5;
+        }
+    }
+    if (chunk.length < 50) {
+        score -= 10;
+    }
+    return score;
+}
+export function selectChunks(chunks, maxLength) {
+    const selected = [];
+    let currentLength = 0;
+    const firstHeading = chunks.find(c => c.type === "heading");
+    if (firstHeading) {
+        selected.push(firstHeading);
+        currentLength += firstHeading.length + 5;
+    }
+    const sortedChunks = [...chunks]
+        .filter(c => !selected.includes(c))
+        .sort((a, b) => b.score - a.score);
+    for (const chunk of sortedChunks) {
+        const chunkLength = chunk.length + 5;
+        if (currentLength + chunkLength <= maxLength) {
+            selected.push(chunk);
+            currentLength += chunkLength;
+        }
+    }
+    return selected.sort((a, b) => a.position - b.position);
+}
+export function assembleChunks(chunks) {
+    if (chunks.length === 0)
+        return "";
+    const parts = [];
+    let lastPosition = -1;
+    for (const chunk of chunks) {
+        if (lastPosition >= 0 && chunk.position > lastPosition + 1) {
+            parts.push("[...]");
+        }
+        parts.push(chunk.content);
+        lastPosition = chunk.position;
+    }
+    return parts.join("\n\n");
+}
+export function balancedTruncate(text, maxLength) {
+    if (text.length <= maxLength)
+        return text;
+    const startLen = Math.floor(maxLength * 0.4);
+    const middleLen = Math.floor(maxLength * 0.3);
+    const endLen = maxLength - startLen - middleLen - 10;
+    const start = text.slice(0, startLen);
+    const middleStart = Math.floor((text.length - middleLen) / 2);
+    const middle = text.slice(middleStart, middleStart + middleLen);
+    const end = text.slice(-endLen);
+    return `${start}\n[...]\n${middle}\n[...]\n${end}`;
+}

package/dist/src/extractors/types.js

@@ -0,0 +1,7 @@
+export const DEFAULT_CONFIG = {
+    ignoreClasses: /nav|sidebar|ads|advertisement|footer|header|menu|comment/i,
+    minTextLength: 50,
+    columnMinText: 30,
+    columnThreshold: 0.25,
+    tagBoosts: { article: 1.7, main: 1.5, section: 1.3 }
+};

package/dist/src/server.js

@@ -0,0 +1,92 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+import { runTwoTierSearch } from "./engines.js";
+import { fetchAndExtract } from "./extract.js";
+const toInt = (v, def) => {
+    const n = Number(v);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : def;
+};
+const DEFAULT_LIMIT = toInt(process.env.MAX_RESULTS, 10);
+const server = new McpServer({ name: "mcp-web-search", version: "1.0.0" });
+server.registerTool("search_web", {
+    title: "Web Search (Fast: DuckDuckGo, Deep: Puppeteer/Bing)",
+    description: "Two-tier web search: runs fast DuckDuckGo HTML search by default, escalates to Puppeteer/Bing if results are insufficient. No API keys required.",
+    inputSchema: {
+        q: z.string(),
+        limit: z.number().int().min(1).max(50).default(DEFAULT_LIMIT).optional(),
+        lang: z.string().default("en").optional(),
+        mode: z.enum(["fast", "deep", "auto"]).default("auto").optional()
+    }
+}, async ({ q, limit = DEFAULT_LIMIT, lang = "en", mode = "auto" }) => {
+    const res = await runTwoTierSearch({ q, limit: Math.min(Math.max(1, limit), 50), lang, mode });
+    const payload = { ...res, items: res.items.slice(0, limit) };
+    return { content: [{ type: "text", text: JSON.stringify(payload, null, 2) }] };
+});
+server.registerTool("fetch_url", {
+    title: "Fetch and Extract URL Content",
+    description: "Fetches content from a URL (HTML/PDF) and extracts readable text. Supports truncation modes: compact (~3000 chars), standard (~8000 chars, default), full (no truncation). Output formats: markdown (default), text, html.",
+    inputSchema: {
+        url: z.string().url(),
+        mode: z.enum(["compact", "standard", "full"]).optional(),
+        max_length: z.number().int().min(1000).max(100000).optional(),
+        format: z.enum(["markdown", "text", "html"]).optional()
+    }
+}, async ({ url, mode, max_length, format }) => {
+    const doc = await fetchAndExtract(url, { mode, max_length, format });
+    return { content: [{ type: "text", text: JSON.stringify(doc, null, 2) }] };
+});
+server.registerTool("summarize_url", {
+    title: "Summarize URL Content",
+    description: "Fetches content from a URL and generates a concise summary.",
+    inputSchema: { url: z.string().url() }
+}, async ({ url }) => {
+    const doc = await fetchAndExtract(url);
+    try {
+        const content = doc.markdown || doc.text || "";
+        const prompt = `Provide a concise summary (<=10 sentences) of the following content:\n\nTitle: ${doc.title || "(none)"}\nURL: ${doc.url}\n\n--- Content ---\n${content.slice(0, 12000)}`;
+        const resp = await server.server.createMessage({
+            messages: [{ role: "user", content: { type: "text", text: prompt } }],
+            maxTokens: 800
+        });
+        const text = resp.content && resp.content.type === "text"
+            ? resp.content.text
+            : "(unable to generate summary)";
+        return { content: [{ type: "text", text }] };
+    }
+    catch {
+        const fallback = (doc.markdown || doc.text || "").slice(0, 2000);
+        return { content: [{ type: "text", text: fallback || "(no content to summarize)" }] };
+    }
+});
+server.registerTool("wiki_get", {
+    title: "Wikipedia: Get Summary",
+    description: "Retrieves a Wikipedia summary for a given title. Supports multiple languages (default: en).",
+    inputSchema: { title: z.string(), lang: z.string().default("en").optional() }
+}, async ({ title, lang = "en" }) => {
+    const { wikiGetSummary } = await import("./wikipedia.js");
+    const summary = await wikiGetSummary(title, lang);
+    return { content: [{ type: "text", text: JSON.stringify(summary, null, 2) }] };
+});
+server.registerTool("wiki_multi", {
+    title: "Wikipedia: Multi-Language Summary",
+    description: "Retrieves Wikipedia summaries in multiple languages for a given term. Uses langlinks to map titles accurately across languages.",
+    inputSchema: {
+        term: z.string(),
+        baseLang: z.string().default("en").optional(),
+        langs: z.array(z.string()).default(["en"]).optional()
+    }
+}, async ({ term, baseLang = "en", langs = ["en"] }) => {
+    const { wikiGetMultiSummary } = await import("./wikipedia.js");
+    const out = await wikiGetMultiSummary(term, baseLang, langs);
+    return { content: [{ type: "text", text: JSON.stringify(out, null, 2) }] };
+});
+async function main() {
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error("mcp-web-search ready (stdio)...");
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/src/wikipedia.js

@@ -0,0 +1,102 @@
+function uaHeaders(lang = process.env.LANG_DEFAULT || "en") {
+    const ua = process.env.USER_AGENT || "mcp-web-search/1.0";
+    const accept = lang === "en" ? "en-US,en;q=0.9" : `${lang};q=0.9,en;q=0.8`;
+    return { "User-Agent": ua, "Accept-Language": accept };
+}
+function toMs(env, def) {
+    const n = Number(env);
+    return Number.isFinite(n) && n > 0 ? n : def;
+}
+async function fetchWithTimeout(input, init = {}, timeoutMs = 15000) {
+    const controller = new AbortController();
+    const t = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(input, { ...init, signal: controller.signal });
+    }
+    finally {
+        clearTimeout(t);
+    }
+}
+export async function wikiGetSummary(title, lang = "en") {
+    const base = `https://${lang}.wikipedia.org`;
+    const sumUrl = new URL(`${base}/api/rest_v1/page/summary/${encodeURIComponent(title)}`);
+    try {
+        const sres = await fetchWithTimeout(sumUrl, { headers: uaHeaders(lang) }, toMs(process.env.HTTP_TIMEOUT, 15000));
+        if (!sres.ok) {
+            return { lang, title, url: `${base}/wiki/${encodeURIComponent(title)}` };
+        }
+        const s = (await sres.json());
+        return {
+            lang,
+            title: s.title ?? title,
+            url: s.content_urls?.desktop?.page ?? `${base}/wiki/${encodeURIComponent(title)}`,
+            description: s.description,
+            extract: s.extract,
+            thumbnailUrl: s.thumbnail?.source
+        };
+    }
+    catch {
+        return { lang, title, url: `${base}/wiki/${encodeURIComponent(title)}` };
+    }
+}
+async function wikiGetLanglinks(baseTitle, baseLang) {
+    const base = `https://${baseLang}.wikipedia.org/w/api.php`;
+    const url = new URL(base);
+    url.searchParams.set("action", "query");
+    url.searchParams.set("titles", baseTitle);
+    url.searchParams.set("prop", "langlinks");
+    url.searchParams.set("lllimit", "max");
+    url.searchParams.set("format", "json");
+    try {
+        const res = await fetchWithTimeout(url, { headers: uaHeaders(baseLang) }, toMs(process.env.HTTP_TIMEOUT, 15000));
+        if (!res.ok)
+            return {};
+        const data = (await res.json());
+        const pages = data?.query?.pages;
+        const first = pages && Object.values(pages)[0];
+        const ll = first?.langlinks || [];
+        const map = {};
+        for (const item of ll)
+            map[item.lang] = item["*"];
+        return map;
+    }
+    catch {
+        return {};
+    }
+}
+export async function wikiGetMultiSummary(term, baseLang = "en", langs = ["en"]) {
+    const want = Array.from(new Set(langs.map(s => s.trim().toLowerCase()).filter(Boolean)));
+    if (!want.includes(baseLang))
+        want.unshift(baseLang);
+    const base = await wikiGetSummary(term, baseLang);
+    const langlinks = await wikiGetLanglinks(base.title, baseLang);
+    const items = {};
+    const resolved = {};
+    items[baseLang] = base;
+    resolved[baseLang] = { title: base.title, source: "base" };
+    const tasks = want
+        .filter(l => l !== baseLang)
+        .map(async (l) => {
+        let title;
+        let source = "none";
+        if (langlinks[l]) {
+            title = langlinks[l];
+            source = "langlinks";
+        }
+        else {
+            title = term;
+            source = "direct";
+        }
+        try {
+            const sum = await wikiGetSummary(title, l);
+            items[l] = sum;
+            resolved[l] = { title: sum.title, source };
+        }
+        catch {
+            items[l] = null;
+            resolved[l] = { title, source: "none" };
+        }
+    });
+    await Promise.all(tasks);
+    return { baseLang, base, items, resolved };
+}

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@zhafron/mcp-web-search",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "MCP server: DuckDuckGo HTML search, Wikipedia summaries, and URL content extraction — no API keys required.",
+  "main": "dist/src/server.js",
+  "scripts": {
+    "build": "tsc -p .",
+    "start": "node dist/src/server.js",
+    "dev": "tsx src/server.ts",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\""
+  },
+  "keywords": [
+    "mcp",
+    "modelcontextprotocol",
+    "web",
+    "search",
+    "extraction",
+    "html",
+    "pdf",
+    "readability",
+    "turndown",
+    "ai"
+  ],
+  "author": "tickernelz",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tickernelz/mcp-web-search.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "package.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.17.0",
+    "@mozilla/readability": "^0.6.0",
+    "jsdom": "^24.1.0",
+    "pdf-parse": "^1.1.1",
+    "puppeteer-core": "^23.11.1",
+    "turndown": "^7.2.2",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/jest": "^29.5.14",
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^20.11.30",
+    "@types/turndown": "^5.0.6",
+    "jest": "^29.7.0",
+    "prettier": "^3.8.0",
+    "ts-jest": "^29.2.5",
+    "tsx": "^4.19.0",
+    "typescript": "^5.5.4"
+  }
+}