@andrhamm/web-mcp 0.0.0-stage → 0.2.1

package/dist/src/claude.js

@@ -0,0 +1,224 @@
+// The only Claude integration: the unmodified `claude` binary in print mode, on the operator's own sign-in.
+import { execFile, spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { MODEL_RESULT, ToolError } from "./contract.js";
+import { limiter, QueueFull } from "./limit.js";
+// Nesting markers (a stdio MCP server inherits them from Claude Code) and API keys (subscription only: never bill the API).
+const STRIP = ["CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_CHILD_SESSION", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"];
+export function childEnv(env = process.env) {
+    const e = { ...env };
+    for (const k of STRIP)
+        delete e[k];
+    return e;
+}
+const bin = () => process.env.CLAUDE_BIN || "claude";
+const envInt = (name, fallback, min) => {
+    const v = Number.parseInt(process.env[name] ?? "", 10);
+    return Number.isFinite(v) && v >= min ? v : fallback;
+};
+// Every run spends the operator's subscription, so concurrency is small and the wait queue is bounded.
+const run = limiter(envInt("WEB_MCP_CONCURRENCY", 2, 1), envInt("WEB_MCP_QUEUE", 8, 0));
+export function claudeArgs(o) {
+    return [
+        "-p",
+        "--model",
+        process.env.WEB_MCP_MODEL || "haiku",
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--json-schema",
+        o.schema,
+        "--system-prompt",
+        o.system,
+        "--max-turns",
+        String(o.maxTurns),
+        "--permission-prompts",
+        "none",
+        "--tools",
+        o.tools.join(","),
+        // --tools only makes tools available; without pre-approval --permission-prompts none denies them.
+        ...(o.tools.length ? ["--allowed-tools", o.tools.join(",")] : []),
+        // No user settings, hooks, plugins, MCP servers, slash commands or session files: only the tools above.
+        "--strict-mcp-config",
+        "--setting-sources",
+        "",
+        "--exclude-dynamic-system-prompt-sections",
+        "--disable-slash-commands",
+        "--no-session-persistence",
+    ];
+}
+const NO_RESULT = "Claude ended without a result";
+const blocks = (e) => (Array.isArray(e.message?.content) ? e.message.content : []);
+/** Read the stream-json events: real tool calls for meta, and the final result. */
+export function parseStream(stdout) {
+    const calls = new Map();
+    const ok = new Set();
+    let result;
+    for (const line of stdout.split("\n")) {
+        if (!line.trim())
+            continue;
+        let e;
+        try {
+            e = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (e.type === "assistant") {
+            for (const b of blocks(e)) {
+                if (b.type !== "tool_use")
+                    continue;
+                if (!b.id)
+                    continue;
+                if (b.name === "WebSearch" && b.input?.query)
+                    calls.set(b.id, { name: b.name, arg: String(b.input.query) });
+                if (b.name === "WebFetch" && b.input?.url)
+                    calls.set(b.id, { name: b.name, arg: String(b.input.url) });
+            }
+        }
+        else if (e.type === "user") {
+            for (const b of blocks(e)) {
+                if (b?.type === "tool_result" && b.is_error !== true && b.tool_use_id)
+                    ok.add(b.tool_use_id);
+            }
+        }
+        else if (e.type === "result")
+            result = e;
+    }
+    // Count only calls that ran (a denied or failed call has an error result).
+    const ran = [...calls].filter(([id]) => ok.has(id)).map(([, c]) => c);
+    const queries = ran.filter((c) => c.name === "WebSearch").map((c) => c.arg);
+    const pages = ran.filter((c) => c.name === "WebFetch").map((c) => c.arg);
+    if (result?.permission_denials?.length)
+        console.error(`[web-mcp] claude denied ${result.permission_denials.length} tool calls`);
+    if (!result)
+        throw new ToolError("bad_output", NO_RESULT);
+    if (result.is_error || result.structured_output === undefined) {
+        // Classify failures only: a successful answer may well mention limits or authentication.
+        const text = String(result.result ?? "");
+        if (result.api_error_status === 429 || /usage limit|rate limit/i.test(text))
+            throw new ToolError("rate_limited", "The Claude subscription limit was reached", "Try again after the limit resets");
+        if (/not logged in|please run \/login|invalid api key|oauth token/i.test(text))
+            throw new ToolError("claude_not_signed_in", "Claude Code is not signed in", "Run `claude` once and sign in");
+        const why = String(result.terminal_reason ?? result.subtype ?? "");
+        throw new ToolError("bad_output", `Claude returned no structured result (${why || "unknown reason"})`, /turn/i.test(why) ? "Ask a narrower question or use more depth" : undefined);
+    }
+    const parsed = MODEL_RESULT.safeParse(result.structured_output);
+    if (!parsed.success)
+        throw new ToolError("bad_output", "Claude's result did not match the schema");
+    return { output: parsed.data, queries, pages, model: Object.keys(result.modelUsage ?? {})[0] ?? "haiku", usage: usageOf(result) };
+}
+/** Tokens and estimated cost of the whole run, summed over every model it used (WebFetch reads pages with its own call). */
+function usageOf(result) {
+    const models = Object.values(result.modelUsage ?? {});
+    const sum = (k) => models.reduce((n, m) => n + (m[k] ?? 0), 0);
+    return {
+        input_tokens: sum("inputTokens"),
+        output_tokens: sum("outputTokens"),
+        cache_read_tokens: sum("cacheReadInputTokens"),
+        cache_write_tokens: sum("cacheCreationInputTokens"),
+        cost_usd: Math.round((result.total_cost_usd ?? sum("costUSD")) * 1e5) / 1e5,
+    };
+}
+const cancelled = () => new ToolError("cancelled", "The caller cancelled the request");
+/** How long a stopped claude process gets to exit after SIGTERM before it is killed. */
+const KILL_GRACE_MS = 3000;
+export const runClaude = (o) => withSlot(o, (deadline) => spawnClaude(o, deadline));
+/**
+ * Run `task` in a Claude slot: wait in the bounded queue, then run with a deadline counted from the call.
+ * Work that only matters if Claude will run (the Brave head start) belongs inside `task`, so a `busy` refusal costs nothing.
+ */
+export async function withSlot(o, task) {
+    const deadline = Date.now() + o.timeoutMs;
+    const wait = AbortSignal.any([AbortSignal.timeout(o.timeoutMs), ...(o.signal ? [o.signal] : [])]);
+    try {
+        return await run(() => task(deadline), wait);
+    }
+    catch (e) {
+        if (e instanceof ToolError)
+            throw e;
+        if (e instanceof QueueFull)
+            throw new ToolError("busy", `Too many research calls at once: ${e.message}`, "Try again shortly");
+        if (o.signal?.aborted)
+            throw cancelled();
+        if (wait.aborted)
+            throw new ToolError("timeout", `No free slot within ${Math.round(o.timeoutMs / 1000)} s`, "The server is busy; try again shortly");
+        throw e;
+    }
+}
+export async function spawnClaude(o, deadline) {
+    if (o.signal?.aborted)
+        throw cancelled();
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+        throw new ToolError("timeout", "No time left after waiting for a free slot", "The server is busy; try again shortly");
+    const cwd = await mkdtemp(join(tmpdir(), "web-mcp-")); // empty: no project CLAUDE.md or .mcp.json
+    try {
+        let stderr = "";
+        const stdout = await new Promise((resolve, reject) => {
+            const p = spawn(bin(), claudeArgs(o), { cwd, env: childEnv(), stdio: ["pipe", "pipe", "pipe"] });
+            let out = "";
+            let err = "";
+            let killer;
+            const stop = (e) => {
+                p.kill("SIGTERM");
+                killer = setTimeout(() => p.kill("SIGKILL"), KILL_GRACE_MS); // force a run that ignores the polite stop
+                killer.unref();
+                reject(e);
+            };
+            const onAbort = () => stop(cancelled());
+            const timer = setTimeout(() => stop(new ToolError("timeout", `Research took longer than ${Math.round(o.timeoutMs / 1000)} s`, "Ask a narrower question or use less depth")), remaining);
+            o.signal?.addEventListener("abort", onAbort, { once: true });
+            const done = () => {
+                clearTimeout(timer);
+                o.signal?.removeEventListener("abort", onAbort);
+            };
+            p.stdout.on("data", (d) => (out += d));
+            p.stderr.on("data", (d) => (err += d));
+            p.on("error", (e) => {
+                done();
+                reject(e.code === "ENOENT"
+                    ? new ToolError("no_backend", "The claude command was not found", "Install Claude Code, or set CLAUDE_BIN")
+                    : e);
+            });
+            p.on("close", () => {
+                clearTimeout(killer);
+                done();
+                stderr = err.trim();
+                if (stderr)
+                    console.error(`[web-mcp] claude stderr: ${stderr.slice(0, 500)}`);
+                resolve(out);
+            });
+            p.stdin.on("error", () => { }); // the child may exit before reading stdin; its exit is handled above
+            p.stdin.end(o.prompt);
+        });
+        try {
+            return parseStream(stdout);
+        }
+        catch (e) {
+            if (e instanceof ToolError && e.message === NO_RESULT && stderr)
+                throw new ToolError("bad_output", `claude failed: ${(stderr.split("\n")[0] ?? "").slice(0, 200)}`);
+            throw e;
+        }
+    }
+    finally {
+        await rm(cwd, { recursive: true, force: true });
+    }
+}
+let signedIn = false;
+/** True once `claude auth status` reports a sign-in; rechecked on every call until it does. Never reads tokens. */
+export async function checkSignedIn() {
+    if (signedIn)
+        return;
+    const out = await new Promise((resolve) => execFile(bin(), ["auth", "status", "--json"], { env: childEnv(), timeout: 15_000 }, (e, stdout) => resolve(e ? "" : stdout)));
+    try {
+        signedIn = JSON.parse(out).loggedIn === true;
+    }
+    catch {
+        signedIn = false;
+    }
+    if (!signedIn)
+        throw new ToolError("claude_not_signed_in", "Claude Code is not signed in, or the claude command is missing", "Run `claude` once and sign in, or set CLAUDE_BIN");
+}

package/dist/src/contract.js

@@ -0,0 +1,84 @@
+// Tool inputs and the result shape. Consumers may copy this file to validate `format: "json"` replies.
+import { z } from "zod";
+export const MAX_TOKENS = z
+    .number()
+    .int()
+    .min(50)
+    .max(4000)
+    .describe("Ceiling on the reply size in tokens. The reply is as short as the goal allows, never longer than this");
+const QUESTION = z.string().trim().min(1).max(500).describe("The question to answer from the web");
+const GOAL = z
+    .string()
+    .trim()
+    .min(1)
+    .max(500)
+    .describe("What you will do with the answer; only what serves it is kept. Never put secrets here: it leaves the machine");
+const URL_ = z.string().url().describe("The page to read");
+const FORMAT = z
+    .enum(["markdown", "json"])
+    .default("markdown")
+    .describe("markdown (default, fewest tokens) or json (the result object, for code)");
+const DEPTH = z.enum(["quick", "standard", "deep"]).default("quick").describe("How many searches and page reads it may spend");
+const RECENCY = z
+    .enum(["any", "day", "week", "month", "year"])
+    .default("any")
+    .describe("Prefer sources from this window. Exact for the Brave head start, best effort for other searches");
+const DOMAINS = z
+    .object({ include: z.array(z.string()).max(20).optional(), exclude: z.array(z.string()).max(20).optional() })
+    .optional()
+    .describe("Hosts to search only, or to leave out");
+const CACHE = z.enum(["use", "refresh"]).default("use").describe("refresh: ignore cached answers and pages");
+export const LOOKUP_INPUT = z.object({ question: QUESTION, max_tokens: MAX_TOKENS.default(300) });
+export const READ_INPUT = z.object({
+    url: URL_,
+    question: z.string().trim().min(1).max(500).optional().describe("What you want from the page; omit for a summary"),
+    max_tokens: MAX_TOKENS.default(300),
+});
+export const RESEARCH_INPUT = z.object({ question: QUESTION, goal: GOAL, max_tokens: MAX_TOKENS.default(400), format: FORMAT });
+export const PAGE_INPUT = z.object({ url: URL_, goal: GOAL, max_tokens: MAX_TOKENS.default(400), format: FORMAT });
+export const INVESTIGATE_INPUT = RESEARCH_INPUT.extend({ depth: DEPTH, recency: RECENCY, domains: DOMAINS, cache: CACHE });
+export const EXTRACT_INPUT = z.object({
+    url: URL_,
+    goal: z.string().trim().min(1).max(500).describe("What you need from the page; the passages that match it are kept"),
+    max_tokens: MAX_TOKENS.default(1000),
+});
+export const SEARCH_INPUT = z.object({
+    query: z.string().trim().min(1).max(400).describe("Search query"),
+    count: z.number().int().min(1).max(20).default(8).describe("Number of results"),
+    recency: RECENCY,
+    domains: DOMAINS,
+});
+function modelResult(l) {
+    const text = (max) => (l ? z.string().max(max) : z.string());
+    return z.object({
+        answer: text(l?.answerChars ?? 0),
+        confidence: z.enum(["high", "medium", "low"]),
+        findings: z
+            .array(z.object({
+            claim: text(200),
+            source: z.number().int().min(1),
+            quote: text(220).optional(),
+            date: z.string().optional(),
+        }))
+            .max(l ? l.findings : 50),
+        sources: z.array(z.object({ url: z.string(), title: z.string(), published: z.string().optional() })).max(l ? l.findings + 3 : 50),
+        gaps: z.array(text(160)).max(l ? 3 : 20),
+    });
+}
+/** What the model returns. `source` is the 1-based position in `sources`, matching [n] markers in `answer`. Parsing is lenient. */
+export const MODEL_RESULT = modelResult();
+/** The JSON schema handed to `claude --json-schema` for one request, with its length limits. */
+export function modelResultSchema(limits) {
+    // Claude Code's validator rejects the draft 2020-12 `$schema` stamp zod adds; the schema itself is plain draft-07 compatible.
+    const { $schema: _drop, ...schema } = z.toJSONSchema(modelResult(limits));
+    return JSON.stringify(schema);
+}
+export class ToolError extends Error {
+    code;
+    hint;
+    constructor(code, message, hint) {
+        super(message);
+        this.code = code;
+        this.hint = hint;
+    }
+}

package/dist/src/index.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+import { createServer } from "node:http";
+import { localhostHostValidation, localhostOriginValidation, toNodeHandler } from "@modelcontextprotocol/node";
+import { createMcpHandler } from "@modelcontextprotocol/server";
+import { serveStdio } from "@modelcontextprotocol/server/stdio";
+import { buildServer, TIERS, VERSION } from "./server.js";
+const USAGE = `web-mcp ${VERSION}
+Usage: web-mcp [--http [host:]port] [--tiers basic,standard,advanced,direct]
+  --http    serve Streamable HTTP at /mcp instead of stdio (default host 127.0.0.1)
+  --tiers   which tool tiers to expose (default: all)
+Environment: BRAVE_API_KEY or BRAVE_API_KEY_FILE, CLAUDE_BIN, WEB_MCP_CACHE_DIR,
+  WEB_MCP_CONCURRENCY, WEB_MCP_MODEL, WEB_MCP_HTTP_TOKEN`;
+function arg(name) {
+    const i = process.argv.indexOf(name);
+    return i >= 0 ? (process.argv[i + 1] ?? "") : undefined;
+}
+if (process.argv.includes("--help") || process.argv.includes("-h")) {
+    console.error(USAGE);
+    process.exit(0);
+}
+const tierArg = arg("--tiers");
+const tiers = new Set(tierArg ? tierArg.split(",").map((t) => t.trim()) : TIERS);
+for (const t of tiers) {
+    if (!TIERS.includes(t)) {
+        console.error(`Unknown tier "${t}". ${USAGE}`);
+        process.exit(2);
+    }
+}
+const factory = (ctx) => buildServer(tiers, ctx.requestInfo?.signal);
+const http = arg("--http");
+if (http === undefined) {
+    serveStdio(factory, { onerror: (e) => console.error(`[web-mcp] ${e.message}`) });
+}
+else {
+    const [host, port] = http.includes(":")
+        ? [http.slice(0, http.lastIndexOf(":")), http.slice(http.lastIndexOf(":") + 1)]
+        : ["127.0.0.1", http];
+    const token = process.env.WEB_MCP_HTTP_TOKEN;
+    const loopback = ["127.0.0.1", "localhost", "::1", "[::1]"].includes(host);
+    if (!loopback && !token) {
+        console.error(`Refusing to listen on ${host} without WEB_MCP_HTTP_TOKEN: the server spends your Claude subscription.`);
+        process.exit(2);
+    }
+    // A runtime that notices a dropped client only on its next write would otherwise keep a cancelled run going up to the
+    // SDK's 15 s default keep-alive; at 2 s an abandoned Claude run stops within about 2 s.
+    const handle = toNodeHandler(createMcpHandler(factory, { keepAliveMs: 2000 }));
+    const validHost = localhostHostValidation();
+    const validOrigin = localhostOriginValidation();
+    createServer((req, res) => {
+        if (new URL(req.url ?? "/", "http://x").pathname !== "/mcp") {
+            res.writeHead(404).end();
+            return;
+        }
+        if (token) {
+            if (req.headers.authorization !== `Bearer ${token}`) {
+                res.writeHead(401, { "www-authenticate": "Bearer" }).end();
+                return;
+            }
+        }
+        else if (!validHost(req, res) || !validOrigin(req, res))
+            return; // DNS-rebinding guard for tokenless loopback use
+        void handle(req, res);
+    }).listen(Number(port), host, () => console.error(`[web-mcp] ${VERSION} listening on http://${host}:${port}/mcp`));
+}

package/dist/src/limit.js

@@ -0,0 +1,41 @@
+/** Raised when every slot is busy and the wait queue is full. */
+export class QueueFull extends Error {
+}
+/**
+ * Counting semaphore: at most `n` tasks run at once and up to `maxQueue` more wait in arrival order.
+ * A finishing task hands its slot straight to the next waiter, so a newcomer can never jump the queue.
+ * A waiter whose `signal` aborts leaves the queue and rejects with the signal's reason.
+ */
+export function limiter(n, maxQueue = Number.POSITIVE_INFINITY) {
+    let active = 0;
+    const waiting = [];
+    return async function run(task, signal) {
+        signal?.throwIfAborted();
+        if (active < n)
+            active++;
+        else {
+            if (waiting.length >= maxQueue)
+                throw new QueueFull(`${waiting.length} calls are already waiting`);
+            await new Promise((resolve, reject) => {
+                waiting.push(resolve);
+                signal?.addEventListener("abort", () => {
+                    const i = waiting.indexOf(resolve);
+                    if (i < 0)
+                        return; // already started
+                    waiting.splice(i, 1);
+                    reject(signal.reason);
+                }, { once: true });
+            });
+        }
+        try {
+            return await task();
+        }
+        finally {
+            const next = waiting.shift();
+            if (next)
+                next();
+            else
+                active--;
+        }
+    };
+}

package/dist/src/reader.js

@@ -0,0 +1,207 @@
+// Fetch one public page, extract its main text as markdown, and keep the passages that serve a goal.
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+import { Readability } from "@mozilla/readability";
+import { parseHTML } from "linkedom";
+import MiniSearch from "minisearch";
+import TurndownService from "turndown";
+import { ToolError } from "./contract.js";
+const MAX_BYTES = 2 * 1024 * 1024;
+const TIMEOUT_MS = 20_000;
+const MAX_REDIRECTS = 5;
+/** Loopback, private, link-local, CGNAT, unspecified and multicast ranges, IPv4 and IPv6 (including IPv4-mapped). */
+export function isPrivateAddress(ip) {
+    const v = ip.toLowerCase().replace(/^\[|\]$/g, "");
+    const mapped = v.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+    if (mapped?.[1])
+        return isPrivateAddress(mapped[1]);
+    if (isIP(v) === 4) {
+        const [a = 0, b = 0] = v.split(".").map(Number);
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && b === 168) ||
+            a >= 224);
+    }
+    if (isIP(v) === 6)
+        return v === "::" || v === "::1" || /^f[cd]/.test(v) || /^fe[89ab]/.test(v) || /^ff/.test(v);
+    return true; // not an IP: callers resolve first
+}
+async function assertPublic(url) {
+    if (url.protocol !== "http:" && url.protocol !== "https:")
+        throw new ToolError("blocked_url", `Only http and https URLs are read (${url.protocol})`);
+    const host = url.hostname.replace(/^\[|\]$/g, "");
+    if (host === "localhost" || host.endsWith(".localhost"))
+        throw new ToolError("blocked_url", "Local addresses are not read");
+    // shortcut: checks DNS before the fetch, so a rebinding host could still flip; pin the resolved address if this runs beside sensitive services.
+    const addrs = isIP(host) ? [host] : (await lookup(host, { all: true }).catch(() => [])).map((a) => a.address);
+    if (!addrs.length)
+        throw new ToolError("fetch_failed", `Could not resolve ${host}`);
+    if (addrs.some(isPrivateAddress))
+        throw new ToolError("blocked_url", "Private and local network addresses are not read");
+}
+export async function fetchPage(rawUrl) {
+    let url = new URL(rawUrl);
+    let res;
+    for (let i = 0; i <= MAX_REDIRECTS; i++) {
+        await assertPublic(url);
+        res = await fetch(url, {
+            redirect: "manual",
+            signal: AbortSignal.timeout(TIMEOUT_MS),
+            headers: { "user-agent": "web-mcp/0.1", accept: "text/html,application/xhtml+xml,text/plain;q=0.9" },
+        }).catch((e) => {
+            throw new ToolError(e.name === "TimeoutError" ? "timeout" : "fetch_failed", `Could not fetch ${url.host}: ${e.message}`);
+        });
+        const loc = res.headers.get("location");
+        if (res.status >= 300 && res.status < 400 && loc) {
+            url = new URL(loc, url);
+            continue;
+        }
+        break;
+    }
+    if (!res?.ok)
+        throw new ToolError("fetch_failed", `The page answered HTTP ${res?.status ?? "redirect loop"}`);
+    const type = res.headers.get("content-type") ?? "";
+    if (!/text\/html|application\/xhtml|text\/plain/.test(type))
+        throw new ToolError("fetch_failed", `Unsupported content type ${type.split(";")[0] || "unknown"}`, "Only HTML and plain text pages are read");
+    const host = url.host;
+    const bytes = await readCapped(res).catch((e) => {
+        // The timeout signal also covers the body, so a slow download fails here, after the headers arrived.
+        throw new ToolError(e.name === "TimeoutError" ? "timeout" : "fetch_failed", `Could not read ${host}: ${e.message}`);
+    });
+    const body = decode(bytes, type);
+    return /text\/plain/.test(type) ? { url: url.href, title: url.href, markdown: stripInvisible(body).trim() } : extract(body, url.href);
+}
+/**
+ * Decode with the charset from the Content-Type header, else from a <meta> tag near the top, else UTF-8.
+ * Labels follow the WHATWG Encoding standard (so "iso-8859-1" decodes as windows-1252); unknown labels fall back to UTF-8.
+ */
+export function decode(bytes, contentType) {
+    const head = new TextDecoder("windows-1252").decode(bytes.subarray(0, 2048)); // ASCII-safe view for sniffing
+    const label = /charset\s*=\s*["']?([\w.:-]+)/i.exec(contentType)?.[1] ?? /<meta[^>]+charset\s*=\s*["']?([\w.:-]+)/i.exec(head)?.[1] ?? "utf-8";
+    try {
+        return new TextDecoder(label).decode(bytes);
+    }
+    catch {
+        return new TextDecoder("utf-8").decode(bytes);
+    }
+}
+async function readCapped(res) {
+    if (!res.body)
+        return new Uint8Array();
+    const reader = res.body.getReader();
+    const parts = [];
+    let size = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        const room = MAX_BYTES - size;
+        parts.push(value.byteLength > room ? value.subarray(0, room) : value);
+        size += Math.min(value.byteLength, room);
+        if (size >= MAX_BYTES) {
+            await reader.cancel();
+            break; // keep what fits; the main text is usually near the top
+        }
+    }
+    return Buffer.concat(parts);
+}
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
+turndown.remove(["script", "style", "noscript", "iframe"]);
+turndown.remove((n) => n.nodeName.toLowerCase() === "svg");
+// Token economy: link targets and images are noise for a reader model; tables become one pipe row per line.
+turndown.addRule("linkText", { filter: "a", replacement: (content) => content });
+turndown.addRule("noImages", { filter: "img", replacement: () => "" }); // added rules outrank remove() for void elements
+turndown.addRule("cell", { filter: ["th", "td"], replacement: (content) => ` ${content.replace(/\s+/g, " ").trim()} |` });
+turndown.addRule("row", { filter: "tr", replacement: (content) => `\n|${content}\n` });
+turndown.addRule("table", { filter: "table", replacement: (content) => `\n\n${content.replace(/\n+/g, "\n").trim()}\n\n` });
+// Characters a person cannot see but a model still reads, so they can carry hidden instructions: soft hyphen, zero-width
+// characters and joiners, bidi controls, invisible operators, BOM, variation selectors and Unicode tag characters.
+// Removing joiners can split emoji sequences and alter some scripts slightly; the text stays readable.
+const INVISIBLE = 
+// biome-ignore lint/suspicious/noMisleadingCharacterClass: each invisible code point, joiners included, is matched on its own on purpose
+/[\u00AD\u061C\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u206F\uFE00-\uFE0F\uFEFF\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]/gu;
+export const stripInvisible = (s) => s.replace(INVISIBLE, "");
+// Inline styles that hide an element. Text hidden by stylesheets stays: the reader applies no CSS.
+const HIDDEN_STYLE = /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0(?:\.0+)?|font-size\s*:\s*0(?:\.0+)?(?:px|em|rem|pt|%)?)\s*(?:!important\s*)?(?:;|$)/i;
+export function extract(html, url) {
+    // linkedom roots a document without <html> at its first tag, which Readability rejects.
+    const source = /<html[\s>]/i.test(html) ? html : `<!doctype html><html><head></head><body>${html}</body></html>`;
+    const { document } = parseHTML(source);
+    for (const el of [...document.querySelectorAll("[hidden], [aria-hidden='true'], [style]")]) {
+        if (el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true" || HIDDEN_STYLE.test(el.getAttribute("style") ?? ""))
+            el.remove();
+    }
+    const title = document.querySelector("title")?.textContent?.trim() || url;
+    const plain = document.body?.textContent ?? ""; // fallback, taken before Readability rewrites the document
+    let article = null;
+    try {
+        article = new Readability(document).parse();
+    }
+    catch (e) {
+        console.error(`[web-mcp] readability failed on ${url}: ${e.message}`);
+    }
+    const markdown = article?.content ? turndown.turndown(article.content) : plain;
+    return {
+        url,
+        title: stripInvisible(article?.title?.trim() || title),
+        markdown: stripInvisible(markdown)
+            .replace(/\n{3,}/g, "\n\n")
+            .trim(),
+        published: article?.publishedTime ?? undefined,
+    };
+}
+/** Split markdown into passages of about 120 words, each carrying its nearest heading. */
+export function chunk(markdown) {
+    const chunks = [];
+    let heading = "";
+    let buf = [];
+    const words = () => buf.join(" ").split(/\s+/).length;
+    const flush = () => {
+        if (buf.length)
+            chunks.push((heading && !buf[0]?.startsWith("#") ? `${heading}\n` : "") + buf.join("\n\n"));
+        buf = [];
+    };
+    for (const para of markdown.split(/\n\s*\n/)) {
+        const p = para.trim();
+        if (!p)
+            continue;
+        if (/^#{1,6}\s/.test(p)) {
+            flush();
+            heading = p.split("\n")[0] ?? "";
+        }
+        buf.push(p);
+        if (words() >= 120)
+            flush();
+    }
+    flush();
+    return chunks;
+}
+/** Keep the first passage and the passages that best match `query`, in page order, within `maxChars`. */
+/** Passage indexes that match `query`, best first (BM25 with prefix and fuzzy matching). Unmatched passages are left out. */
+export function rank(chunks, query) {
+    const ms = new MiniSearch({ fields: ["text"], storeFields: [] });
+    ms.addAll(chunks.map((text, id) => ({ id, text })));
+    return ms.search(query, { prefix: true, fuzzy: 0.2, combineWith: "OR" }).map((h) => h.id);
+}
+export function select(chunks, query, maxChars) {
+    if (chunks.join("\n\n").length <= maxChars)
+        return chunks.join("\n\n");
+    const ranked = rank(chunks, query);
+    const len = (i) => chunks[i]?.length ?? 0;
+    const keep = new Set([0]);
+    let used = len(0);
+    for (const id of ranked) {
+        if (keep.has(id) || used + len(id) > maxChars)
+            continue;
+        keep.add(id);
+        used += len(id) + 2;
+    }
+    return [...keep]
+        .sort((a, b) => a - b)
+        .map((i) => chunks[i] ?? "")
+        .join("\n\n");
+}