@url2md-io/mcp 0.1.0

package/dist/index.js

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+import { realpathSync } from "node:fs";
+import { McpServer } from "@modelcontextprotocol/server";
+import { serveStdio } from "@modelcontextprotocol/server/stdio";
+import * as z from "zod/v4";
+import { Budget, ceilingLooksWrong, CEILING_WORTH_MENTIONING_USD, convert, discoverPricing, payerAddress, readConfig, RENDER_MODES, Url2mdToolError } from "./client.js";
+/**
+ * url2md as an MCP server: one tool, `fetch_markdown`, that turns any URL into clean Markdown and
+ * pays for the call from the user's own wallet over x402.
+ *
+ * Local stdio, not a hosted remote MCP. A remote server has no wallet of the caller's to pay with:
+ * it would have to pay from its own and then bill the caller some other way — an account, a key, an
+ * invoice — which is the exact thing x402 exists to remove. Running it beside the client keeps the
+ * key on the user's machine and the payment the user's own.
+ *
+ * The key is read once, at startup, and the only thing ever derived from it that leaves this process
+ * is the public address. Nothing writes it to stdout (which is the MCP wire), to stderr, to an error
+ * message or into a tool result.
+ */
+export const SERVER_NAME = "url2md";
+export const SERVER_VERSION = "0.1.0";
+export const TOOL_NAME = "fetch_markdown";
+export const inputSchema = z.object({
+    url: z.string().describe("The absolute http(s) URL to convert. Public hosts only: private, loopback and link-local addresses are refused, and the origin's robots.txt is obeyed."),
+    render: z
+        .enum(RENDER_MODES)
+        .default("auto")
+        .describe("auto (default): render with a headless browser only if the page looks client-rendered. static: never render, fetch the HTML as served. browser: always render — costs the same but takes longer and can fail on pages a plain fetch would have handled."),
+});
+/**
+ * Everything a call may not have is **absent**, never `null`. Zod emits a nullable field as
+ * `type: ["string","null"]`, and the reference inspector's `--strict` mode reports that several MCP
+ * clients read `type` as a single string and either reject the tool or drop the constraint. Absent
+ * is also the truer contract here: there is nothing a model can do with a `null` title that it
+ * cannot do with no title at all.
+ */
+const receiptSchema = z
+    .object({
+    amountUsd: z.string().describe("What this call cost, in USD. This is what the client signed, not what the service's receipt claims; a disagreement is reported in warnings."),
+    amountAtomic: z.string().describe("The same amount in atomic units of the asset (USDC has 6 decimals)."),
+    network: z.string(),
+    payTo: z.string(),
+    settled: z.boolean().describe("True when the service returned a settlement receipt naming an on-chain transaction. That is the service's word, not proof: follow explorerUrl to verify. False means the payment was authorized and the service answered without confirming it — see warnings."),
+    payer: z.string().optional().describe("The wallet that paid: yours."),
+    transaction: z.string().optional(),
+    explorerUrl: z.string().optional().describe("The settlement on a block explorer."),
+    ledgerId: z.string().optional().describe("url2md's own row for this payment, quotable if anything needs chasing."),
+})
+    .optional()
+    .describe("What this call cost. Present whenever a payment was authorized — absent only when nothing was paid at all (a free deployment). Check `settled`.");
+export const outputSchema = z.object({
+    markdown: z.string().describe("The page as Markdown."),
+    url: z.string(),
+    finalUrl: z.string().optional().describe("Where the request ended up after redirects."),
+    title: z.string().optional(),
+    source: z.string().describe("static | browser | pdf | origin-markdown | text — how the Markdown was produced."),
+    words: z.number(),
+    tokensEstimate: z.number(),
+    warnings: z.array(z.string()).describe("Non-fatal notes you should read: a browser render that fell back to the static result, or a payment the service did not confirm."),
+    payment: receiptSchema,
+    toll: z
+        .object({
+        amountUsd: z.string(),
+        payTo: z.string().optional(),
+        transaction: z.string().optional(),
+        explorerUrl: z.string().optional(),
+    })
+        .optional()
+        .describe("Present when the page itself charged an x402 toll that url2md paid on your behalf; the amount is already included in what you paid."),
+    paymentRounds: z.number().describe("402s answered: 1 for an ordinary page, 2 when the page charged a toll of its own."),
+    committedUsd: z.string().describe("Everything this call authorized, summed. Above `payment.amountUsd` when a toll quote replaced an earlier authorization the service can still redeem."),
+    signed: z
+        .array(z.object({
+        amountUsd: z.string(),
+        amountAtomic: z.string(),
+        network: z.string(),
+        payTo: z.string(),
+        payer: z.string(),
+        nonce: z.string().optional().describe("The EIP-3009 nonce. USDC emits AuthorizationUsed(payer, nonce) when it is redeemed, so this is how you find it on-chain."),
+        validBefore: z.string().optional().describe("After this instant the authorization can no longer be redeemed."),
+    }))
+        .describe("Every payment authorization this call put on the wire, in order. Empty on a free deployment."),
+});
+/** The tool result for a failure: `isError`, with the machine-readable cause as the text, never a sentence that reads like success. */
+export function errorResult(e) {
+    const err = e instanceof Url2mdToolError
+        ? e
+        : new Url2mdToolError("INTERNAL", e instanceof Error ? e.message : String(e), { retryable: false });
+    const payload = {
+        error: {
+            code: err.code,
+            message: err.message,
+            retryable: err.retryable,
+            ...(err.httpStatus === null ? {} : { httpStatus: err.httpStatus }),
+            ...(Object.keys(err.details).length ? { details: err.details } : {}),
+        },
+    };
+    return { content: [{ type: "text", text: JSON.stringify(payload, null, 2) }], isError: true };
+}
+/** The successful result: the Markdown as the text a model reads, the whole record as structured content. */
+export function successResult(c) {
+    const structured = {
+        markdown: c.result.markdown ?? "",
+        url: c.result.url,
+        ...present("finalUrl", c.result.finalUrl),
+        ...present("title", c.result.title),
+        source: c.result.source,
+        words: c.result.words ?? 0,
+        tokensEstimate: c.result.tokensEstimate ?? 0,
+        warnings: c.warnings,
+        ...present("payment", c.payment && dropNulls(c.payment)),
+        ...present("toll", c.toll && dropNulls(c.toll)),
+        paymentRounds: c.paymentRounds,
+        committedUsd: c.committedUsd,
+        signed: (c.signed ?? []).map((a) => dropNulls(a)),
+    };
+    // Half the MCP clients in use render `content` and drop `structuredContent`. Everything about the
+    // money lives in the structured half, so a call whose payment was never confirmed came back as
+    // clean Markdown and nothing else — the model saw a success. Anything the caller must act on is
+    // written above the Markdown, in words, where every client shows it.
+    const banner = paymentBanner(c);
+    return { content: [{ type: "text", text: banner ? `${banner}\n\n${structured.markdown}` : structured.markdown }], structuredContent: structured };
+}
+/**
+ * The line that goes above the Markdown when there is something about this call's money that a
+ * reader must not miss. Silent on the ordinary case: a settled payment with no warnings needs no
+ * preamble, and one on every call trains a reader to skip it.
+ */
+export function paymentBanner(c) {
+    const parts = [];
+    if (c.payment && !c.payment.settled) {
+        parts.push(`PAYMENT NOT CONFIRMED: $${c.payment.amountUsd} was authorized to ${c.payment.payTo} on ${c.payment.network}, and the service returned no usable settlement receipt. The content below may still be correct; the payment is not confirmed.`);
+    }
+    if (c.warnings.length)
+        parts.push(...c.warnings.map((w) => `NOTE: ${w}`));
+    return parts.length ? parts.join("\n") : null;
+}
+/** `{ key: value }` when there is a value, `{}` when there is not: absent, never null (see outputSchema). */
+function present(key, value) {
+    return value === null || value === undefined ? {} : { [key]: value };
+}
+/** The same rule one level down: the client fills unknown fields with null, the wire leaves them out. */
+function dropNulls(o) {
+    return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== null && v !== undefined));
+}
+/**
+ * What the tool says about itself. The price and the network are the ones this deployment actually
+ * publishes, read once at startup — a model deciding whether to spend needs the real number and,
+ * above all, needs to be able to tell a test network from one that costs real money. When `GET /`
+ * could not be reached the description says where the price is published instead of inventing one.
+ */
+export function toolDescription(config, pricing) {
+    const cost = pricing?.model === "x402" && pricing.priceAtomic
+        ? `Each call costs ${usdLabel(pricing.priceAtomic)} in ${pricing.asset ?? "USDC"} on ${pricing.network}${isTestNetwork(pricing.network) ? " — a TEST network, so the money is not real" : ""}, paid on-chain from your own wallet; the settlement receipt comes back with the result.`
+        : pricing?.model === "free"
+            ? `This deployment (${config.baseUrl}) is not charging: calls are free.`
+            : `Calls are paid per call over x402 from your own wallet; the live price is published at ${config.baseUrl}/.`;
+    return ("Fetch any web page or PDF and return it as clean Markdown, ready to read. Handles JavaScript-rendered pages (a headless browser is used when the page needs one) and PDFs. " +
+        `${cost} This server refuses to sign more than $${config.maxPriceUsd} for one call, or $${config.maxSpendUsd} in total for as long as it is running. ` +
+        "If the page itself charges an x402 toll, url2md pays it and the cost is included in what you pay. " +
+        "Errors, including a payment that could not be made, come back as an error result whose text is a JSON object with a stable `code`.");
+}
+const isTestNetwork = (network) => network === "eip155:84532" || (network ?? "").includes("sepolia");
+/** Atomic units of a 6-decimal asset as a short USD label. */
+function usdLabel(atomic) {
+    try {
+        const n = Number(BigInt(atomic)) / 1e6;
+        return n < 0.01 ? `$${n} (${(n * 100).toFixed(1)} cents)` : `$${n}`;
+    }
+    catch {
+        return `${atomic} atomic units`;
+    }
+}
+export function createServer(deps) {
+    const server = new McpServer({ name: SERVER_NAME, version: SERVER_VERSION }, { capabilities: { tools: {} } });
+    server.registerTool(TOOL_NAME, {
+        title: "Fetch a URL as Markdown",
+        description: toolDescription(deps.config, deps.pricingSnapshot ?? null),
+        inputSchema,
+        outputSchema,
+        // Not read-only: this tool signs a USDC transfer from the caller's wallet. Clients use
+        // readOnlyHint to decide what may run without asking a human, and a tool that spends money
+        // must never be in that set — it was, until review pointed at it.
+        annotations: { readOnlyHint: false, openWorldHint: true, idempotentHint: false },
+    }, async ({ url, render }) => {
+        try {
+            const run = deps.convertImpl ?? convert;
+            return successResult(await run({ url, render }, deps.config, await deps.pricing(), deps.fetchImpl, deps.budget));
+        }
+        catch (e) {
+            return errorResult(e);
+        }
+    });
+    return server;
+}
+/** stderr, never stdout: stdout is the MCP wire and anything written there corrupts the protocol. */
+function note(line) {
+    process.stderr.write(`${line}\n`);
+}
+/**
+ * The only thing a fatal error is allowed to say. Our own errors carry wording we wrote; anything
+ * else is named by its type and nothing more — a library that rejects a bad private key quotes the
+ * number it was given, which is the key, and stderr is what MCP clients spool to a log file
+ * (rule 12 of the service repository's CLAUDE.md).
+ */
+export function fatalLine(e) {
+    if (e instanceof Url2mdToolError)
+        return `@url2md-io/mcp: ${e.code}: ${e.message}`;
+    return `@url2md-io/mcp: INTERNAL: an unexpected ${e instanceof Error ? e.name : typeof e} was thrown during startup. Its message is withheld because it may quote a value that was given to this process.`;
+}
+export async function main() {
+    let config;
+    try {
+        config = readConfig(process.env);
+    }
+    catch (e) {
+        // A configuration failure is fatal and must say what to fix, without ever echoing what was given.
+        note(fatalLine(e));
+        process.exit(2);
+    }
+    let cached = null;
+    const deps = {
+        config,
+        pricing: () => {
+            // Re-asked after a failure, so a service that was down at startup is not down forever.
+            if (!cached) {
+                cached = discoverPricing(config, deps.fetchImpl)
+                    .then((p) => {
+                    // serveStdio builds a server per connection, so a price learned late reaches the next one.
+                    deps.pricingSnapshot = p;
+                    return p;
+                })
+                    .catch((e) => ((cached = null), Promise.reject(e)));
+            }
+            return cached;
+        },
+        fetchImpl: (input, init) => fetch(input, init),
+        budget: new Budget(config.maxSpendUsd),
+    };
+    // The price and the network go into the tool's own description, so a model can tell what a call
+    // costs — and whether it is a test network — before it decides to call. `GET /` is free and this
+    // is one round trip at startup; if it fails the description says where the price is published and
+    // the first tool call asks again.
+    let snapshot = null;
+    try {
+        snapshot = await deps.pricing();
+    }
+    catch (e) {
+        note(`@url2md-io/mcp: could not read the price from ${config.baseUrl} at startup (${e instanceof Url2mdToolError ? e.code : "error"}); it will be read again on the first call`);
+    }
+    deps.pricingSnapshot = snapshot;
+    note(`@url2md-io/mcp ${SERVER_VERSION} → ${config.baseUrl}` +
+        (snapshot
+            ? `; ${snapshot.model === "x402" ? `${snapshot.network}${isTestNetwork(snapshot.network) ? " (TEST network)" : " (real money)"}` : snapshot.model}`
+            : "; network unknown — the service did not answer, so nothing has been checked yet") +
+        `; paying from ${payerAddress(config)}, at most $${config.maxPriceUsd} per call and $${config.maxSpendUsd} in total`);
+    if (ceilingLooksWrong(config.maxPriceUsd)) {
+        note(`@url2md-io/mcp: WARNING — URL2MD_MAX_PRICE_USD is $${config.maxPriceUsd}. A call to this service costs a fraction of a cent; a ceiling above $${CEILING_WORTH_MENTIONING_USD} is usually a typo.`);
+    }
+    serveStdio(() => createServer(deps), { onerror: (err) => note(`@url2md-io/mcp: ${err.message}`) });
+}
+/**
+ * Whether this file is the program being run, rather than a module something imported (the tests
+ * import it).
+ *
+ * Both paths are resolved through the filesystem before they are compared, because when the package
+ * is installed the thing that starts is `node_modules/.bin/url2md` — a **symlink** to this file.
+ * `process.argv[1]` is then the symlink and `import.meta.filename` the file it points at, a plain
+ * `===` is false, and the server exits 0 having done nothing at all: no startup line, no tool, no
+ * error. The client sees only its connection close. It cannot happen when the file is run from a
+ * clone by its own path, which is why it survived to a cold install of the tarball to find.
+ */
+export function runningAsProgram(argv1 = process.argv[1], self = import.meta.filename) {
+    if (!argv1)
+        return false;
+    try {
+        return realpathSync(argv1) === realpathSync(self);
+    }
+    catch {
+        return false; // a path that cannot be resolved is not this file
+    }
+}
+if (runningAsProgram()) {
+    main().catch((e) => {
+        note(fatalLine(e));
+        process.exit(1);
+    });
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@url2md-io/mcp",
+  "version": "0.1.0",
+  "description": "MCP server for url2md — any URL in, clean Markdown out. Pays per call over x402 from your own wallet.",
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "markdown",
+    "x402",
+    "scraping",
+    "url-to-markdown"
+  ],
+  "homepage": "https://url2md.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/glassyeah/url2md.git",
+    "directory": "mcp"
+  },
+  "bugs": {
+    "url": "https://url2md.io"
+  },
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "node": ">=22.18"
+  },
+  "bin": {
+    "url2md": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "start": "node src/index.ts",
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/server": "^2.0.0",
+    "@x402/core": "^2.25.0",
+    "@x402/evm": "^2.25.0",
+    "viem": "^2.56.3",
+    "zod": "^4.1.13"
+  },
+  "devDependencies": {
+    "@modelcontextprotocol/client": "^2.0.0",
+    "typescript": "^5.9.3",
+    "@types/node": "^24.13.3",
+    "vitest": "^4.1.11"
+  }
+}