@j0hanz/fetch-url-mcp 1.1.3 → 1.2.0

package/README.md

@@ -52,6 +52,7 @@ URL → Validate → DNS Preflight → HTTP Fetch → Decompress
 ```text
 fetch-url-mcp/
 ├── assets/              # Server icon (logo.svg)
+├── examples/            # Client examples
 ├── scripts/             # Build & test orchestration
 ├── src/
 │   ├── workers/         # Worker-thread child for HTML transforms
@@ -95,6 +96,23 @@ Add to your MCP client configuration:
 }
 ```
 
+## Client Example (CLI)
+
+Build the server and examples, then run the client:
+
+```bash
+npm run build
+node dist/examples/mcp-fetch-url-client.js https://example.com
+```
+
+Optional flags:
+
+- `--full` reads the cached markdown resource to avoid inline truncation.
+- `--task` enables task-based execution with streamed status updates.
+- `--task-ttl <ms>` sets task TTL; `--task-poll <ms>` sets poll interval.
+- `--http http://localhost:3000/mcp` connects to the Streamable HTTP server.
+- Progress updates (when emitted) are printed to stderr.
+
 ## Installation
 
 ### NPX (Recommended)

package/dist/examples/mcp-fetch-url-client.js

@@ -0,0 +1,313 @@
+import { access, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { Client } from '@modelcontextprotocol/sdk/client';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+function printUsage() {
+    const usage = `
+Usage:
+  node dist/examples/mcp-fetch-url-client.js <url> [options]
+
+Options:
+  --http <url>         Connect via Streamable HTTP (e.g. http://localhost:3000/mcp)
+  --task               Use task-based execution with streamed status updates
+  --task-ttl <ms>      Task TTL in milliseconds (optional)
+  --task-poll <ms>     Task poll interval in milliseconds (optional)
+  --no-noise           Skip noise removal
+  --force              Force refresh (bypass cache)
+  --max-inline <n>     Max inline chars before truncation
+  --full               If truncated, read cached resource for full markdown
+  --out <path>         Write markdown to file instead of stdout
+  --json               Print full structured JSON instead of markdown
+  --cmd <executable>   Stdio: command to spawn (default: node)
+  --server <path>      Stdio: server entry (default: dist/index.js)
+  --cwd <path>         Stdio: working directory for server (default: repo root)
+  --env KEY=VALUE      Stdio: add/override environment variable (repeatable)
+  -h, --help           Show help
+`;
+    process.stderr.write(usage);
+}
+async function findRepoRoot(startDir) {
+    let current = startDir;
+    while (true) {
+        const candidate = path.join(current, 'package.json');
+        try {
+            await access(candidate);
+            return current;
+        }
+        catch {
+            const parent = path.dirname(current);
+            if (parent === current) {
+                return startDir;
+            }
+            current = parent;
+        }
+    }
+}
+function parseEnvOverrides(values) {
+    const env = {};
+    if (!values) {
+        return env;
+    }
+    for (const item of values) {
+        const index = item.indexOf('=');
+        if (index <= 0) {
+            throw new Error(`Invalid --env value: ${item}`);
+        }
+        const key = item.slice(0, index).trim();
+        const value = item.slice(index + 1);
+        if (!key) {
+            throw new Error(`Invalid --env key in: ${item}`);
+        }
+        env[key] = value;
+    }
+    return env;
+}
+function buildInheritedEnv() {
+    const env = {};
+    for (const [key, value] of Object.entries(process.env)) {
+        if (typeof value === 'string') {
+            env[key] = value;
+        }
+    }
+    return env;
+}
+function pickTextResource(contents) {
+    for (const entry of contents) {
+        const text = entry['text'];
+        if (typeof text === 'string') {
+            return text;
+        }
+    }
+    return null;
+}
+function formatProgress(progress) {
+    const { message, total } = progress;
+    if (typeof total === 'number' && total > 0) {
+        const percent = Math.round((progress.progress / total) * 100);
+        return `${percent}%${message ? ` ${message}` : ''}`;
+    }
+    return message ? `${progress.progress} ${message}` : `${progress.progress}`;
+}
+function getStructuredContent(result) {
+    if (typeof result !== 'object' || result === null) {
+        return null;
+    }
+    const candidate = result;
+    if (candidate.structuredContent &&
+        typeof candidate.structuredContent === 'object' &&
+        !Array.isArray(candidate.structuredContent)) {
+        return candidate.structuredContent;
+    }
+    return null;
+}
+function isToolError(result) {
+    if (typeof result !== 'object' || result === null) {
+        return false;
+    }
+    const candidate = result;
+    return candidate.isError === true;
+}
+function getStringField(structured, key) {
+    if (!structured) {
+        return null;
+    }
+    const value = structured[key];
+    return typeof value === 'string' ? value : null;
+}
+const options = {
+    help: { type: 'boolean', short: 'h' },
+    http: { type: 'string' },
+    task: { type: 'boolean' },
+    'task-ttl': { type: 'string' },
+    'task-poll': { type: 'string' },
+    'no-noise': { type: 'boolean' },
+    force: { type: 'boolean' },
+    'max-inline': { type: 'string' },
+    full: { type: 'boolean' },
+    out: { type: 'string' },
+    json: { type: 'boolean' },
+    cmd: { type: 'string' },
+    server: { type: 'string' },
+    cwd: { type: 'string' },
+    env: { type: 'string', multiple: true },
+};
+const { values, positionals } = parseArgs({
+    allowPositionals: true,
+    options,
+});
+if (values.help) {
+    printUsage();
+    process.exit(0);
+}
+function requireUrl(positionals) {
+    const url = positionals[0];
+    if (typeof url !== 'string' || url.length === 0) {
+        printUsage();
+        throw new Error('Missing URL.');
+    }
+    return url;
+}
+const targetUrl = requireUrl(positionals);
+const maxInlineRaw = values['max-inline'];
+const maxInlineChars = maxInlineRaw !== undefined ? Number(maxInlineRaw) : undefined;
+if (maxInlineRaw !== undefined && Number.isNaN(maxInlineChars)) {
+    throw new Error(`Invalid --max-inline value: ${maxInlineRaw}`);
+}
+const taskTtlRaw = values['task-ttl'];
+const taskTtl = taskTtlRaw !== undefined ? Number(taskTtlRaw) : undefined;
+if (taskTtlRaw !== undefined && Number.isNaN(taskTtl)) {
+    throw new Error(`Invalid --task-ttl value: ${taskTtlRaw}`);
+}
+const taskPollRaw = values['task-poll'];
+const taskPoll = taskPollRaw !== undefined ? Number(taskPollRaw) : undefined;
+if (taskPollRaw !== undefined && Number.isNaN(taskPoll)) {
+    throw new Error(`Invalid --task-poll value: ${taskPollRaw}`);
+}
+const onProgress = (progress) => {
+    process.stderr.write(`[progress] ${formatProgress(progress)}\n`);
+};
+async function run() {
+    let transport = null;
+    const client = new Client({ name: 'fetch-url-mcp-client', version: '0.1.0' }, { capabilities: {} });
+    try {
+        if (values.http) {
+            const endpoint = new URL(values.http);
+            transport = new StreamableHTTPClientTransport(endpoint);
+        }
+        else {
+            const command = values.cmd ?? process.execPath;
+            const repoRoot = await findRepoRoot(__dirname);
+            const serverPath = values.server ?? path.join(repoRoot, 'dist/index.js');
+            const cwd = values.cwd ?? repoRoot;
+            try {
+                await access(serverPath);
+            }
+            catch {
+                throw new Error(`Server entry not found at ${serverPath}. Run \"npm run build\" first or set --server.`);
+            }
+            const env = {
+                ...buildInheritedEnv(),
+                ...parseEnvOverrides(values.env),
+            };
+            transport = new StdioClientTransport({
+                command,
+                args: [serverPath, '--stdio'],
+                cwd,
+                env,
+                stderr: 'inherit',
+            });
+        }
+        await client.connect(transport);
+        const toolArguments = {
+            url: targetUrl,
+            skipNoiseRemoval: values['no-noise'] ?? false,
+            forceRefresh: values.force ?? false,
+        };
+        if (typeof maxInlineChars === 'number') {
+            toolArguments.maxInlineChars = maxInlineChars;
+        }
+        const taskOptions = {};
+        if (typeof taskTtl === 'number') {
+            taskOptions.ttl = taskTtl;
+        }
+        if (typeof taskPoll === 'number') {
+            taskOptions.pollInterval = taskPoll;
+        }
+        const requestOptions = values.task
+            ? { onprogress: onProgress, task: taskOptions }
+            : { onprogress: onProgress };
+        const result = values.task
+            ? await callToolStream(client, toolArguments, requestOptions)
+            : await client.callTool({
+                name: 'fetch-url',
+                arguments: toolArguments,
+            }, undefined, requestOptions);
+        if (isToolError(result)) {
+            const errorPayload = getStructuredContent(result) ?? {
+                message: 'Fetch failed',
+            };
+            process.stderr.write(`${JSON.stringify(errorPayload, null, 2)}\n`);
+            process.exitCode = 2;
+            return;
+        }
+        const structured = getStructuredContent(result);
+        if (values.json) {
+            const payload = structured ?? result;
+            process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
+            return;
+        }
+        let markdown = getStringField(structured, 'markdown');
+        if (values.full) {
+            const cacheResourceUri = getStringField(structured, 'cacheResourceUri');
+            if (cacheResourceUri) {
+                const resource = await client.readResource({
+                    uri: cacheResourceUri,
+                });
+                const text = pickTextResource(resource.contents);
+                if (text) {
+                    markdown = text;
+                }
+            }
+        }
+        if (!markdown) {
+            throw new Error('No markdown returned from fetch-url.');
+        }
+        if (values.out) {
+            await writeFile(values.out, markdown, 'utf8');
+        }
+        else {
+            process.stdout.write(`${markdown}\n`);
+        }
+    }
+    finally {
+        if (transport) {
+            await transport.close();
+        }
+    }
+}
+async function callToolStream(client, toolArguments, requestOptions) {
+    const stream = client.experimental.tasks.callToolStream({
+        name: 'fetch-url',
+        arguments: toolArguments,
+    }, undefined, requestOptions);
+    let finalResult = null;
+    for await (const message of stream) {
+        if (message.type === 'taskCreated') {
+            process.stderr.write(`[task] created ${message.task.taskId}\n`);
+            continue;
+        }
+        if (message.type === 'taskStatus') {
+            const statusMessage = message.task.statusMessage
+                ? ` ${message.task.statusMessage}`
+                : '';
+            process.stderr.write(`[task] ${message.task.status}${statusMessage}\n`);
+            continue;
+        }
+        if (message.type === 'result') {
+            finalResult = message.result;
+            continue;
+        }
+        if (message.type === 'error') {
+            throw new Error(message.error.message ?? 'Task failed');
+        }
+    }
+    if (finalResult === null) {
+        throw new Error('Task stream ended without a result.');
+    }
+    return finalResult;
+}
+try {
+    await run();
+}
+catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`${message}\n`);
+    process.exitCode = 1;
+}
+//# sourceMappingURL=mcp-fetch-url-client.js.map

package/dist/examples/mcp-fetch-url-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-fetch-url-client.js","sourceRoot":"","sources":["../../examples/mcp-fetch-url-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAUnG,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,SAAS,UAAU;IACjB,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;CAoBf,CAAC;IACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB;IAC1C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,MAA4B;IAE5B,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAwC;IAExC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,QAAkB;IACxC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAe;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAG,MAAoB,CAAC;IACvC,IACE,SAAS,CAAC,iBAAiB;QAC3B,OAAO,SAAS,CAAC,iBAAiB,KAAK,QAAQ;QAC/C,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAC3C,CAAC;QACD,OAAO,SAAS,CAAC,iBAAiB,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,MAAe;IAClC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,SAAS,GAAG,MAAoB,CAAC;IACvC,OAAO,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CACrB,UAAoC,EACpC,GAAW;IAEX,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,MAAM,OAAO,GAAG;IACd,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;IACrC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/B,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/B,CAAC;AAEX,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;IACxC,gBAAgB,EAAE,IAAI;IACtB,OAAO;CACR,CAAC,CAAC;AAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;IAChB,UAAU,EAAE,CAAC;IACb,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAAqB;IACvC,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,UAAU,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;AAE1C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;AAC1C,MAAM,cAAc,GAClB,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,IAAI,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;IAC/D,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,OAAO,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AACxC,MAAM,QAAQ,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,IAAI,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAQ,EAAE;IAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,IAAI,SAAS,GAAqB,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,OAAO,EAAE,EAClD,EAAE,YAAY,EAAE,EAAE,EAAE,CACrB,CAAC;IAEF,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtC,SAAS,GAAG,IAAI,6BAA6B,CAAC,QAAQ,CAAc,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;YACzE,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,QAAQ,CAAC;YAEnC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YAC3B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CACb,6BAA6B,UAAU,gDAAgD,CACxF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG;gBACV,GAAG,iBAAiB,EAAE;gBACtB,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC;aACjC,CAAC;YAEF,SAAS,GAAG,IAAI,oBAAoB,CAAC;gBACnC,OAAO;gBACP,IAAI,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;gBAC7B,GAAG;gBACH,GAAG;gBACH,MAAM,EAAE,SAAS;aAClB,CAAc,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEhC,MAAM,aAAa,GAKf;YACF,GAAG,EAAE,SAAS;YACd,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,KAAK;YAC7C,YAAY,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK;SACpC,CAAC;QAEF,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;YACvC,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAChD,CAAC;QAED,MAAM,WAAW,GAA4C,EAAE,CAAC;QAChE,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC;QAC5B,CAAC;QACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,WAAW,CAAC,YAAY,GAAG,QAAQ,CAAC;QACtC,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI;YAChC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE;YAC/C,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI;YACxB,CAAC,CAAC,MAAM,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,cAAc,CAAC;YAC7D,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CACnB;gBACE,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,aAAa;aACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;QAEN,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,YAAY,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI;gBACnD,OAAO,EAAE,cAAc;aACxB,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAEhD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;YACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9D,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACxE,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;oBACzC,GAAG,EAAE,gBAAgB;iBACtB,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACjD,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAc,EACd,aAKC,EACD,cAMC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CACrD;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,aAAa;KACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;IAEF,IAAI,WAAW,GAAY,IAAI,CAAC;IAEhC,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa;gBAC9C,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE;gBAClC,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,IAAI,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAC7B,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,aAAa,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,IAAI,CAAC;IACH,MAAM,GAAG,EAAE,CAAC;AACd,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC"}

package/dist/fetch.js

@@ -441,6 +441,9 @@ class SafeDnsResolver {
         if (signal?.aborted) {
             throw createAbortSignalError();
         }
+        if (this.isBlockedHostname(normalizedHostname)) {
+            throw createErrorWithCode(`Blocked host: ${normalizedHostname}. Internal hosts are not allowed`, 'EBLOCKED');
+        }
         if (isIP(normalizedHostname)) {
             if (this.ipBlocker.isBlockedIp(normalizedHostname)) {
                 throw createErrorWithCode(`Blocked IP range: ${normalizedHostname}. Private IPs are not allowed`, 'EBLOCKED');
@@ -1121,7 +1124,7 @@ function createDecompressor(encoding) {
 function createPumpedStream(initialChunk, reader) {
     return new ReadableStream({
         start(controller) {
-            if (initialChunk.byteLength > 0) {
+            if (initialChunk && initialChunk.byteLength > 0) {
                 controller.enqueue(initialChunk);
             }
         },
@@ -1144,31 +1147,6 @@ function createPumpedStream(initialChunk, reader) {
         },
     });
 }
-function isLikelyCompressed(chunk, encoding) {
-    if (chunk.byteLength === 0)
-        return false;
-    if (encoding === 'gzip') {
-        return chunk.byteLength >= 2 && chunk[0] === 0x1f && chunk[1] === 0x8b;
-    }
-    if (encoding === 'deflate') {
-        if (chunk.byteLength < 2)
-            return false;
-        const byte0 = chunk[0] ?? 0;
-        const byte1 = chunk[1] ?? 0;
-        const cm = byte0 & 0x0f;
-        if (cm !== 8)
-            return false;
-        return (byte0 * 256 + byte1) % 31 === 0;
-    }
-    let nonPrintable = 0;
-    const limit = Math.min(chunk.length, 50);
-    for (let i = 0; i < limit; i += 1) {
-        const b = chunk[i] ?? 0;
-        if (b < 0x09 || (b > 0x0d && b < 0x20) || b === 0x7f)
-            nonPrintable += 1;
-    }
-    return nonPrintable / limit > 0.1;
-}
 async function decodeResponseIfNeeded(response, url, signal) {
     const encodingHeader = response.headers.get('content-encoding');
     const parsedEncodings = parseContentEncodings(encodingHeader);
@@ -1184,76 +1162,81 @@ async function decodeResponseIfNeeded(response, url, signal) {
     }
     if (!response.body)
         return response;
-    // Peek at first chunk to check if actually compressed
-    const reader = response.body.getReader();
-    let initialChunk;
-    try {
-        const { done, value } = await reader.read();
-        if (done) {
-            return new Response(null, {
-                status: response.status,
-                statusText: response.statusText,
-                headers: response.headers,
-            });
-        }
-        initialChunk = value;
-    }
-    catch (error) {
-        // If read fails, throw properly
-        throw new FetchError(`Failed to read response body: ${isError(error) ? error.message : String(error)}`, url, 502);
-    }
+    const [decodeBranch, passthroughBranch] = response.body.tee();
     const decodeOrder = encodings
         .slice()
         .reverse()
         .filter(isSupportedContentEncoding);
-    const firstDecodeEncoding = decodeOrder[0];
-    if (!firstDecodeEncoding ||
-        !isLikelyCompressed(initialChunk, firstDecodeEncoding)) {
-        const body = createPumpedStream(initialChunk, reader);
-        const headers = new Headers(response.headers);
-        headers.delete('content-encoding');
-        headers.delete('content-length');
-        return new Response(body, {
-            status: response.status,
-            statusText: response.statusText,
-            headers,
-        });
-    }
     const decompressors = decodeOrder.map((encoding) => createDecompressor(encoding));
-    const sourceStream = Readable.fromWeb(toNodeReadableStream(createPumpedStream(initialChunk, reader), url, 'response:decode-content-encoding'));
+    const decodeSource = Readable.fromWeb(toNodeReadableStream(decodeBranch, url, 'response:decode-content-encoding'));
     const decodedNodeStream = new PassThrough();
-    const pipelinePromise = pipeline([
-        sourceStream,
+    const decodedPipeline = pipeline([
+        decodeSource,
         ...decompressors,
         decodedNodeStream,
     ]);
-    const abortHandler = () => {
-        sourceStream.destroy();
+    const headers = new Headers(response.headers);
+    headers.delete('content-encoding');
+    headers.delete('content-length');
+    const abortDecodePipeline = () => {
+        decodeSource.destroy();
         for (const decompressor of decompressors) {
             decompressor.destroy();
         }
         decodedNodeStream.destroy();
     };
     if (signal) {
-        signal.addEventListener('abort', abortHandler, { once: true });
+        signal.addEventListener('abort', abortDecodePipeline, { once: true });
     }
-    void pipelinePromise.catch((error) => {
+    void decodedPipeline.catch((error) => {
         decodedNodeStream.destroy(error instanceof Error ? error : new Error(String(error)));
     });
-    const decodedBody = toWebReadableStream(decodedNodeStream, url, 'response:decode-content-encoding');
-    const headers = new Headers(response.headers);
-    headers.delete('content-encoding');
-    headers.delete('content-length');
-    if (signal) {
-        void finished(decodedNodeStream, { cleanup: true }).finally(() => {
-            signal.removeEventListener('abort', abortHandler);
+    const decodedBodyStream = toWebReadableStream(decodedNodeStream, url, 'response:decode-content-encoding');
+    const decodedReader = decodedBodyStream.getReader();
+    const clearAbortListener = () => {
+        if (!signal)
+            return;
+        signal.removeEventListener('abort', abortDecodePipeline);
+    };
+    try {
+        const first = await decodedReader.read();
+        if (first.done) {
+            clearAbortListener();
+            void passthroughBranch.cancel().catch(() => undefined);
+            return new Response(null, {
+                status: response.status,
+                statusText: response.statusText,
+                headers,
+            });
+        }
+        void passthroughBranch.cancel().catch(() => undefined);
+        const body = createPumpedStream(first.value, decodedReader);
+        if (signal) {
+            void finished(decodedNodeStream, { cleanup: true }).finally(() => {
+                clearAbortListener();
+            });
+        }
+        return new Response(body, {
+            status: response.status,
+            statusText: response.statusText,
+            headers,
+        });
+    }
+    catch (error) {
+        clearAbortListener();
+        abortDecodePipeline();
+        void decodedReader.cancel(error).catch(() => undefined);
+        logDebug('Content-Encoding decode failed; using passthrough body', {
+            url: redactUrl(url),
+            encoding: encodingHeader ?? encodings.join(','),
+            error: isError(error) ? error.message : String(error),
+        });
+        return new Response(passthroughBranch, {
+            status: response.status,
+            statusText: response.statusText,
+            headers,
         });
     }
-    return new Response(decodedBody, {
-        status: response.status,
-        statusText: response.statusText,
-        headers,
-    });
 }
 async function readAndRecordDecodedResponse(response, finalUrl, ctx, telemetry, reader, maxBytes, mode, signal) {
     const responseError = resolveResponseError(response, finalUrl);

package/dist/tools.js

@@ -446,8 +446,21 @@ function buildToolContentBlocks(structuredContent, resourceLink, embeddedResourc
 }
 function resolveNormalizedUrl(url) {
     const { normalizedUrl: validatedUrl } = normalizeUrl(url);
-    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
-    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+    const transformedResult = transformToRawUrl(validatedUrl);
+    if (!transformedResult.transformed) {
+        return {
+            normalizedUrl: validatedUrl,
+            originalUrl: validatedUrl,
+            transformed: false,
+        };
+    }
+    // Re-validate transformed URLs so blocked-host and length policies still apply.
+    const { normalizedUrl: transformedUrl } = normalizeUrl(transformedResult.url);
+    return {
+        normalizedUrl: transformedUrl,
+        originalUrl: validatedUrl,
+        transformed: true,
+    };
 }
 function logRawUrlTransformation(resolvedUrl) {
     if (!resolvedUrl.transformed)
@@ -493,10 +506,12 @@ function attemptCacheRetrieval(params) {
         return null;
     }
     logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
+    const finalUrl = cached.url !== normalizedUrl ? cached.url : undefined;
     return {
         data,
         fromCache: true,
         url: normalizedUrl,
+        ...(finalUrl ? { finalUrl } : {}),
         fetchedAt: cached.fetchedAt,
         cacheKey,
     };
@@ -675,12 +690,16 @@ export function parseCachedMarkdownResult(cached) {
     if (typeof markdown !== 'string')
         return undefined;
     const metadata = normalizeExtractedMetadata(result.data.metadata);
+    const truncated = result.data.truncated ?? false;
+    const persistedMarkdown = truncated
+        ? appendTruncationMarker(markdown, TRUNCATION_MARKER)
+        : markdown;
     return {
-        content: markdown,
-        markdown,
+        content: persistedMarkdown,
+        markdown: persistedMarkdown,
         title: result.data.title,
         ...(metadata ? { metadata } : {}),
-        truncated: result.data.truncated ?? false,
+        truncated,
     };
 }
 const markdownTransform = async (input, url, signal, skipNoiseRemoval) => {
@@ -695,8 +714,11 @@ const markdownTransform = async (input, url, signal, skipNoiseRemoval) => {
     return { ...result, content: result.markdown, truncated };
 };
 function serializeMarkdownResult(result) {
+    const persistedMarkdown = result.truncated
+        ? appendTruncationMarker(result.markdown, TRUNCATION_MARKER)
+        : result.markdown;
     return JSON.stringify({
-        markdown: result.markdown,
+        markdown: persistedMarkdown,
         title: result.title,
         metadata: result.metadata,
         truncated: result.truncated,

package/dist/transform.js

@@ -2369,19 +2369,15 @@ function resolveWorkerFallback(error, htmlOrBuffer, url, options) {
         });
         return transformInputInProcess(htmlOrBuffer, url, options);
     }
+    abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
     if (error instanceof FetchError)
         throw error;
-    abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
     const message = getErrorMessage(error);
-    logWarn('Transform worker failed; refusing in-process fallback', {
+    logWarn('Transform worker failed; falling back to in-process', {
         url: redactUrl(url),
         error: message,
     });
-    throw new FetchError('Transform worker failed', url, 503, {
-        reason: 'worker_failed',
-        stage: 'transform:worker',
-        error: message,
-    });
+    return transformInputInProcess(htmlOrBuffer, url, options);
 }
 async function runWorkerTransformWithFallback(htmlOrBuffer, url, options) {
     const workerStage = stageTracker.start(url, 'transform:worker');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",