@j0hanz/fetch-url-mcp 1.1.2 → 1.2.0

package/README.md

@@ -52,6 +52,7 @@ URL → Validate → DNS Preflight → HTTP Fetch → Decompress
 ```text
 fetch-url-mcp/
 ├── assets/              # Server icon (logo.svg)
+├── examples/            # Client examples
 ├── scripts/             # Build & test orchestration
 ├── src/
 │   ├── workers/         # Worker-thread child for HTML transforms
@@ -95,6 +96,23 @@ Add to your MCP client configuration:
 }
 ```
 
+## Client Example (CLI)
+
+Build the server and examples, then run the client:
+
+```bash
+npm run build
+node dist/examples/mcp-fetch-url-client.js https://example.com
+```
+
+Optional flags:
+
+- `--full` reads the cached markdown resource to avoid inline truncation.
+- `--task` enables task-based execution with streamed status updates.
+- `--task-ttl <ms>` sets task TTL; `--task-poll <ms>` sets poll interval.
+- `--http http://localhost:3000/mcp` connects to the Streamable HTTP server.
+- Progress updates (when emitted) are printed to stderr.
+
 ## Installation
 
 ### NPX (Recommended)

package/dist/AGENTS.md

@@ -34,7 +34,7 @@
 - `tests/` — Unit/integration tests (46+ test files) using Node.js built-in test runner
 - `scripts/` — Build & test orchestration (`tasks.mjs`)
 - `assets/` — Server icon (`logo.svg`)
-- `.github/workflows/` — CI/CD (`release.yml`: lint → type-check → test → build → publish to npm, MCP Registry, Docker)
+- `.github/workflows/` — CI/CD (`release.yml`: lint → type-check → type-check:tests → test → build → publish to npm, MCP Registry, Docker)
 
 > Ignore: `dist/`, `node_modules/`, `coverage/`, `.cache/`, `.tsbuildinfo`
 
@@ -49,6 +49,7 @@ All commands verified from `.github/workflows/release.yml` (CI) and `package.jso
 - **Start:** `npm run start` → `node dist/index.js` (see `package.json`)
 - **Build:** `npm run build` → `node scripts/tasks.mjs build` — cleans `dist/`, compiles TS, validates `instructions.md`, copies assets, sets executable bit (see `scripts/tasks.mjs`, `package.json`)
 - **Type-check:** `npm run type-check` → `tsc -p tsconfig.json --noEmit` (see `scripts/tasks.mjs`, `.github/workflows/release.yml`)
+- **Type-check (tests):** `npm run type-check:tests` → build output + `tsc -p tsconfig.tests.json --noEmit` (see `scripts/tasks.mjs`, `.github/workflows/release.yml`)
 - **Lint:** `npm run lint` → `eslint .` (see `package.json`, `.github/workflows/release.yml`)
 - **Lint (fix):** `npm run lint:fix` → `eslint . --fix` (see `package.json`)
 - **Format:** `npm run format` → `prettier --write .` (see `package.json`)
@@ -135,7 +136,7 @@ All commands verified from `.github/workflows/release.yml` (CI) and `package.jso
   - Config values temporarily overridden per test with `try/finally` cleanup (observed in `tests/fetch-url-tool.test.ts`)
   - Worker pool shutdown in `after()` hooks for clean teardown (observed in `tests/fetch-url-tool.test.ts`)
   - No external services (DB/containers) required for tests
-- **CI validation order:** `lint` → `type-check` → `test` → `build` (see `.github/workflows/release.yml`)
+- **CI validation order:** `lint` → `type-check` → `type-check:tests` → `test` → `build` (see `.github/workflows/release.yml`)
 
 ## 7) Common Pitfalls (Verified Only)
 

package/dist/examples/mcp-fetch-url-client.js

@@ -0,0 +1,313 @@
+import { access, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { Client } from '@modelcontextprotocol/sdk/client';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+function printUsage() {
+    const usage = `
+Usage:
+  node dist/examples/mcp-fetch-url-client.js <url> [options]
+
+Options:
+  --http <url>         Connect via Streamable HTTP (e.g. http://localhost:3000/mcp)
+  --task               Use task-based execution with streamed status updates
+  --task-ttl <ms>      Task TTL in milliseconds (optional)
+  --task-poll <ms>     Task poll interval in milliseconds (optional)
+  --no-noise           Skip noise removal
+  --force              Force refresh (bypass cache)
+  --max-inline <n>     Max inline chars before truncation
+  --full               If truncated, read cached resource for full markdown
+  --out <path>         Write markdown to file instead of stdout
+  --json               Print full structured JSON instead of markdown
+  --cmd <executable>   Stdio: command to spawn (default: node)
+  --server <path>      Stdio: server entry (default: dist/index.js)
+  --cwd <path>         Stdio: working directory for server (default: repo root)
+  --env KEY=VALUE      Stdio: add/override environment variable (repeatable)
+  -h, --help           Show help
+`;
+    process.stderr.write(usage);
+}
+async function findRepoRoot(startDir) {
+    let current = startDir;
+    while (true) {
+        const candidate = path.join(current, 'package.json');
+        try {
+            await access(candidate);
+            return current;
+        }
+        catch {
+            const parent = path.dirname(current);
+            if (parent === current) {
+                return startDir;
+            }
+            current = parent;
+        }
+    }
+}
+function parseEnvOverrides(values) {
+    const env = {};
+    if (!values) {
+        return env;
+    }
+    for (const item of values) {
+        const index = item.indexOf('=');
+        if (index <= 0) {
+            throw new Error(`Invalid --env value: ${item}`);
+        }
+        const key = item.slice(0, index).trim();
+        const value = item.slice(index + 1);
+        if (!key) {
+            throw new Error(`Invalid --env key in: ${item}`);
+        }
+        env[key] = value;
+    }
+    return env;
+}
+function buildInheritedEnv() {
+    const env = {};
+    for (const [key, value] of Object.entries(process.env)) {
+        if (typeof value === 'string') {
+            env[key] = value;
+        }
+    }
+    return env;
+}
+function pickTextResource(contents) {
+    for (const entry of contents) {
+        const text = entry['text'];
+        if (typeof text === 'string') {
+            return text;
+        }
+    }
+    return null;
+}
+function formatProgress(progress) {
+    const { message, total } = progress;
+    if (typeof total === 'number' && total > 0) {
+        const percent = Math.round((progress.progress / total) * 100);
+        return `${percent}%${message ? ` ${message}` : ''}`;
+    }
+    return message ? `${progress.progress} ${message}` : `${progress.progress}`;
+}
+function getStructuredContent(result) {
+    if (typeof result !== 'object' || result === null) {
+        return null;
+    }
+    const candidate = result;
+    if (candidate.structuredContent &&
+        typeof candidate.structuredContent === 'object' &&
+        !Array.isArray(candidate.structuredContent)) {
+        return candidate.structuredContent;
+    }
+    return null;
+}
+function isToolError(result) {
+    if (typeof result !== 'object' || result === null) {
+        return false;
+    }
+    const candidate = result;
+    return candidate.isError === true;
+}
+function getStringField(structured, key) {
+    if (!structured) {
+        return null;
+    }
+    const value = structured[key];
+    return typeof value === 'string' ? value : null;
+}
+const options = {
+    help: { type: 'boolean', short: 'h' },
+    http: { type: 'string' },
+    task: { type: 'boolean' },
+    'task-ttl': { type: 'string' },
+    'task-poll': { type: 'string' },
+    'no-noise': { type: 'boolean' },
+    force: { type: 'boolean' },
+    'max-inline': { type: 'string' },
+    full: { type: 'boolean' },
+    out: { type: 'string' },
+    json: { type: 'boolean' },
+    cmd: { type: 'string' },
+    server: { type: 'string' },
+    cwd: { type: 'string' },
+    env: { type: 'string', multiple: true },
+};
+const { values, positionals } = parseArgs({
+    allowPositionals: true,
+    options,
+});
+if (values.help) {
+    printUsage();
+    process.exit(0);
+}
+function requireUrl(positionals) {
+    const url = positionals[0];
+    if (typeof url !== 'string' || url.length === 0) {
+        printUsage();
+        throw new Error('Missing URL.');
+    }
+    return url;
+}
+const targetUrl = requireUrl(positionals);
+const maxInlineRaw = values['max-inline'];
+const maxInlineChars = maxInlineRaw !== undefined ? Number(maxInlineRaw) : undefined;
+if (maxInlineRaw !== undefined && Number.isNaN(maxInlineChars)) {
+    throw new Error(`Invalid --max-inline value: ${maxInlineRaw}`);
+}
+const taskTtlRaw = values['task-ttl'];
+const taskTtl = taskTtlRaw !== undefined ? Number(taskTtlRaw) : undefined;
+if (taskTtlRaw !== undefined && Number.isNaN(taskTtl)) {
+    throw new Error(`Invalid --task-ttl value: ${taskTtlRaw}`);
+}
+const taskPollRaw = values['task-poll'];
+const taskPoll = taskPollRaw !== undefined ? Number(taskPollRaw) : undefined;
+if (taskPollRaw !== undefined && Number.isNaN(taskPoll)) {
+    throw new Error(`Invalid --task-poll value: ${taskPollRaw}`);
+}
+const onProgress = (progress) => {
+    process.stderr.write(`[progress] ${formatProgress(progress)}\n`);
+};
+async function run() {
+    let transport = null;
+    const client = new Client({ name: 'fetch-url-mcp-client', version: '0.1.0' }, { capabilities: {} });
+    try {
+        if (values.http) {
+            const endpoint = new URL(values.http);
+            transport = new StreamableHTTPClientTransport(endpoint);
+        }
+        else {
+            const command = values.cmd ?? process.execPath;
+            const repoRoot = await findRepoRoot(__dirname);
+            const serverPath = values.server ?? path.join(repoRoot, 'dist/index.js');
+            const cwd = values.cwd ?? repoRoot;
+            try {
+                await access(serverPath);
+            }
+            catch {
+                throw new Error(`Server entry not found at ${serverPath}. Run \"npm run build\" first or set --server.`);
+            }
+            const env = {
+                ...buildInheritedEnv(),
+                ...parseEnvOverrides(values.env),
+            };
+            transport = new StdioClientTransport({
+                command,
+                args: [serverPath, '--stdio'],
+                cwd,
+                env,
+                stderr: 'inherit',
+            });
+        }
+        await client.connect(transport);
+        const toolArguments = {
+            url: targetUrl,
+            skipNoiseRemoval: values['no-noise'] ?? false,
+            forceRefresh: values.force ?? false,
+        };
+        if (typeof maxInlineChars === 'number') {
+            toolArguments.maxInlineChars = maxInlineChars;
+        }
+        const taskOptions = {};
+        if (typeof taskTtl === 'number') {
+            taskOptions.ttl = taskTtl;
+        }
+        if (typeof taskPoll === 'number') {
+            taskOptions.pollInterval = taskPoll;
+        }
+        const requestOptions = values.task
+            ? { onprogress: onProgress, task: taskOptions }
+            : { onprogress: onProgress };
+        const result = values.task
+            ? await callToolStream(client, toolArguments, requestOptions)
+            : await client.callTool({
+                name: 'fetch-url',
+                arguments: toolArguments,
+            }, undefined, requestOptions);
+        if (isToolError(result)) {
+            const errorPayload = getStructuredContent(result) ?? {
+                message: 'Fetch failed',
+            };
+            process.stderr.write(`${JSON.stringify(errorPayload, null, 2)}\n`);
+            process.exitCode = 2;
+            return;
+        }
+        const structured = getStructuredContent(result);
+        if (values.json) {
+            const payload = structured ?? result;
+            process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
+            return;
+        }
+        let markdown = getStringField(structured, 'markdown');
+        if (values.full) {
+            const cacheResourceUri = getStringField(structured, 'cacheResourceUri');
+            if (cacheResourceUri) {
+                const resource = await client.readResource({
+                    uri: cacheResourceUri,
+                });
+                const text = pickTextResource(resource.contents);
+                if (text) {
+                    markdown = text;
+                }
+            }
+        }
+        if (!markdown) {
+            throw new Error('No markdown returned from fetch-url.');
+        }
+        if (values.out) {
+            await writeFile(values.out, markdown, 'utf8');
+        }
+        else {
+            process.stdout.write(`${markdown}\n`);
+        }
+    }
+    finally {
+        if (transport) {
+            await transport.close();
+        }
+    }
+}
+async function callToolStream(client, toolArguments, requestOptions) {
+    const stream = client.experimental.tasks.callToolStream({
+        name: 'fetch-url',
+        arguments: toolArguments,
+    }, undefined, requestOptions);
+    let finalResult = null;
+    for await (const message of stream) {
+        if (message.type === 'taskCreated') {
+            process.stderr.write(`[task] created ${message.task.taskId}\n`);
+            continue;
+        }
+        if (message.type === 'taskStatus') {
+            const statusMessage = message.task.statusMessage
+                ? ` ${message.task.statusMessage}`
+                : '';
+            process.stderr.write(`[task] ${message.task.status}${statusMessage}\n`);
+            continue;
+        }
+        if (message.type === 'result') {
+            finalResult = message.result;
+            continue;
+        }
+        if (message.type === 'error') {
+            throw new Error(message.error.message ?? 'Task failed');
+        }
+    }
+    if (finalResult === null) {
+        throw new Error('Task stream ended without a result.');
+    }
+    return finalResult;
+}
+try {
+    await run();
+}
+catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`${message}\n`);
+    process.exitCode = 1;
+}
+//# sourceMappingURL=mcp-fetch-url-client.js.map

package/dist/examples/mcp-fetch-url-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-fetch-url-client.js","sourceRoot":"","sources":["../../examples/mcp-fetch-url-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAUnG,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,SAAS,UAAU;IACjB,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;CAoBf,CAAC;IACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB;IAC1C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,MAA4B;IAE5B,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAwC;IAExC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,QAAkB;IACxC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAe;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAG,MAAoB,CAAC;IACvC,IACE,SAAS,CAAC,iBAAiB;QAC3B,OAAO,SAAS,CAAC,iBAAiB,KAAK,QAAQ;QAC/C,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAC3C,CAAC;QACD,OAAO,SAAS,CAAC,iBAAiB,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,MAAe;IAClC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,SAAS,GAAG,MAAoB,CAAC;IACvC,OAAO,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CACrB,UAAoC,EACpC,GAAW;IAEX,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,MAAM,OAAO,GAAG;IACd,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;IACrC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/B,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/B,CAAC;AAEX,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;IACxC,gBAAgB,EAAE,IAAI;IACtB,OAAO;CACR,CAAC,CAAC;AAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;IAChB,UAAU,EAAE,CAAC;IACb,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAAqB;IACvC,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,UAAU,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;AAE1C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;AAC1C,MAAM,cAAc,GAClB,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,IAAI,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;IAC/D,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,OAAO,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AACxC,MAAM,QAAQ,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,IAAI,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAQ,EAAE;IAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,IAAI,SAAS,GAAqB,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,OAAO,EAAE,EAClD,EAAE,YAAY,EAAE,EAAE,EAAE,CACrB,CAAC;IAEF,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtC,SAAS,GAAG,IAAI,6BAA6B,CAAC,QAAQ,CAAc,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;YACzE,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,QAAQ,CAAC;YAEnC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YAC3B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CACb,6BAA6B,UAAU,gDAAgD,CACxF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG;gBACV,GAAG,iBAAiB,EAAE;gBACtB,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC;aACjC,CAAC;YAEF,SAAS,GAAG,IAAI,oBAAoB,CAAC;gBACnC,OAAO;gBACP,IAAI,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;gBAC7B,GAAG;gBACH,GAAG;gBACH,MAAM,EAAE,SAAS;aAClB,CAAc,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEhC,MAAM,aAAa,GAKf;YACF,GAAG,EAAE,SAAS;YACd,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,KAAK;YAC7C,YAAY,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK;SACpC,CAAC;QAEF,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;YACvC,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAChD,CAAC;QAED,MAAM,WAAW,GAA4C,EAAE,CAAC;QAChE,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC;QAC5B,CAAC;QACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,WAAW,CAAC,YAAY,GAAG,QAAQ,CAAC;QACtC,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI;YAChC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE;YAC/C,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI;YACxB,CAAC,CAAC,MAAM,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,cAAc,CAAC;YAC7D,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CACnB;gBACE,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,aAAa;aACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;QAEN,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,YAAY,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI;gBACnD,OAAO,EAAE,cAAc;aACxB,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAEhD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;YACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9D,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACxE,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;oBACzC,GAAG,EAAE,gBAAgB;iBACtB,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACjD,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAc,EACd,aAKC,EACD,cAMC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CACrD;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,aAAa;KACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;IAEF,IAAI,WAAW,GAAY,IAAI,CAAC;IAEhC,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa;gBAC9C,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE;gBAClC,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,IAAI,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAC7B,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,aAAa,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,IAAI,CAAC;IACH,MAAM,GAAG,EAAE,CAAC;AACd,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC"}

package/dist/fetch-content.d.ts

@@ -0,0 +1,4 @@
+export declare function getCharsetFromContentType(contentType: string | null): string | undefined;
+export declare function decodeBuffer(buffer: Uint8Array, encoding: string): string;
+export declare function resolveEncoding(declaredEncoding: string | undefined, sample: Uint8Array): string | undefined;
+export declare function isBinaryContent(buffer: Uint8Array, encoding?: string): boolean;

package/dist/fetch-content.js

@@ -0,0 +1,163 @@
+import { Buffer } from 'node:buffer';
+export function getCharsetFromContentType(contentType) {
+    if (!contentType)
+        return undefined;
+    const match = /charset=([^;]+)/i.exec(contentType);
+    const charsetGroup = match?.[1];
+    if (!charsetGroup)
+        return undefined;
+    let charset = charsetGroup.trim();
+    if (charset.startsWith('"') && charset.endsWith('"')) {
+        charset = charset.slice(1, -1);
+    }
+    return charset.trim();
+}
+function createDecoder(encoding) {
+    if (!encoding)
+        return new TextDecoder('utf-8');
+    try {
+        return new TextDecoder(encoding);
+    }
+    catch {
+        return new TextDecoder('utf-8');
+    }
+}
+export function decodeBuffer(buffer, encoding) {
+    return createDecoder(encoding).decode(buffer);
+}
+function normalizeEncodingLabel(encoding) {
+    return encoding?.trim().toLowerCase() ?? '';
+}
+function isUnicodeWideEncoding(encoding) {
+    const normalized = normalizeEncodingLabel(encoding);
+    return (normalized.startsWith('utf-16') ||
+        normalized.startsWith('utf-32') ||
+        normalized === 'ucs-2' ||
+        normalized === 'unicodefffe' ||
+        normalized === 'unicodefeff');
+}
+const BOM_SIGNATURES = [
+    // 4-byte BOMs must come first to avoid false matches with 2-byte prefixes
+    { bytes: [0xff, 0xfe, 0x00, 0x00], encoding: 'utf-32le' },
+    { bytes: [0x00, 0x00, 0xfe, 0xff], encoding: 'utf-32be' },
+    { bytes: [0xef, 0xbb, 0xbf], encoding: 'utf-8' },
+    { bytes: [0xff, 0xfe], encoding: 'utf-16le' },
+    { bytes: [0xfe, 0xff], encoding: 'utf-16be' },
+];
+function startsWithBytes(buffer, signature) {
+    const sigLen = signature.length;
+    if (buffer.length < sigLen)
+        return false;
+    for (let i = 0; i < sigLen; i += 1) {
+        if (buffer[i] !== signature[i])
+            return false;
+    }
+    return true;
+}
+function detectBomEncoding(buffer) {
+    for (const { bytes, encoding } of BOM_SIGNATURES) {
+        if (startsWithBytes(buffer, bytes))
+            return encoding;
+    }
+    return undefined;
+}
+function readQuotedValue(input, startIndex) {
+    const first = input[startIndex];
+    if (!first)
+        return '';
+    const quoted = first === '"' || first === "'";
+    if (quoted) {
+        const end = input.indexOf(first, startIndex + 1);
+        return end === -1 ? '' : input.slice(startIndex + 1, end).trim();
+    }
+    const tail = input.slice(startIndex);
+    const stop = tail.search(/[\s/>]/);
+    return (stop === -1 ? tail : tail.slice(0, stop)).trim();
+}
+function extractHtmlCharset(headSnippet) {
+    const lower = headSnippet.toLowerCase();
+    const charsetToken = 'charset=';
+    const charsetIdx = lower.indexOf(charsetToken);
+    if (charsetIdx === -1)
+        return undefined;
+    const valueStart = charsetIdx + charsetToken.length;
+    const charset = readQuotedValue(headSnippet, valueStart);
+    return charset ? charset.toLowerCase() : undefined;
+}
+function extractXmlEncoding(headSnippet) {
+    const lower = headSnippet.toLowerCase();
+    const xmlStart = lower.indexOf('<?xml');
+    if (xmlStart === -1)
+        return undefined;
+    const xmlEnd = lower.indexOf('?>', xmlStart);
+    const declaration = xmlEnd === -1
+        ? headSnippet.slice(xmlStart)
+        : headSnippet.slice(xmlStart, xmlEnd + 2);
+    const declarationLower = declaration.toLowerCase();
+    const encodingToken = 'encoding=';
+    const encodingIdx = declarationLower.indexOf(encodingToken);
+    if (encodingIdx === -1)
+        return undefined;
+    const valueStart = encodingIdx + encodingToken.length;
+    const encoding = readQuotedValue(declaration, valueStart);
+    return encoding ? encoding.toLowerCase() : undefined;
+}
+function detectHtmlDeclaredEncoding(buffer) {
+    const scanSize = Math.min(buffer.length, 8_192);
+    if (scanSize === 0)
+        return undefined;
+    const headSnippet = Buffer.from(buffer.buffer, buffer.byteOffset, scanSize).toString('latin1');
+    return extractHtmlCharset(headSnippet) ?? extractXmlEncoding(headSnippet);
+}
+export function resolveEncoding(declaredEncoding, sample) {
+    const bomEncoding = detectBomEncoding(sample);
+    if (bomEncoding)
+        return bomEncoding;
+    if (declaredEncoding)
+        return declaredEncoding;
+    return detectHtmlDeclaredEncoding(sample);
+}
+const BINARY_SIGNATURES = [
+    [0x25, 0x50, 0x44, 0x46],
+    [0x89, 0x50, 0x4e, 0x47],
+    [0x47, 0x49, 0x46, 0x38],
+    [0xff, 0xd8, 0xff],
+    [0x52, 0x49, 0x46, 0x46],
+    [0x42, 0x4d],
+    [0x49, 0x49, 0x2a, 0x00],
+    [0x4d, 0x4d, 0x00, 0x2a],
+    [0x00, 0x00, 0x01, 0x00],
+    [0x50, 0x4b, 0x03, 0x04],
+    [0x1f, 0x8b],
+    [0x42, 0x5a, 0x68],
+    [0x52, 0x61, 0x72, 0x21],
+    [0x37, 0x7a, 0xbc, 0xaf],
+    [0x7f, 0x45, 0x4c, 0x46],
+    [0x4d, 0x5a],
+    [0xcf, 0xfa, 0xed, 0xfe],
+    [0x00, 0x61, 0x73, 0x6d],
+    [0x1a, 0x45, 0xdf, 0xa3],
+    [0x66, 0x74, 0x79, 0x70],
+    [0x46, 0x4c, 0x56],
+    [0x49, 0x44, 0x33],
+    [0xff, 0xfb],
+    [0xff, 0xfa],
+    [0x4f, 0x67, 0x67, 0x53],
+    [0x66, 0x4c, 0x61, 0x43],
+    [0x4d, 0x54, 0x68, 0x64],
+    [0x77, 0x4f, 0x46, 0x46],
+    [0x00, 0x01, 0x00, 0x00],
+    [0x4f, 0x54, 0x54, 0x4f],
+    [0x53, 0x51, 0x4c, 0x69],
+];
+function hasNullByte(buffer, limit) {
+    const checkLen = Math.min(buffer.length, limit);
+    return buffer.subarray(0, checkLen).includes(0x00);
+}
+export function isBinaryContent(buffer, encoding) {
+    for (const signature of BINARY_SIGNATURES) {
+        if (startsWithBytes(buffer, signature))
+            return true;
+    }
+    return !isUnicodeWideEncoding(encoding) && hasNullByte(buffer, 1000);
+}

package/dist/fetch-stream.d.ts

@@ -0,0 +1,4 @@
+import { Readable } from 'node:stream';
+import type { ReadableStream as NodeReadableStream } from 'node:stream/web';
+export declare function toNodeReadableStream(stream: ReadableStream<Uint8Array>, url: string, stage: string): NodeReadableStream<Uint8Array>;
+export declare function toWebReadableStream(stream: Readable, url: string, stage: string): ReadableStream<Uint8Array>;

package/dist/fetch-stream.js

@@ -0,0 +1,28 @@
+import { Readable } from 'node:stream';
+import { FetchError } from './errors.js';
+import { isObject } from './type-guards.js';
+function isReadableStreamLike(value) {
+    if (!isObject(value))
+        return false;
+    return (typeof value['getReader'] === 'function' &&
+        typeof value['cancel'] === 'function' &&
+        typeof value['tee'] === 'function' &&
+        typeof value['locked'] === 'boolean');
+}
+function assertReadableStreamLike(stream, url, stage) {
+    if (isReadableStreamLike(stream))
+        return;
+    throw new FetchError('Invalid response stream', url, 500, {
+        reason: 'invalid_stream',
+        stage,
+    });
+}
+export function toNodeReadableStream(stream, url, stage) {
+    assertReadableStreamLike(stream, url, stage);
+    return stream;
+}
+export function toWebReadableStream(stream, url, stage) {
+    const converted = Readable.toWeb(stream);
+    assertReadableStreamLike(converted, url, stage);
+    return converted;
+}

package/dist/fetch.js

@@ -10,6 +10,8 @@ import { finished, pipeline } from 'node:stream/promises';
 import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib';
 import { config } from './config.js';
 import { createErrorWithCode, FetchError, isSystemError } from './errors.js';
+import { decodeBuffer, getCharsetFromContentType, isBinaryContent, resolveEncoding, } from './fetch-content.js';
+import { toNodeReadableStream, toWebReadableStream } from './fetch-stream.js';
 import { createDefaultBlockList, normalizeIpForBlockList, } from './ip-blocklist.js';
 import { getOperationId, getRequestId, logDebug, logError, logWarn, redactUrl, } from './observability.js';
 import { isError, isObject } from './type-guards.js';
@@ -26,23 +28,6 @@ const defaultRedactor = {
     redact: redactUrl,
 };
 const defaultFetch = (input, init) => globalThis.fetch(input, init);
-function assertReadableStreamLike(stream, url, stage) {
-    if (isObject(stream) && typeof stream['getReader'] === 'function')
-        return;
-    throw new FetchError('Invalid response stream', url, 500, {
-        reason: 'invalid_stream',
-        stage,
-    });
-}
-function toNodeReadableStream(stream, url, stage) {
-    assertReadableStreamLike(stream, url, stage);
-    return stream;
-}
-function toWebReadableStream(stream, url, stage) {
-    const converted = Readable.toWeb(stream);
-    assertReadableStreamLike(converted, url, stage);
-    return converted;
-}
 class IpBlocker {
     security;
     blockList = createDefaultBlockList();
@@ -456,6 +441,9 @@ class SafeDnsResolver {
         if (signal?.aborted) {
             throw createAbortSignalError();
         }
+        if (this.isBlockedHostname(normalizedHostname)) {
+            throw createErrorWithCode(`Blocked host: ${normalizedHostname}. Internal hosts are not allowed`, 'EBLOCKED');
+        }
         if (isIP(normalizedHostname)) {
             if (this.ipBlocker.isBlockedIp(normalizedHostname)) {
                 throw createErrorWithCode(`Blocked IP range: ${normalizedHostname}. Private IPs are not allowed`, 'EBLOCKED');
@@ -871,168 +859,6 @@ class RedirectFollower {
         }
     }
 }
-function getCharsetFromContentType(contentType) {
-    if (!contentType)
-        return undefined;
-    const match = /charset=([^;]+)/i.exec(contentType);
-    const charsetGroup = match?.[1];
-    if (!charsetGroup)
-        return undefined;
-    let charset = charsetGroup.trim();
-    if (charset.startsWith('"') && charset.endsWith('"')) {
-        charset = charset.slice(1, -1);
-    }
-    return charset.trim();
-}
-function createDecoder(encoding) {
-    if (!encoding)
-        return new TextDecoder('utf-8');
-    try {
-        return new TextDecoder(encoding);
-    }
-    catch {
-        return new TextDecoder('utf-8');
-    }
-}
-function decodeBuffer(buffer, encoding) {
-    return createDecoder(encoding).decode(buffer);
-}
-function normalizeEncodingLabel(encoding) {
-    return encoding?.trim().toLowerCase() ?? '';
-}
-function isUnicodeWideEncoding(encoding) {
-    const normalized = normalizeEncodingLabel(encoding);
-    return (normalized.startsWith('utf-16') ||
-        normalized.startsWith('utf-32') ||
-        normalized === 'ucs-2' ||
-        normalized === 'unicodefffe' ||
-        normalized === 'unicodefeff');
-}
-const BOM_SIGNATURES = [
-    // 4-byte BOMs must come first to avoid false matches with 2-byte prefixes
-    { bytes: [0xff, 0xfe, 0x00, 0x00], encoding: 'utf-32le' },
-    { bytes: [0x00, 0x00, 0xfe, 0xff], encoding: 'utf-32be' },
-    { bytes: [0xef, 0xbb, 0xbf], encoding: 'utf-8' },
-    { bytes: [0xff, 0xfe], encoding: 'utf-16le' },
-    { bytes: [0xfe, 0xff], encoding: 'utf-16be' },
-];
-function detectBomEncoding(buffer) {
-    for (const { bytes, encoding } of BOM_SIGNATURES) {
-        if (startsWithBytes(buffer, bytes))
-            return encoding;
-    }
-    return undefined;
-}
-function readQuotedValue(input, startIndex) {
-    const first = input[startIndex];
-    if (!first)
-        return '';
-    const quoted = first === '"' || first === "'";
-    if (quoted) {
-        const end = input.indexOf(first, startIndex + 1);
-        return end === -1 ? '' : input.slice(startIndex + 1, end).trim();
-    }
-    const tail = input.slice(startIndex);
-    const stop = tail.search(/[\s/>]/);
-    return (stop === -1 ? tail : tail.slice(0, stop)).trim();
-}
-function extractHtmlCharset(headSnippet) {
-    const lower = headSnippet.toLowerCase();
-    const charsetToken = 'charset=';
-    const charsetIdx = lower.indexOf(charsetToken);
-    if (charsetIdx === -1)
-        return undefined;
-    const valueStart = charsetIdx + charsetToken.length;
-    const charset = readQuotedValue(headSnippet, valueStart);
-    return charset ? charset.toLowerCase() : undefined;
-}
-function extractXmlEncoding(headSnippet) {
-    const lower = headSnippet.toLowerCase();
-    const xmlStart = lower.indexOf('<?xml');
-    if (xmlStart === -1)
-        return undefined;
-    const xmlEnd = lower.indexOf('?>', xmlStart);
-    const declaration = xmlEnd === -1
-        ? headSnippet.slice(xmlStart)
-        : headSnippet.slice(xmlStart, xmlEnd + 2);
-    const declarationLower = declaration.toLowerCase();
-    const encodingToken = 'encoding=';
-    const encodingIdx = declarationLower.indexOf(encodingToken);
-    if (encodingIdx === -1)
-        return undefined;
-    const valueStart = encodingIdx + encodingToken.length;
-    const encoding = readQuotedValue(declaration, valueStart);
-    return encoding ? encoding.toLowerCase() : undefined;
-}
-function detectHtmlDeclaredEncoding(buffer) {
-    const scanSize = Math.min(buffer.length, 8_192);
-    if (scanSize === 0)
-        return undefined;
-    const headSnippet = Buffer.from(buffer.buffer, buffer.byteOffset, scanSize).toString('latin1');
-    return extractHtmlCharset(headSnippet) ?? extractXmlEncoding(headSnippet);
-}
-function resolveEncoding(declaredEncoding, sample) {
-    const bomEncoding = detectBomEncoding(sample);
-    if (bomEncoding)
-        return bomEncoding;
-    if (declaredEncoding)
-        return declaredEncoding;
-    return detectHtmlDeclaredEncoding(sample);
-}
-const BINARY_SIGNATURES = [
-    [0x25, 0x50, 0x44, 0x46],
-    [0x89, 0x50, 0x4e, 0x47],
-    [0x47, 0x49, 0x46, 0x38],
-    [0xff, 0xd8, 0xff],
-    [0x52, 0x49, 0x46, 0x46],
-    [0x42, 0x4d],
-    [0x49, 0x49, 0x2a, 0x00],
-    [0x4d, 0x4d, 0x00, 0x2a],
-    [0x00, 0x00, 0x01, 0x00],
-    [0x50, 0x4b, 0x03, 0x04],
-    [0x1f, 0x8b],
-    [0x42, 0x5a, 0x68],
-    [0x52, 0x61, 0x72, 0x21],
-    [0x37, 0x7a, 0xbc, 0xaf],
-    [0x7f, 0x45, 0x4c, 0x46],
-    [0x4d, 0x5a],
-    [0xcf, 0xfa, 0xed, 0xfe],
-    [0x00, 0x61, 0x73, 0x6d],
-    [0x1a, 0x45, 0xdf, 0xa3],
-    [0x66, 0x74, 0x79, 0x70],
-    [0x46, 0x4c, 0x56],
-    [0x49, 0x44, 0x33],
-    [0xff, 0xfb],
-    [0xff, 0xfa],
-    [0x4f, 0x67, 0x67, 0x53],
-    [0x66, 0x4c, 0x61, 0x43],
-    [0x4d, 0x54, 0x68, 0x64],
-    [0x77, 0x4f, 0x46, 0x46],
-    [0x00, 0x01, 0x00, 0x00],
-    [0x4f, 0x54, 0x54, 0x4f],
-    [0x53, 0x51, 0x4c, 0x69],
-];
-function startsWithBytes(buffer, signature) {
-    const sigLen = signature.length;
-    if (buffer.length < sigLen)
-        return false;
-    for (let i = 0; i < sigLen; i += 1) {
-        if (buffer[i] !== signature[i])
-            return false;
-    }
-    return true;
-}
-function hasNullByte(buffer, limit) {
-    const checkLen = Math.min(buffer.length, limit);
-    return buffer.subarray(0, checkLen).includes(0x00);
-}
-function isBinaryContent(buffer, encoding) {
-    for (const signature of BINARY_SIGNATURES) {
-        if (startsWithBytes(buffer, signature))
-            return true;
-    }
-    return !isUnicodeWideEncoding(encoding) && hasNullByte(buffer, 1000);
-}
 class ResponseTextReader {
     async read(response, url, maxBytes, signal, encoding) {
         const { buffer, encoding: effectiveEncoding, truncated, } = await this.readBuffer(response, url, maxBytes, signal, encoding);
@@ -1298,7 +1124,7 @@ function createDecompressor(encoding) {
 function createPumpedStream(initialChunk, reader) {
     return new ReadableStream({
         start(controller) {
-            if (initialChunk.byteLength > 0) {
+            if (initialChunk && initialChunk.byteLength > 0) {
                 controller.enqueue(initialChunk);
             }
         },
@@ -1321,31 +1147,6 @@ function createPumpedStream(initialChunk, reader) {
         },
     });
 }
-function isLikelyCompressed(chunk, encoding) {
-    if (chunk.byteLength === 0)
-        return false;
-    if (encoding === 'gzip') {
-        return chunk.byteLength >= 2 && chunk[0] === 0x1f && chunk[1] === 0x8b;
-    }
-    if (encoding === 'deflate') {
-        if (chunk.byteLength < 2)
-            return false;
-        const byte0 = chunk[0] ?? 0;
-        const byte1 = chunk[1] ?? 0;
-        const cm = byte0 & 0x0f;
-        if (cm !== 8)
-            return false;
-        return (byte0 * 256 + byte1) % 31 === 0;
-    }
-    let nonPrintable = 0;
-    const limit = Math.min(chunk.length, 50);
-    for (let i = 0; i < limit; i += 1) {
-        const b = chunk[i] ?? 0;
-        if (b < 0x09 || (b > 0x0d && b < 0x20) || b === 0x7f)
-            nonPrintable += 1;
-    }
-    return nonPrintable / limit > 0.1;
-}
 async function decodeResponseIfNeeded(response, url, signal) {
     const encodingHeader = response.headers.get('content-encoding');
     const parsedEncodings = parseContentEncodings(encodingHeader);
@@ -1361,76 +1162,81 @@ async function decodeResponseIfNeeded(response, url, signal) {
     }
     if (!response.body)
         return response;
-    // Peek at first chunk to check if actually compressed
-    const reader = response.body.getReader();
-    let initialChunk;
-    try {
-        const { done, value } = await reader.read();
-        if (done) {
-            return new Response(null, {
-                status: response.status,
-                statusText: response.statusText,
-                headers: response.headers,
-            });
-        }
-        initialChunk = value;
-    }
-    catch (error) {
-        // If read fails, throw properly
-        throw new FetchError(`Failed to read response body: ${isError(error) ? error.message : String(error)}`, url, 502);
-    }
+    const [decodeBranch, passthroughBranch] = response.body.tee();
     const decodeOrder = encodings
         .slice()
         .reverse()
         .filter(isSupportedContentEncoding);
-    const firstDecodeEncoding = decodeOrder[0];
-    if (!firstDecodeEncoding ||
-        !isLikelyCompressed(initialChunk, firstDecodeEncoding)) {
-        const body = createPumpedStream(initialChunk, reader);
-        const headers = new Headers(response.headers);
-        headers.delete('content-encoding');
-        headers.delete('content-length');
-        return new Response(body, {
-            status: response.status,
-            statusText: response.statusText,
-            headers,
-        });
-    }
     const decompressors = decodeOrder.map((encoding) => createDecompressor(encoding));
-    const sourceStream = Readable.fromWeb(toNodeReadableStream(createPumpedStream(initialChunk, reader), url, 'response:decode-content-encoding'));
+    const decodeSource = Readable.fromWeb(toNodeReadableStream(decodeBranch, url, 'response:decode-content-encoding'));
     const decodedNodeStream = new PassThrough();
-    const pipelinePromise = pipeline([
-        sourceStream,
+    const decodedPipeline = pipeline([
+        decodeSource,
         ...decompressors,
         decodedNodeStream,
     ]);
-    const abortHandler = () => {
-        sourceStream.destroy();
+    const headers = new Headers(response.headers);
+    headers.delete('content-encoding');
+    headers.delete('content-length');
+    const abortDecodePipeline = () => {
+        decodeSource.destroy();
         for (const decompressor of decompressors) {
             decompressor.destroy();
         }
         decodedNodeStream.destroy();
     };
     if (signal) {
-        signal.addEventListener('abort', abortHandler, { once: true });
+        signal.addEventListener('abort', abortDecodePipeline, { once: true });
     }
-    void pipelinePromise.catch((error) => {
+    void decodedPipeline.catch((error) => {
         decodedNodeStream.destroy(error instanceof Error ? error : new Error(String(error)));
     });
-    const decodedBody = toWebReadableStream(decodedNodeStream, url, 'response:decode-content-encoding');
-    const headers = new Headers(response.headers);
-    headers.delete('content-encoding');
-    headers.delete('content-length');
-    if (signal) {
-        void finished(decodedNodeStream, { cleanup: true }).finally(() => {
-            signal.removeEventListener('abort', abortHandler);
+    const decodedBodyStream = toWebReadableStream(decodedNodeStream, url, 'response:decode-content-encoding');
+    const decodedReader = decodedBodyStream.getReader();
+    const clearAbortListener = () => {
+        if (!signal)
+            return;
+        signal.removeEventListener('abort', abortDecodePipeline);
+    };
+    try {
+        const first = await decodedReader.read();
+        if (first.done) {
+            clearAbortListener();
+            void passthroughBranch.cancel().catch(() => undefined);
+            return new Response(null, {
+                status: response.status,
+                statusText: response.statusText,
+                headers,
+            });
+        }
+        void passthroughBranch.cancel().catch(() => undefined);
+        const body = createPumpedStream(first.value, decodedReader);
+        if (signal) {
+            void finished(decodedNodeStream, { cleanup: true }).finally(() => {
+                clearAbortListener();
+            });
+        }
+        return new Response(body, {
+            status: response.status,
+            statusText: response.statusText,
+            headers,
+        });
+    }
+    catch (error) {
+        clearAbortListener();
+        abortDecodePipeline();
+        void decodedReader.cancel(error).catch(() => undefined);
+        logDebug('Content-Encoding decode failed; using passthrough body', {
+            url: redactUrl(url),
+            encoding: encodingHeader ?? encodings.join(','),
+            error: isError(error) ? error.message : String(error),
+        });
+        return new Response(passthroughBranch, {
+            status: response.status,
+            statusText: response.statusText,
+            headers,
         });
     }
-    return new Response(decodedBody, {
-        status: response.status,
-        statusText: response.statusText,
-        headers,
-    });
 }
 async function readAndRecordDecodedResponse(response, finalUrl, ctx, telemetry, reader, maxBytes, mode, signal) {
     const responseError = resolveResponseError(response, finalUrl);

package/dist/tools.js

@@ -446,8 +446,21 @@ function buildToolContentBlocks(structuredContent, resourceLink, embeddedResourc
 }
 function resolveNormalizedUrl(url) {
     const { normalizedUrl: validatedUrl } = normalizeUrl(url);
-    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
-    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+    const transformedResult = transformToRawUrl(validatedUrl);
+    if (!transformedResult.transformed) {
+        return {
+            normalizedUrl: validatedUrl,
+            originalUrl: validatedUrl,
+            transformed: false,
+        };
+    }
+    // Re-validate transformed URLs so blocked-host and length policies still apply.
+    const { normalizedUrl: transformedUrl } = normalizeUrl(transformedResult.url);
+    return {
+        normalizedUrl: transformedUrl,
+        originalUrl: validatedUrl,
+        transformed: true,
+    };
 }
 function logRawUrlTransformation(resolvedUrl) {
     if (!resolvedUrl.transformed)
@@ -493,10 +506,12 @@ function attemptCacheRetrieval(params) {
         return null;
     }
     logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
+    const finalUrl = cached.url !== normalizedUrl ? cached.url : undefined;
     return {
         data,
         fromCache: true,
         url: normalizedUrl,
+        ...(finalUrl ? { finalUrl } : {}),
         fetchedAt: cached.fetchedAt,
         cacheKey,
     };
@@ -675,12 +690,16 @@ export function parseCachedMarkdownResult(cached) {
     if (typeof markdown !== 'string')
         return undefined;
     const metadata = normalizeExtractedMetadata(result.data.metadata);
+    const truncated = result.data.truncated ?? false;
+    const persistedMarkdown = truncated
+        ? appendTruncationMarker(markdown, TRUNCATION_MARKER)
+        : markdown;
     return {
-        content: markdown,
-        markdown,
+        content: persistedMarkdown,
+        markdown: persistedMarkdown,
         title: result.data.title,
         ...(metadata ? { metadata } : {}),
-        truncated: result.data.truncated ?? false,
+        truncated,
     };
 }
 const markdownTransform = async (input, url, signal, skipNoiseRemoval) => {
@@ -695,8 +714,11 @@ const markdownTransform = async (input, url, signal, skipNoiseRemoval) => {
     return { ...result, content: result.markdown, truncated };
 };
 function serializeMarkdownResult(result) {
+    const persistedMarkdown = result.truncated
+        ? appendTruncationMarker(result.markdown, TRUNCATION_MARKER)
+        : result.markdown;
     return JSON.stringify({
-        markdown: result.markdown,
+        markdown: persistedMarkdown,
         title: result.title,
         metadata: result.metadata,
         truncated: result.truncated,

package/dist/transform.js

@@ -2369,19 +2369,15 @@ function resolveWorkerFallback(error, htmlOrBuffer, url, options) {
         });
         return transformInputInProcess(htmlOrBuffer, url, options);
     }
+    abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
     if (error instanceof FetchError)
         throw error;
-    abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
     const message = getErrorMessage(error);
-    logWarn('Transform worker failed; refusing in-process fallback', {
+    logWarn('Transform worker failed; falling back to in-process', {
         url: redactUrl(url),
         error: message,
     });
-    throw new FetchError('Transform worker failed', url, 503, {
-        reason: 'worker_failed',
-        stage: 'transform:worker',
-        error: message,
-    });
+    return transformInputInProcess(htmlOrBuffer, url, options);
 }
 async function runWorkerTransformWithFallback(htmlOrBuffer, url, options) {
     const workerStage = stageTracker.start(url, 'transform:worker');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",
@@ -52,6 +52,7 @@
     "start": "node dist/index.js",
     "format": "prettier --write .",
     "type-check": "node scripts/tasks.mjs type-check",
+    "type-check:tests": "node scripts/tasks.mjs type-check:tests",
     "type-check:diagnostics": "tsc --noEmit --extendedDiagnostics",
     "type-check:trace": "node -e \"require('fs').rmSync('.ts-trace',{recursive:true,force:true})\" && tsc --noEmit --generateTrace .ts-trace",
     "lint": "eslint .",