@j0hanz/fetch-url-mcp 1.6.0 → 1.7.0

package/dist/transform/worker-pool.js

@@ -1,19 +1,18 @@
 import { AsyncLocalStorage, AsyncResource } from 'node:async_hooks';
 import { Buffer } from 'node:buffer';
-import { fork } from 'node:child_process';
 import { availableParallelism } from 'node:os';
-import { fileURLToPath } from 'node:url';
+import process from 'node:process';
 import { isSharedArrayBuffer } from 'node:util/types';
-import { Worker, } from 'node:worker_threads';
+import { isMainThread, parentPort, Worker, } from 'node:worker_threads';
 import { config } from '../lib/core.js';
 import { logWarn } from '../lib/core.js';
 import { createAbortError } from '../lib/utils.js';
 import { FetchError, getErrorMessage } from '../lib/utils.js';
 import { createUnrefTimeout } from '../lib/utils.js';
 import { isObject } from '../lib/utils.js';
-// ---------------------------------------------------------------------------
+import { createTransformMessageHandler } from './shared.js';
+import { transformHtmlToMarkdownInProcess } from './transform.js';
 // Worker message validation
-// ---------------------------------------------------------------------------
 function isWorkerResultPayload(value) {
     if (!isObject(value))
         return false;
@@ -91,7 +90,7 @@ function ensureTightBuffer(buffer) {
     }
     return Buffer.from(buffer);
 }
-function buildWorkerDispatchPayload(task, supportsTransferList) {
+function buildWorkerDispatchPayload(task) {
     const message = {
         type: 'transform',
         id: task.id,
@@ -105,12 +104,6 @@ function buildWorkerDispatchPayload(task, supportsTransferList) {
         return { message };
     }
     const htmlBuffer = ensureTightBuffer(task.htmlBuffer);
-    if (!supportsTransferList) {
-        message.htmlBuffer = htmlBuffer;
-        if (task.encoding)
-            message.encoding = task.encoding;
-        return { message };
-    }
     const transferableHtmlBuffer = Uint8Array.from(htmlBuffer);
     message.htmlBuffer = transferableHtmlBuffer;
     if (task.encoding)
@@ -120,139 +113,15 @@ function buildWorkerDispatchPayload(task, supportsTransferList) {
         return { message };
     return { message, transferList: [backingBuffer] };
 }
-// ---------------------------------------------------------------------------
 // Pool sizing & constants
-// ---------------------------------------------------------------------------
-/**
- * Worker Pool Sizing Configuration
- *
- * Default: min(4, floor(availableParallelism() / 2)), constrained to [2, N]
- *
- * Tuning Guidance:
- * - **Default behavior**: Appropriate for most deployments. Uses half of available
- *   CPU threads (capped at 4) to balance throughput with system resource availability.
- *
- * - **CPU-limited containers**: If running in a container with strict CPU limits
- *   (e.g., Docker with --cpus=2), the default may over-subscribe. Consider setting
- *   maxWorkerScale to match the container's CPU limit.
- *
- * - **High-concurrency workloads**: For dedicated servers handling many concurrent
- *   fetch requests, increasing maxWorkerScale to (availableParallelism() + 2) may
- *   improve throughput by overlapping I/O wait with computation.
- *
- * - **Memory-constrained environments**: Each worker allocates ~50-100MB for DOM
- *   parsing. If memory is limited, reduce maxWorkerScale to (availableParallelism() / 2)
- *   or lower to prevent OOM errors.
- *
- * - **Shared hosting**: On shared systems where CPU is contested, reducing the pool
- *   size prevents starving other processes. Consider maxWorkerScale = 2 or using
- *   process-based workers (TRANSFORM_WORKER_MODE=process) for better isolation.
- *
- * Configuration:
- * - TRANSFORM_MAX_WORKER_SCALE env var (default: availableParallelism())
- * - TRANSFORM_WORKER_MODE env var: 'threads' (default) or 'process'
- *
- * See config.ts for full worker configuration options.
- */
+// Core tuning: ~half of available CPUs as baseline, capped by config limits.
 const POOL_MIN_WORKERS = Math.max(2, Math.min(4, Math.floor(availableParallelism() / 2)));
 const POOL_MAX_WORKERS = config.transform.maxWorkerScale;
 const POOL_SCALE_THRESHOLD = 0.5;
 const WORKER_NAME_PREFIX = 'fetch-url-mcp-transform';
 const DEFAULT_TIMEOUT_MS = config.transform.timeoutMs;
-const TRANSFORM_CHILD_PATH = fileURLToPath(new URL('./workers/transform-child.js', import.meta.url));
-// ---------------------------------------------------------------------------
-// Worker host spawners
-// ---------------------------------------------------------------------------
-function createThreadWorkerHost(_workerIndex, name) {
-    const resourceLimits = config.transform.workerResourceLimits;
-    const worker = new Worker(new URL('./workers/transform-worker.js', import.meta.url), {
-        name,
-        ...(resourceLimits ? { resourceLimits } : {}),
-    });
-    return {
-        kind: 'thread',
-        supportsTransferList: true,
-        threadId: worker.threadId,
-        postMessage: (message, transferList) => {
-            worker.postMessage(message, transferList);
-        },
-        terminate: async () => {
-            await worker.terminate();
-        },
-        unref: () => {
-            worker.unref();
-        },
-        onMessage: (handler) => {
-            worker.on('message', handler);
-        },
-        onError: (handler) => {
-            worker.on('error', handler);
-            worker.on('messageerror', handler);
-        },
-        onExit: (handler) => {
-            worker.on('exit', (code) => {
-                handler(code, null);
-            });
-        },
-    };
-}
-function createProcessWorkerHost(workerIndex, name) {
-    const child = fork(TRANSFORM_CHILD_PATH, [], {
-        stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
-        serialization: 'advanced',
-        env: {
-            ...process.env,
-            FETCH_URL_MCP_WORKER_INDEX: String(workerIndex),
-            FETCH_URL_MCP_WORKER_NAME: name,
-        },
-    });
-    if (child.pid === undefined) {
-        throw new Error('Failed to fork process');
-    }
-    return {
-        kind: 'process',
-        supportsTransferList: false,
-        pid: child.pid,
-        postMessage: (message) => {
-            if (!child.connected) {
-                throw new Error('Transform worker IPC channel is closed');
-            }
-            child.send(message);
-        },
-        terminate: () => new Promise((resolve) => {
-            if (child.exitCode !== null || child.killed) {
-                resolve();
-                return;
-            }
-            child.once('exit', () => {
-                resolve();
-            });
-            try {
-                child.kill();
-            }
-            catch {
-                resolve();
-            }
-        }),
-        unref: () => {
-            child.unref();
-        },
-        onMessage: (handler) => {
-            child.on('message', handler);
-        },
-        onError: (handler) => {
-            child.on('error', handler);
-        },
-        onExit: (handler) => {
-            child.on('exit', (code, signal) => {
-                handler(code, signal);
-            });
-        },
-    };
-}
-// ---------------------------------------------------------------------------
+const TRANSFORM_WORKER_PATH = new URL(import.meta.url);
 // WorkerPool
-// ---------------------------------------------------------------------------
 class WorkerPool {
     static CLOSED_MESSAGE = 'Transform worker pool closed';
     workers = [];
@@ -265,17 +134,15 @@ class WorkerPool {
     cancelAcks = new Map();
     timeoutMs;
     queueMax;
-    spawnWorkerImpl;
     closed = false;
     taskIdSeq = 0;
-    constructor(size, timeoutMs, spawnWorker) {
+    constructor(size, timeoutMs) {
         this.capacity =
             size === 0
                 ? 0
                 : Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
         this.timeoutMs = timeoutMs;
         this.queueMax = this.maxCapacity * 4;
-        this.spawnWorkerImpl = spawnWorker;
     }
     async transform(htmlOrBuffer, url, options) {
         this.ensureOpen();
@@ -315,7 +182,7 @@ class WorkerPool {
             return;
         this.closed = true;
         const terminations = this.workers
-            .map((slot) => slot?.host.terminate())
+            .map((slot) => slot?.worker.terminate().catch(() => undefined))
             .filter((p) => p !== undefined);
         this.workers.fill(undefined);
         this.workers.length = 0;
@@ -442,7 +309,7 @@ class WorkerPool {
         }
         if (slot) {
             try {
-                slot.host.postMessage({ type: 'cancel', id });
+                slot.worker.postMessage({ type: 'cancel', id });
             }
             catch {
                 // Worker may be unavailable; failure is acceptable during abort
@@ -465,19 +332,25 @@ class WorkerPool {
     }
     spawnWorker(workerIndex) {
         const name = `${WORKER_NAME_PREFIX}-${workerIndex + 1}`;
-        const host = this.spawnWorkerImpl(workerIndex, name);
-        host.unref();
-        host.onMessage((raw) => {
+        const resourceLimits = config.transform.workerResourceLimits;
+        const worker = new Worker(TRANSFORM_WORKER_PATH, {
+            name,
+            ...(resourceLimits ? { resourceLimits } : {}),
+        });
+        worker.unref();
+        worker.on('message', (raw) => {
             this.onWorkerMessage(workerIndex, raw);
         });
-        host.onError((error) => {
+        worker.on('error', (error) => {
+            this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
+        });
+        worker.on('messageerror', (error) => {
             this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
         });
-        host.onExit((code, signal) => {
-            const suffix = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`;
-            this.onWorkerBroken(workerIndex, `Transform worker exited (${suffix})`);
+        worker.on('exit', (code) => {
+            this.onWorkerBroken(workerIndex, `Transform worker exited (code ${code ?? 'unknown'})`);
         });
-        return { host, busy: false, currentTaskId: null, name };
+        return { worker, busy: false, currentTaskId: null, name };
     }
     onWorkerBroken(workerIndex, message) {
         if (this.closed)
@@ -488,11 +361,8 @@ class WorkerPool {
         logWarn('Transform worker unavailable; restarting', {
             reason: message,
             workerIndex,
-            workerKind: slot.host.kind,
             workerName: slot.name,
-            ...(slot.host.kind === 'process'
-                ? { pid: slot.host.pid }
-                : { threadId: slot.host.threadId }),
+            threadId: slot.worker.threadId,
         });
         if (slot.busy && slot.currentTaskId) {
             this.failTask(slot.currentTaskId, new FetchError(message, '', 503, { reason: 'worker_exit' }));
@@ -504,7 +374,7 @@ class WorkerPool {
             return;
         const target = slot ?? this.workers[workerIndex];
         if (target) {
-            target.host.terminate().catch(() => undefined);
+            target.worker.terminate().catch(() => undefined);
         }
         this.workers[workerIndex] = this.spawnWorker(workerIndex);
         this.drainQueue();
@@ -645,7 +515,7 @@ class WorkerPool {
         void timeout.promise
             .then(() => {
             try {
-                slot.host.postMessage({ type: 'cancel', id: task.id });
+                slot.worker.postMessage({ type: 'cancel', id: task.id });
             }
             catch {
                 // Worker may be unavailable; proceed with timeout handling
@@ -675,8 +545,8 @@ class WorkerPool {
             cancelPending: false,
         });
         try {
-            const { message, transferList } = buildWorkerDispatchPayload(task, slot.host.supportsTransferList);
-            slot.host.postMessage(message, transferList);
+            const { message, transferList } = buildWorkerDispatchPayload(task);
+            slot.worker.postMessage(message, transferList);
         }
         catch (error) {
             timeout.cancel();
@@ -717,18 +587,11 @@ class WorkerPool {
         }
     }
 }
-// ---------------------------------------------------------------------------
 // Pool singleton management
-// ---------------------------------------------------------------------------
 let workerPool = null;
-function resolveWorkerSpawner() {
-    return config.transform.workerMode === 'process'
-        ? createProcessWorkerHost
-        : createThreadWorkerHost;
-}
 export function getOrCreateWorkerPool() {
     const size = config.transform.maxWorkerScale === 0 ? 0 : POOL_MIN_WORKERS;
-    workerPool ??= new WorkerPool(size, DEFAULT_TIMEOUT_MS, resolveWorkerSpawner());
+    workerPool ??= new WorkerPool(size, DEFAULT_TIMEOUT_MS);
     return workerPool;
 }
 export function getWorkerPoolStats() {
@@ -746,3 +609,24 @@ export async function shutdownWorkerPool() {
     await workerPool.close();
     workerPool = null;
 }
+// Worker thread message handling
+if (!isMainThread && parentPort) {
+    const port = parentPort;
+    const onMessage = createTransformMessageHandler({
+        sendMessage: (message) => {
+            port.postMessage(message);
+        },
+        runTransform: transformHtmlToMarkdownInProcess,
+    });
+    port.on('message', onMessage);
+}
+else if (process.send) {
+    const send = process.send.bind(process);
+    const onMessage = createTransformMessageHandler({
+        sendMessage: (message) => {
+            send(message);
+        },
+        runTransform: transformHtmlToMarkdownInProcess,
+    });
+    process.on('message', onMessage);
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
-  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
+  "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -33,12 +33,15 @@
   "keywords": [
     "mcp",
     "mcp-server",
+    "model-context-protocol",
     "web-fetching",
     "content-extraction",
     "readability",
     "markdown",
+    "html-to-markdown",
+    "web-scraper",
+    "llm-context",
     "ai-tools",
-    "model-context-protocol",
     "fetch-url-mcp"
   ],
   "scripts": {
@@ -78,16 +81,16 @@
     "@eslint/js": "^10.0.1",
     "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/node": "^24",
-    "eslint": "^10.0.2",
+    "eslint": "^10.0.3",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-de-morgan": "^2.1.1",
     "eslint-plugin-depend": "^1.5.0",
     "eslint-plugin-unused-imports": "^4.4.1",
-    "knip": "^5.85.0",
+    "knip": "^5.86.0",
     "prettier": "^3.8.1",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.56.1"
+    "typescript-eslint": "^8.57.0"
   },
   "engines": {
     "node": ">=24"

package/dist/transform/workers/transform-child.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/transform/workers/transform-child.js

@@ -1,15 +0,0 @@
-import process from 'node:process';
-import { transformHtmlToMarkdownInProcess } from '../transform.js';
-import { createTransformMessageHandler } from './shared.js';
-const send = process.send?.bind(process);
-if (!send)
-    throw new Error('transform-child started without IPC channel');
-const sendMessage = send;
-function postMessage(message) {
-    sendMessage(message);
-}
-const onMessage = createTransformMessageHandler({
-    sendMessage: postMessage,
-    runTransform: transformHtmlToMarkdownInProcess,
-});
-process.on('message', onMessage);

package/dist/transform/workers/transform-worker.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/transform/workers/transform-worker.js

@@ -1,13 +0,0 @@
-import { parentPort } from 'node:worker_threads';
-import { transformHtmlToMarkdownInProcess } from '../transform.js';
-import { createTransformMessageHandler } from './shared.js';
-if (!parentPort)
-    throw new Error('transform-worker started without parentPort');
-const port = parentPort;
-const onMessage = createTransformMessageHandler({
-    sendMessage: (message) => {
-        port.postMessage(message);
-    },
-    runTransform: transformHtmlToMarkdownInProcess,
-});
-port.on('message', onMessage);