@j0hanz/fetch-url-mcp 1.9.1 → 1.9.2

package/dist/transform/worker-pool.js

@@ -5,11 +5,8 @@ import process from 'node:process';
 import { isSharedArrayBuffer } from 'node:util/types';
 import { isMainThread, parentPort, Worker, } from 'node:worker_threads';
 import { z } from 'zod';
-import { config } from '../lib/core.js';
-import { logWarn } from '../lib/core.js';
-import { createAbortError } from '../lib/utils.js';
-import { FetchError, getErrorMessage } from '../lib/utils.js';
-import { createUnrefTimeout } from '../lib/utils.js';
+import { config, logWarn } from '../lib/core.js';
+import { createAbortError, createUnrefTimeout, FetchError, getErrorMessage, } from '../lib/utils.js';
 import { normalizeExtractedMetadata } from '../schemas.js';
 import { createTransformMessageHandler } from './shared.js';
 import { transformHtmlToMarkdownInProcess } from './transform.js';
@@ -107,6 +104,93 @@ const POOL_SCALE_THRESHOLD = 0.5;
 const WORKER_NAME_PREFIX = 'fetch-url-mcp-transform';
 const DEFAULT_TIMEOUT_MS = config.transform.timeoutMs;
 const TRANSFORM_WORKER_PATH = new URL(import.meta.url);
+// TaskQueue — array-deque with auto-compaction
+class TaskQueue {
+    items = [];
+    head = 0;
+    get depth() {
+        const d = this.items.length - this.head;
+        return d > 0 ? d : 0;
+    }
+    enqueue(item) {
+        this.items.push(item);
+    }
+    dequeue() {
+        while (this.head < this.items.length) {
+            const item = this.items[this.head];
+            this.head += 1;
+            if (item) {
+                this.compact();
+                return item;
+            }
+        }
+        this.compact();
+        return null;
+    }
+    removeById(id) {
+        for (let i = this.head; i < this.items.length; i += 1) {
+            const item = this.items[i];
+            if (item?.id === id) {
+                this.items.splice(i, 1);
+                this.compact();
+                return item;
+            }
+        }
+        return undefined;
+    }
+    drain(callback) {
+        for (let i = this.head; i < this.items.length; i += 1) {
+            const item = this.items[i];
+            if (item)
+                callback(item);
+        }
+        this.items.length = 0;
+        this.head = 0;
+    }
+    compact() {
+        if (this.head === 0)
+            return;
+        if (this.head >= this.items.length ||
+            (this.head > 1024 && this.head > this.items.length / 2)) {
+            this.items.splice(0, this.head);
+            this.head = 0;
+        }
+    }
+}
+// CancelAckTracker — isolates the cancel-acknowledgement protocol
+class CancelAckTracker {
+    pending = new Map();
+    resolve(id) {
+        const entry = this.pending.get(id);
+        if (!entry)
+            return;
+        entry.timeout.cancel();
+        entry.resolve();
+    }
+    wait(id, timeoutMs) {
+        const existing = this.pending.get(id);
+        if (existing)
+            return existing.promise;
+        let resolve = () => { };
+        const timeout = createUnrefTimeout(timeoutMs, undefined);
+        const racePromise = new Promise((finish) => {
+            resolve = finish;
+        });
+        const promise = Promise.race([racePromise, timeout.promise]).finally(() => {
+            this.pending.delete(id);
+            timeout.cancel();
+        });
+        this.pending.set(id, { promise, resolve, timeout });
+        return promise;
+    }
+    dispose() {
+        for (const entry of this.pending.values()) {
+            entry.timeout.cancel();
+            entry.resolve();
+        }
+        this.pending.clear();
+    }
+}
 // WorkerPool
 class WorkerPool {
     static CLOSED_MESSAGE = 'Transform worker pool closed';
@@ -114,14 +198,14 @@ class WorkerPool {
     capacity;
     minCapacity = POOL_MIN_WORKERS;
     maxCapacity = POOL_MAX_WORKERS;
-    queue = [];
-    queueHead = 0;
+    queue = new TaskQueue();
     inflight = new Map();
-    cancelAcks = new Map();
+    cancelAcks = new CancelAckTracker();
     timeoutMs;
     queueMax;
     closed = false;
     taskIdSeq = 0;
+    busyCount = 0;
     constructor(size, timeoutMs) {
         this.capacity =
             size === 0
@@ -134,7 +218,7 @@ class WorkerPool {
         this.ensureOpen();
         if (options.signal?.aborted)
             throw createAbortError(url, 'transform:enqueue');
-        if (this.getQueueDepth() >= this.queueMax) {
+        if (this.queue.depth >= this.queueMax) {
             throw new FetchError('Transform worker queue is full', url, 503, {
                 reason: 'queue_full',
                 stage: 'transform:enqueue',
@@ -142,16 +226,15 @@ class WorkerPool {
         }
         return new Promise((resolve, reject) => {
             const task = this.createPendingTask(htmlOrBuffer, url, options, resolve, reject);
-            this.queue.push(task);
+            this.queue.enqueue(task);
             this.drainQueue();
         });
     }
     getQueueDepth() {
-        const depth = this.queue.length - this.queueHead;
-        return depth > 0 ? depth : 0;
+        return this.queue.depth;
     }
     getActiveWorkers() {
-        return this.workers.filter((s) => s?.busy).length;
+        return this.busyCount;
     }
     getCapacity() {
         return this.capacity;
@@ -172,6 +255,8 @@ class WorkerPool {
             .filter((p) => p !== undefined);
         this.workers.fill(undefined);
         this.workers.length = 0;
+        this.busyCount = 0;
+        this.cancelAcks.dispose();
         for (const id of Array.from(this.inflight.keys())) {
             const inflight = this.takeInflight(id);
             if (!inflight)
@@ -180,17 +265,12 @@ class WorkerPool {
                 inflight.reject(new Error(WorkerPool.CLOSED_MESSAGE));
             });
         }
-        for (let i = this.queueHead; i < this.queue.length; i += 1) {
-            const task = this.queue[i];
-            if (!task)
-                continue;
+        this.queue.drain((task) => {
             this.clearAbortListener(task.signal, task.abortListener);
             this.finalizeTask(task.context, () => {
                 task.reject(new Error(WorkerPool.CLOSED_MESSAGE));
             });
-        }
-        this.queue.length = 0;
-        this.queueHead = 0;
+        });
         await Promise.allSettled(terminations);
     }
     ensureOpen() {
@@ -243,48 +323,13 @@ class WorkerPool {
             void this.abortInflight(id, url, inflight.workerIndex);
             return;
         }
-        const queuedIndex = this.findQueuedIndex(id);
-        if (queuedIndex !== null) {
-            const task = this.queue[queuedIndex];
-            if (task)
-                this.clearAbortListener(task.signal, task.abortListener);
-            this.queue.splice(queuedIndex, 1);
-            if (task) {
-                this.finalizeTask(task.context, () => {
-                    task.reject(createAbortError(url, 'transform:queued-abort'));
-                });
-            }
-            else {
-                this.finalizeTask(context, () => {
-                    reject(createAbortError(url, 'transform:queued-abort'));
-                });
-            }
-            this.maybeCompactQueue();
-        }
-    }
-    resolveCancelAck(id) {
-        const pending = this.cancelAcks.get(id);
-        if (!pending)
-            return;
-        pending.timeout.cancel();
-        pending.resolve();
-    }
-    waitForCancelAck(id) {
-        const existing = this.cancelAcks.get(id);
-        if (existing) {
-            return existing.promise;
+        const queuedTask = this.queue.removeById(id);
+        if (queuedTask) {
+            this.clearAbortListener(queuedTask.signal, queuedTask.abortListener);
+            this.finalizeTask(queuedTask.context, () => {
+                queuedTask.reject(createAbortError(url, 'transform:queued-abort'));
+            });
         }
-        let resolve = () => { };
-        const timeout = createUnrefTimeout(config.transform.cancelAckTimeoutMs, undefined);
-        const racePromise = new Promise((finish) => {
-            resolve = finish;
-        });
-        const promise = Promise.race([racePromise, timeout.promise]).finally(() => {
-            this.cancelAcks.delete(id);
-            timeout.cancel();
-        });
-        this.cancelAcks.set(id, { promise, resolve, timeout });
-        return promise;
     }
     async abortInflight(id, url, workerIndex) {
         const slot = this.workers[workerIndex];
@@ -300,7 +345,7 @@ class WorkerPool {
                 // Worker may be unavailable; failure is acceptable during abort
             }
         }
-        await this.waitForCancelAck(id);
+        await this.cancelAcks.wait(id, config.transform.cancelAckTimeoutMs);
         const taken = this.failTask(id, createAbortError(url, 'transform:signal-abort'));
         if (taken && slot)
             this.restartWorker(workerIndex, slot);
@@ -369,12 +414,12 @@ class WorkerPool {
         if (!message)
             return;
         if (message.type === 'cancelled') {
-            this.resolveCancelAck(message.id);
+            this.cancelAcks.resolve(message.id);
             return;
         }
         const inflightPeek = this.inflight.get(message.id);
         if (inflightPeek?.cancelPending) {
-            this.resolveCancelAck(message.id);
+            this.cancelAcks.resolve(message.id);
             return;
         }
         const inflight = this.takeInflight(message.id);
@@ -421,7 +466,10 @@ class WorkerPool {
         const slot = this.workers[workerIndex];
         if (!slot)
             return;
-        slot.busy = false;
+        if (slot.busy) {
+            slot.busy = false;
+            this.busyCount -= 1;
+        }
         slot.currentTaskId = null;
     }
     failTask(id, error) {
@@ -441,43 +489,31 @@ class WorkerPool {
         }
     }
     drainQueue() {
-        if (this.closed || this.getQueueDepth() === 0)
+        if (this.closed || this.queue.depth === 0)
             return;
         this.maybeScaleUp();
         for (let i = 0; i < this.workers.length; i += 1) {
             const slot = this.workers[i];
             if (slot && !slot.busy) {
                 this.dispatchFromQueue(i, slot);
-                if (this.getQueueDepth() === 0)
+                if (this.queue.depth === 0)
                     return;
             }
         }
-        if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
+        if (this.workers.length < this.capacity && this.queue.depth > 0) {
             const workerIndex = this.workers.length;
             const slot = this.spawnWorker(workerIndex);
             this.workers.push(slot);
             this.dispatchFromQueue(workerIndex, slot);
-            if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
+            if (this.workers.length < this.capacity && this.queue.depth > 0) {
                 setImmediate(() => {
                     this.drainQueue();
                 });
             }
         }
     }
-    takeNextQueuedTask() {
-        while (this.queueHead < this.queue.length) {
-            const task = this.queue[this.queueHead];
-            this.queueHead += 1;
-            if (task) {
-                this.maybeCompactQueue();
-                return task;
-            }
-        }
-        this.maybeCompactQueue();
-        return null;
-    }
     dispatchFromQueue(workerIndex, slot) {
-        const task = this.takeNextQueuedTask();
+        const task = this.queue.dequeue();
         if (!task)
             return;
         if (this.closed) {
@@ -496,6 +532,11 @@ class WorkerPool {
         }
         slot.busy = true;
         slot.currentTaskId = task.id;
+        this.busyCount += 1;
+        const timeout = this.registerInflight(task, workerIndex, slot);
+        this.sendToWorker(task, slot, workerIndex, timeout);
+    }
+    registerInflight(task, workerIndex, slot) {
         const timeout = createUnrefTimeout(this.timeoutMs, null);
         void timeout.promise
             .then(() => {
@@ -529,6 +570,9 @@ class WorkerPool {
             context: task.context,
             cancelPending: false,
         });
+        return timeout;
+    }
+    sendToWorker(task, slot, workerIndex, timeout) {
         try {
             const { message, transferList } = buildWorkerDispatchPayload(task);
             slot.worker.postMessage(message, transferList);
@@ -554,23 +598,6 @@ class WorkerPool {
             context.dispose();
         }
     }
-    findQueuedIndex(id) {
-        for (let i = this.queueHead; i < this.queue.length; i += 1) {
-            const task = this.queue[i];
-            if (task?.id === id)
-                return i;
-        }
-        return null;
-    }
-    maybeCompactQueue() {
-        if (this.queueHead === 0)
-            return;
-        if (this.queueHead >= this.queue.length ||
-            (this.queueHead > 1024 && this.queueHead > this.queue.length / 2)) {
-            this.queue.splice(0, this.queueHead);
-            this.queueHead = 0;
-        }
-    }
 }
 // Pool singleton management
 let workerPool = null;
@@ -595,23 +622,26 @@ export async function shutdownWorkerPool() {
     workerPool = null;
 }
 // Worker thread message handling
-if (!isMainThread && parentPort) {
-    const port = parentPort;
-    const onMessage = createTransformMessageHandler({
-        sendMessage: (message) => {
-            port.postMessage(message);
-        },
-        runTransform: transformHtmlToMarkdownInProcess,
-    });
-    port.on('message', onMessage);
-}
-else if (process.send) {
-    const send = process.send.bind(process);
-    const onMessage = createTransformMessageHandler({
-        sendMessage: (message) => {
-            send(message);
-        },
-        runTransform: transformHtmlToMarkdownInProcess,
-    });
-    process.on('message', onMessage);
+function bootstrapWorkerThread() {
+    if (!isMainThread && parentPort) {
+        const port = parentPort;
+        const onMessage = createTransformMessageHandler({
+            sendMessage: (message) => {
+                port.postMessage(message);
+            },
+            runTransform: transformHtmlToMarkdownInProcess,
+        });
+        port.on('message', onMessage);
+    }
+    else if (process.send) {
+        const send = process.send.bind(process);
+        const onMessage = createTransformMessageHandler({
+            sendMessage: (message) => {
+                send(message);
+            },
+            runTransform: transformHtmlToMarkdownInProcess,
+        });
+        process.on('message', onMessage);
+    }
 }
+bootstrapWorkerThread();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.9.1",
+  "version": "1.9.2",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
   "type": "module",
@@ -74,6 +74,7 @@
     "@mozilla/readability": "^0.6.0",
     "linkedom": "^0.18.12",
     "node-html-markdown": "^2.0.0",
+    "ts-morph": "^27.0.2",
     "undici": "^7.24.1",
     "zod": "^4.3.6"
   },