@j0hanz/fetch-url-mcp 1.12.7 → 1.12.8

package/dist/transform/worker-pool.js

@@ -1,725 +0,0 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { availableParallelism } from 'node:os';
-import process from 'node:process';
-import { isSharedArrayBuffer } from 'node:util/types';
-import { isMainThread, isMarkedAsUntransferable, parentPort, Worker, } from 'node:worker_threads';
-import { z } from 'zod';
-import { config, logDebug, logInfo, logWarn } from '../lib/core.js';
-import { SystemErrors } from '../lib/error-codes.js';
-import { Loggers } from '../lib/logger-names.js';
-import { createAbortError, createUnrefTimeout, FetchError, getErrorMessage, } from '../lib/utils.js';
-import { formatZodError } from '../lib/zod.js';
-import { extractedMetadataSchema } from '../schemas.js';
-import { createTransformMessageHandler } from './shared.js';
-import { transformHtmlToMarkdownInProcess } from './transform.js';
-// Worker message validation
-const workerResultPayloadSchema = z.strictObject({
-    markdown: z.string(),
-    title: z.string().optional(),
-    metadata: extractedMetadataSchema.optional(),
-    truncated: z.boolean(),
-});
-const workerErrorPayloadSchema = z.strictObject({
-    name: z.string(),
-    message: z.string(),
-    url: z.string(),
-    statusCode: z.number().int().optional(),
-    details: z.record(z.string(), z.unknown()).optional(),
-});
-const workerResponseSchema = z.discriminatedUnion('type', [
-    z.strictObject({
-        type: z.literal('result'),
-        id: z.string(),
-        result: workerResultPayloadSchema,
-    }),
-    z.strictObject({
-        type: z.literal('error'),
-        id: z.string(),
-        error: workerErrorPayloadSchema,
-    }),
-    z.strictObject({
-        type: z.literal('cancelled'),
-        id: z.string(),
-    }),
-]);
-function createTaskContext() {
-    const runWithStore = AsyncLocalStorage.snapshot();
-    return {
-        run: (fn) => {
-            runWithStore(fn);
-        },
-    };
-}
-function ensureTightBuffer(buffer) {
-    if (buffer.byteOffset === 0 &&
-        buffer.byteLength === buffer.buffer.byteLength) {
-        return buffer;
-    }
-    const copy = new Uint8Array(buffer);
-    return copy;
-}
-function getTransferableBuffer(buffer) {
-    const backingBuffer = buffer.buffer;
-    if (isSharedArrayBuffer(backingBuffer))
-        return null;
-    if (!(backingBuffer instanceof ArrayBuffer))
-        return null;
-    return isMarkedAsUntransferable(backingBuffer) ? null : backingBuffer;
-}
-function buildWorkerDispatchPayload(task) {
-    const message = {
-        type: 'transform',
-        id: task.id,
-        url: task.url,
-        includeMetadataFooter: task.includeMetadataFooter,
-        ...(task.inputTruncated ? { inputTruncated: true } : {}),
-    };
-    if (!task.htmlBuffer) {
-        message.html = task.html;
-        return { message };
-    }
-    const htmlBuffer = ensureTightBuffer(task.htmlBuffer);
-    message.htmlBuffer = htmlBuffer;
-    if (task.encoding)
-        message.encoding = task.encoding;
-    const transferableBuffer = getTransferableBuffer(htmlBuffer);
-    return transferableBuffer
-        ? { message, transferList: [transferableBuffer] }
-        : { message };
-}
-// Pool sizing & constants
-// Core tuning: ~half of available CPUs as baseline, capped by config limits.
-const POOL_MIN_WORKERS = Math.max(2, Math.min(4, Math.floor(availableParallelism() / 2)));
-const POOL_MAX_WORKERS = config.transform.maxWorkerScale;
-const POOL_SCALE_THRESHOLD = 0.5;
-const WORKER_NAME_PREFIX = 'fetch-url-mcp-transform';
-const DEFAULT_TIMEOUT_MS = config.transform.timeoutMs;
-const TRANSFORM_WORKER_PATH = new URL(import.meta.url);
-const COMPACTION_HEAD_THRESHOLD = 1024;
-const QUEUE_CAPACITY_MULTIPLIER = 4;
-const HTTP_SERVICE_UNAVAILABLE = 503;
-const HTTP_GATEWAY_TIMEOUT = 504;
-// TaskQueue — array-deque with auto-compaction
-class TaskQueue {
-    items = [];
-    head = 0;
-    activeCount = 0;
-    get depth() {
-        return this.activeCount;
-    }
-    enqueue(item) {
-        this.items.push(item);
-        this.activeCount += 1;
-    }
-    dequeue() {
-        let found = null;
-        while (this.head < this.items.length) {
-            const item = this.items[this.head];
-            this.head += 1;
-            if (item) {
-                this.activeCount -= 1;
-                found = item;
-                break;
-            }
-        }
-        this.compact();
-        return found;
-    }
-    removeById(id) {
-        for (let i = this.head; i < this.items.length; i += 1) {
-            const item = this.items[i];
-            if (item?.id === id) {
-                this.items[i] = undefined;
-                this.activeCount -= 1;
-                this.compact();
-                return item;
-            }
-        }
-        return undefined;
-    }
-    drain(callback) {
-        for (let i = this.head; i < this.items.length; i += 1) {
-            const item = this.items[i];
-            if (item)
-                callback(item);
-        }
-        this.items.length = 0;
-        this.head = 0;
-        this.activeCount = 0;
-    }
-    compact() {
-        if (this.head === 0)
-            return;
-        if (this.head >= this.items.length ||
-            (this.head > COMPACTION_HEAD_THRESHOLD &&
-                this.head > this.items.length / 2)) {
-            this.items.splice(0, this.head);
-            this.head = 0;
-        }
-    }
-}
-// CancelAckTracker — isolates the cancel-acknowledgement protocol
-class CancelAckTracker {
-    pending = new Map();
-    earlyResolutions = new Set();
-    resolve(id) {
-        const entry = this.pending.get(id);
-        if (!entry) {
-            this.earlyResolutions.add(id);
-            return;
-        }
-        entry.timeout.cancel();
-        entry.resolve();
-    }
-    wait(id, timeoutMs) {
-        if (this.earlyResolutions.has(id)) {
-            this.earlyResolutions.delete(id);
-            return Promise.resolve();
-        }
-        const existing = this.pending.get(id);
-        if (existing)
-            return existing.promise;
-        const timeout = createUnrefTimeout(timeoutMs, undefined);
-        const { promise: racePromise, resolve } = Promise.withResolvers();
-        const promise = Promise.race([racePromise, timeout.promise])
-            .finally(() => {
-            this.pending.delete(id);
-            timeout.cancel();
-        })
-            .then(() => {
-            return;
-        });
-        this.pending.set(id, {
-            promise,
-            resolve: resolve,
-            timeout,
-        });
-        return promise;
-    }
-    dispose() {
-        for (const entry of this.pending.values()) {
-            entry.timeout.cancel();
-            entry.resolve();
-        }
-        this.pending.clear();
-        this.earlyResolutions.clear();
-    }
-}
-// WorkerPool
-class WorkerPool {
-    static CLOSED_MESSAGE = 'Transform worker pool closed';
-    workers = [];
-    capacity;
-    minCapacity = POOL_MIN_WORKERS;
-    maxCapacity = POOL_MAX_WORKERS;
-    queue = new TaskQueue();
-    inflight = new Map();
-    cancelAcks = new CancelAckTracker();
-    timeoutMs;
-    queueMax;
-    closed = false;
-    taskIdSeq = 0;
-    busyCount = 0;
-    draining = false;
-    restartBackoff = new Map();
-    constructor(size, timeoutMs) {
-        this.capacity =
-            size === 0
-                ? 0
-                : Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
-        this.timeoutMs = timeoutMs;
-        this.queueMax = this.maxCapacity * QUEUE_CAPACITY_MULTIPLIER;
-    }
-    async transform(htmlOrBuffer, url, options) {
-        this.ensureOpen();
-        if (options.signal?.aborted)
-            throw createAbortError(url, 'transform:enqueue');
-        if (this.queue.depth >= this.queueMax) {
-            logWarn('Transform worker queue capacity reached', {
-                queueDepth: this.queue.depth,
-                queueMax: this.queueMax,
-                activeWorkers: this.busyCount,
-                capacity: this.capacity,
-                url,
-            }, Loggers.LOG_TRANSFORM);
-            const error = new FetchError('Transform worker queue is full', url, HTTP_SERVICE_UNAVAILABLE, {
-                reason: SystemErrors.QUEUE_FULL,
-                stage: 'transform:enqueue',
-            });
-            throw error;
-        }
-        const { promise, resolve, reject } = Promise.withResolvers();
-        const task = this.createPendingTask(htmlOrBuffer, url, options, resolve, reject);
-        this.queue.enqueue(task);
-        this.drainQueue();
-        return promise;
-    }
-    getQueueDepth() {
-        return this.queue.depth;
-    }
-    getActiveWorkers() {
-        return this.busyCount;
-    }
-    getCapacity() {
-        return this.capacity;
-    }
-    resize(size) {
-        const newCapacity = Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
-        if (newCapacity === this.capacity)
-            return;
-        this.capacity = newCapacity;
-        this.drainQueue();
-    }
-    async close() {
-        if (this.closed)
-            return;
-        this.closed = true;
-        logInfo('Shutting down transform worker pool', {
-            workers: this.workers.length,
-            activeWorkers: this.busyCount,
-            queueDepth: this.queue.depth,
-            inflight: this.inflight.size,
-        }, Loggers.LOG_TRANSFORM);
-        const terminations = this.workers
-            .map((slot) => slot?.worker.terminate().catch(() => undefined))
-            .filter((p) => p !== undefined);
-        this.workers.fill(undefined);
-        this.workers.length = 0;
-        this.busyCount = 0;
-        this.cancelAcks.dispose();
-        for (const id of Array.from(this.inflight.keys())) {
-            const inflight = this.takeInflight(id);
-            if (!inflight)
-                continue;
-            this.abortAndCleanTask(inflight, new Error(WorkerPool.CLOSED_MESSAGE));
-        }
-        this.queue.drain((task) => {
-            this.abortAndCleanTask(task, new Error(WorkerPool.CLOSED_MESSAGE));
-        });
-        await Promise.allSettled(terminations);
-    }
-    ensureOpen() {
-        if (this.closed)
-            throw Error(WorkerPool.CLOSED_MESSAGE);
-    }
-    createPendingTask(htmlOrBuffer, url, options, resolve, reject) {
-        const id = (this.taskIdSeq++).toString(36);
-        // Preserve request context for resolve/reject even when callbacks fire
-        // from worker thread events.
-        const context = createTaskContext();
-        let abortListener;
-        if (options.signal) {
-            abortListener = () => {
-                this.onAbortSignal(id, url, context, reject);
-            };
-            options.signal.addEventListener('abort', abortListener, { once: true });
-        }
-        const task = {
-            id,
-            url,
-            includeMetadataFooter: options.includeMetadataFooter,
-            ...(options.inputTruncated
-                ? { inputTruncated: options.inputTruncated }
-                : {}),
-            signal: options.signal,
-            abortListener,
-            context,
-            resolve,
-            reject,
-        };
-        if (typeof htmlOrBuffer === 'string') {
-            task.html = htmlOrBuffer;
-        }
-        else {
-            task.htmlBuffer = htmlOrBuffer;
-            if (options.encoding) {
-                task.encoding = options.encoding;
-            }
-        }
-        return task;
-    }
-    onAbortSignal(id, url, context, reject) {
-        if (this.closed) {
-            this.finalizeTask(context, () => {
-                reject(new Error(WorkerPool.CLOSED_MESSAGE));
-            });
-            return;
-        }
-        const inflight = this.inflight.get(id);
-        if (inflight) {
-            void this.abortInflight(id, url, inflight.workerIndex);
-            return;
-        }
-        const queuedTask = this.queue.removeById(id);
-        if (queuedTask) {
-            this.abortAndCleanTask(queuedTask, createAbortError(url, 'transform:queued-abort'));
-        }
-    }
-    async abortInflight(id, url, workerIndex) {
-        const slot = this.workers[workerIndex];
-        const inflight = this.inflight.get(id);
-        if (inflight) {
-            inflight.cancelPending = true;
-        }
-        if (slot) {
-            try {
-                slot.worker.postMessage({ type: 'cancel', id });
-            }
-            catch {
-                // Worker may be unavailable; failure is acceptable during abort
-            }
-        }
-        await this.cancelAcks.wait(id, config.transform.cancelAckTimeoutMs);
-        const taken = this.failTask(id, createAbortError(url, 'transform:signal-abort'));
-        if (taken && slot)
-            this.restartWorker(workerIndex, slot);
-    }
-    clearAbortListener(signal, listener) {
-        if (!signal || !listener)
-            return;
-        try {
-            signal.removeEventListener('abort', listener);
-        }
-        catch {
-            // Defensive: removeEventListener should not throw, but handle edge cases
-        }
-    }
-    spawnWorker(workerIndex) {
-        const name = `${WORKER_NAME_PREFIX}-${workerIndex + 1}`;
-        const resourceLimits = config.transform.workerResourceLimits;
-        const worker = new Worker(TRANSFORM_WORKER_PATH, {
-            name,
-            ...(resourceLimits ? { resourceLimits } : {}),
-        });
-        logDebug('Spawned transform worker', {
-            workerIndex,
-            workerName: name,
-        }, Loggers.LOG_TRANSFORM);
-        worker.unref();
-        worker.on('message', (raw) => {
-            this.onWorkerMessage(workerIndex, raw);
-        });
-        worker.on('error', (error) => {
-            this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
-        });
-        worker.on('messageerror', (error) => {
-            this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
-        });
-        worker.on('exit', (code) => {
-            this.onWorkerBroken(workerIndex, `Transform worker exited (code ${code ?? 'unknown'})`);
-        });
-        return { worker, busy: false, currentTaskId: null, name };
-    }
-    onWorkerBroken(workerIndex, message) {
-        if (this.closed)
-            return;
-        const slot = this.workers[workerIndex];
-        if (!slot)
-            return;
-        logWarn('Transform worker unavailable; restarting', {
-            reason: message,
-            workerIndex,
-            workerName: slot.name,
-            threadId: slot.worker.threadId,
-        }, Loggers.LOG_TRANSFORM);
-        if (slot.busy && slot.currentTaskId) {
-            try {
-                this.failTask(slot.currentTaskId, new FetchError(message, '', HTTP_SERVICE_UNAVAILABLE, {
-                    reason: 'worker_exit',
-                }));
-            }
-            catch {
-                this.markIdle(workerIndex);
-            }
-        }
-        this.restartWorker(workerIndex, slot);
-    }
-    restartWorker(workerIndex, slot) {
-        if (this.closed)
-            return;
-        const target = slot ?? this.workers[workerIndex];
-        if (target) {
-            target.worker.terminate().catch(() => undefined);
-        }
-        const attempts = this.restartBackoff.get(workerIndex) ?? 0;
-        this.restartBackoff.set(workerIndex, attempts + 1);
-        if (attempts > 0) {
-            const delayMs = Math.min(1000 * 2 ** (attempts - 1), 30_000);
-            logWarn('Scheduling transform worker restart with backoff', {
-                workerIndex,
-                delayMs,
-                attempt: attempts + 1,
-            }, Loggers.LOG_TRANSFORM);
-            setTimeout(() => {
-                if (this.closed)
-                    return;
-                this.workers[workerIndex] = this.spawnWorker(workerIndex);
-                this.drainQueue();
-            }, delayMs).unref();
-            return;
-        }
-        this.workers[workerIndex] = this.spawnWorker(workerIndex);
-        this.drainQueue();
-    }
-    onWorkerMessage(workerIndex, raw) {
-        const parsed = workerResponseSchema.safeParse(raw);
-        if (!parsed.success) {
-            this.onWorkerBroken(workerIndex, `Transform worker sent invalid message: ${formatZodError(parsed.error)}`);
-            return;
-        }
-        const message = parsed.data;
-        if (message.type === 'cancelled') {
-            this.cancelAcks.resolve(message.id);
-            return;
-        }
-        const inflightPeek = this.inflight.get(message.id);
-        if (inflightPeek?.cancelPending) {
-            this.cancelAcks.resolve(message.id);
-            return;
-        }
-        const inflight = this.takeInflight(message.id);
-        if (!inflight)
-            return;
-        this.restartBackoff.delete(workerIndex);
-        this.markIdle(workerIndex);
-        this.resolveWorkerResult(inflight, message);
-        this.drainQueue();
-    }
-    resolveWorkerResult(inflight, message) {
-        this.finalizeTask(inflight.context, () => {
-            if (message.type === 'result') {
-                inflight.resolve({
-                    markdown: message.result.markdown,
-                    truncated: message.result.truncated,
-                    title: message.result.title,
-                    ...(message.result.metadata
-                        ? { metadata: message.result.metadata }
-                        : {}),
-                });
-            }
-            else {
-                const err = message.error;
-                inflight.reject(err.name === 'FetchError'
-                    ? new FetchError(err.message, err.url, err.statusCode, err.details ?? {})
-                    : new Error(err.message));
-            }
-        });
-    }
-    takeInflight(id) {
-        const inflight = this.inflight.get(id);
-        if (!inflight)
-            return null;
-        inflight.timeout.cancel();
-        this.clearAbortListener(inflight.signal, inflight.abortListener);
-        this.inflight.delete(id);
-        return inflight;
-    }
-    markBusy(workerIndex, taskId) {
-        const slot = this.workers[workerIndex];
-        if (!slot)
-            return;
-        if (!slot.busy) {
-            slot.busy = true;
-            this.busyCount += 1;
-        }
-        slot.currentTaskId = taskId;
-    }
-    markIdle(workerIndex) {
-        const slot = this.workers[workerIndex];
-        if (!slot)
-            return;
-        if (slot.busy) {
-            slot.busy = false;
-            this.busyCount -= 1;
-        }
-        slot.currentTaskId = null;
-    }
-    failTask(id, error) {
-        const inflight = this.takeInflight(id);
-        if (!inflight)
-            return false;
-        this.abortAndCleanTask(inflight, error);
-        this.markIdle(inflight.workerIndex);
-        return true;
-    }
-    maybeScaleUp() {
-        if (this.getQueueDepth() > this.capacity * POOL_SCALE_THRESHOLD &&
-            this.capacity < this.maxCapacity) {
-            const previousCapacity = this.capacity;
-            this.capacity += 1;
-            logInfo('Scaled transform worker pool', {
-                fromCapacity: previousCapacity,
-                toCapacity: this.capacity,
-                queueDepth: this.getQueueDepth(),
-            }, Loggers.LOG_TRANSFORM);
-        }
-    }
-    drainQueue() {
-        if (this.closed || this.queue.depth === 0 || this.draining)
-            return;
-        this.draining = true;
-        try {
-            this.maybeScaleUp();
-            for (let i = 0; i < this.workers.length; i += 1) {
-                const slot = this.workers[i];
-                if (slot && !slot.busy) {
-                    this.dispatchFromQueue(i, slot);
-                    if (this.queue.depth === 0)
-                        return;
-                }
-            }
-            if (this.workers.length < this.capacity && this.queue.depth > 0) {
-                const workerIndex = this.workers.length;
-                const slot = this.spawnWorker(workerIndex);
-                this.workers.push(slot);
-                this.dispatchFromQueue(workerIndex, slot);
-                if (this.workers.length < this.capacity && this.queue.depth > 0) {
-                    setImmediate(() => {
-                        this.drainQueue();
-                    });
-                }
-            }
-        }
-        finally {
-            this.draining = false;
-        }
-    }
-    dispatchFromQueue(workerIndex, slot) {
-        let task = this.queue.dequeue();
-        while (task) {
-            const currentTask = task;
-            if (this.closed) {
-                this.abortAndCleanTask(currentTask, new Error(WorkerPool.CLOSED_MESSAGE));
-                return;
-            }
-            if (currentTask.signal?.aborted) {
-                this.abortAndCleanTask(currentTask, createAbortError(currentTask.url, 'transform:dispatch'));
-                task = this.queue.dequeue();
-                continue;
-            }
-            break;
-        }
-        if (!task)
-            return;
-        this.markBusy(workerIndex, task.id);
-        const timeout = this.registerInflight(task, workerIndex, slot);
-        this.sendToWorker(task, slot, workerIndex, timeout);
-    }
-    registerInflight(task, workerIndex, slot) {
-        const timeout = createUnrefTimeout(this.timeoutMs, null);
-        void timeout.promise
-            .then(() => {
-            try {
-                slot.worker.postMessage({ type: 'cancel', id: task.id });
-            }
-            catch {
-                // Worker may be unavailable; proceed with timeout handling
-            }
-            const inflight = this.takeInflight(task.id);
-            if (!inflight)
-                return;
-            logWarn('Transform worker task timed out', {
-                taskId: task.id,
-                url: task.url,
-                workerIndex,
-                timeoutMs: this.timeoutMs,
-            }, Loggers.LOG_TRANSFORM);
-            this.abortAndCleanTask(inflight, new FetchError('Request timeout', task.url, HTTP_GATEWAY_TIMEOUT, {
-                reason: 'timeout',
-                stage: 'transform:worker-timeout',
-            }));
-            this.markIdle(workerIndex);
-            this.restartWorker(workerIndex, slot);
-        })
-            .catch((error) => {
-            this.failTask(task.id, error);
-        });
-        this.inflight.set(task.id, {
-            resolve: task.resolve,
-            reject: task.reject,
-            timeout,
-            signal: task.signal,
-            abortListener: task.abortListener,
-            workerIndex,
-            context: task.context,
-            cancelPending: false,
-        });
-        return timeout;
-    }
-    sendToWorker(task, slot, workerIndex, timeout) {
-        try {
-            const { message, transferList } = buildWorkerDispatchPayload(task);
-            slot.worker.postMessage(message, transferList);
-        }
-        catch (error) {
-            timeout.cancel();
-            this.inflight.delete(task.id);
-            this.markIdle(workerIndex);
-            this.abortAndCleanTask(task, error instanceof Error
-                ? error
-                : new Error('Failed to dispatch transform worker message'));
-            this.restartWorker(workerIndex, slot);
-        }
-    }
-    finalizeTask(context, fn) {
-        context.run(fn);
-    }
-    abortAndCleanTask(task, error) {
-        this.clearAbortListener(task.signal, task.abortListener);
-        this.finalizeTask(task.context, () => {
-            task.reject(error);
-        });
-    }
-}
-// Pool singleton management
-let workerPool = null;
-export function getOrCreateWorkerPool() {
-    const size = config.transform.maxWorkerScale === 0 ? 0 : POOL_MIN_WORKERS;
-    if (!workerPool) {
-        workerPool = new WorkerPool(size, DEFAULT_TIMEOUT_MS);
-        logInfo('Initialized transform worker pool', {
-            initialCapacity: workerPool.getCapacity(),
-            timeoutMs: DEFAULT_TIMEOUT_MS,
-        }, Loggers.LOG_TRANSFORM);
-    }
-    return workerPool;
-}
-export function getWorkerPoolStats() {
-    if (!workerPool)
-        return null;
-    return {
-        queueDepth: workerPool.getQueueDepth(),
-        activeWorkers: workerPool.getActiveWorkers(),
-        capacity: workerPool.getCapacity(),
-    };
-}
-export async function shutdownWorkerPool() {
-    if (!workerPool)
-        return;
-    await workerPool.close();
-    workerPool = null;
-}
-// Worker thread message handling
-function bootstrapWorkerThread() {
-    if (!isMainThread && parentPort) {
-        const port = parentPort;
-        const onMessage = createTransformMessageHandler({
-            sendMessage: (message) => {
-                port.postMessage(message);
-            },
-            runTransform: transformHtmlToMarkdownInProcess,
-        });
-        port.on('message', onMessage);
-    }
-    else if (process.send) {
-        const send = process.send.bind(process);
-        const onMessage = createTransformMessageHandler({
-            sendMessage: (message) => {
-                send(message);
-            },
-            runTransform: transformHtmlToMarkdownInProcess,
-        });
-        process.on('message', onMessage);
-    }
-}
-bootstrapWorkerThread();