@j0hanz/fetch-url-mcp 1.2.0 → 1.3.1

package/dist/{transform.js → transform/transform.js}

@@ -1,24 +1,18 @@
-import { AsyncLocalStorage, AsyncResource } from 'node:async_hooks';
 import { Buffer } from 'node:buffer';
-import { fork } from 'node:child_process';
 import diagnosticsChannel from 'node:diagnostics_channel';
-import { availableParallelism } from 'node:os';
 import { performance } from 'node:perf_hooks';
-import { fileURLToPath } from 'node:url';
-import { isSharedArrayBuffer } from 'node:util/types';
-import { Worker, } from 'node:worker_threads';
+import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
 import { NodeHtmlMarkdown, } from 'node-html-markdown';
-import { isProbablyReaderable, Readability } from '@mozilla/readability';
-import { config } from './config.js';
-import { removeNoiseFromHtml } from './dom-noise-removal.js';
-import { FetchError, getErrorMessage } from './errors.js';
-import { isRawTextContentUrl } from './fetch.js';
-import { detectLanguageFromCode, resolveLanguageFromAttributes, } from './language-detection.js';
-import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from './markdown-cleanup.js';
-import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from './observability.js';
-import { createUnrefTimeout } from './timer-utils.js';
-import { isLikeNode, isObject } from './type-guards.js';
+import { config } from '../config.js';
+import { removeNoiseFromHtml } from '../dom-noise-removal.js';
+import { FetchError, getErrorMessage } from '../errors.js';
+import { isRawTextContentUrl } from '../fetch.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../language-detection.js';
+import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../markdown-cleanup.js';
+import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../observability.js';
+import { isLikeNode, isObject } from '../type-guards.js';
+import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
 const utf8Decoder = new TextDecoder('utf-8');
 function decodeInput(input, encoding) {
     if (typeof input === 'string')
@@ -84,7 +78,7 @@ function buildTransformSignal(signal) {
 class StageTracker {
     channel = diagnosticsChannel.channel('fetch-url-mcp.transform');
     start(url, stage, budget) {
-        if (!this.channel.hasSubscribers && !budget)
+        if (this.shouldSkipTracking(budget))
             return null;
         const remainingBudgetMs = budget
             ? budget.totalBudgetMs - budget.elapsedMs
@@ -136,7 +130,7 @@ class StageTracker {
         return durationMs;
     }
     run(url, stage, fn, budget) {
-        if (!this.channel.hasSubscribers && !budget) {
+        if (this.shouldSkipTracking(budget)) {
             return fn();
         }
         if (budget && budget.elapsedMs >= budget.totalBudgetMs) {
@@ -156,6 +150,9 @@ class StageTracker {
         }
     }
     async runAsync(url, stage, fn) {
+        if (this.shouldSkipTracking()) {
+            return fn();
+        }
         const ctx = this.start(url, stage);
         try {
             return await fn();
@@ -164,6 +161,9 @@ class StageTracker {
             this.end(ctx);
         }
     }
+    shouldSkipTracking(budget) {
+        return !this.channel.hasSubscribers && !budget;
+    }
     publish(event) {
         if (!this.channel.hasSubscribers)
             return;
@@ -219,26 +219,27 @@ function truncateHtml(html, inputTruncated = false) {
     const maxSize = config.constants.maxHtmlSize;
     if (maxSize <= 0)
         return { html, truncated: false };
-    // Fast path: V8 optimized byte length check (no allocation)
-    const byteLength = Buffer.byteLength(html, 'utf8');
-    if (byteLength <= maxSize && !inputTruncated)
-        return { html, truncated: false };
+    if (html.length <= maxSize) {
+        const byteLength = getUtf8ByteLength(html);
+        if (byteLength <= maxSize && !inputTruncated)
+            return { html, truncated: false };
+    }
     const sliced = html.slice(0, maxSize);
-    if (Buffer.byteLength(sliced, 'utf8') <= maxSize) {
+    if (getUtf8ByteLength(sliced) <= maxSize) {
         return { html: trimDanglingTagFragment(sliced), truncated: true };
     }
     const htmlBuffer = Buffer.from(sliced, 'utf8');
     const content = trimDanglingTagFragment(trimUtf8Buffer(htmlBuffer, maxSize).toString('utf8'));
     logWarn('HTML content exceeds maximum size, truncating', {
-        size: byteLength,
+        size: getUtf8ByteLength(html),
         maxSize,
-        truncatedSize: Buffer.byteLength(content, 'utf8'),
+        truncatedSize: getUtf8ByteLength(content),
     });
     return { html: content, truncated: true };
 }
 function willTruncate(html) {
     const maxSize = config.constants.maxHtmlSize;
-    return maxSize > 0 && getUtf8ByteLength(html) > maxSize;
+    return (maxSize > 0 && (html.length > maxSize || getUtf8ByteLength(html) > maxSize));
 }
 const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
 const MAX_HEAD_SCAN_LENGTH = 50_000;
@@ -694,11 +695,12 @@ function buildInlineCodeTranslator() {
     };
 }
 function buildCodeTranslator(ctx) {
+    const inlineCodeTranslator = buildInlineCodeTranslator();
     if (!isObject(ctx))
-        return buildInlineCodeTranslator();
+        return inlineCodeTranslator;
     const { parent } = ctx;
     if (!isCodeBlock(parent))
-        return buildInlineCodeTranslator();
+        return inlineCodeTranslator;
     return { noEscape: true, preserveWhitespace: true };
 }
 function extractFirstSrcsetUrl(srcset) {
@@ -713,14 +715,17 @@ const LAZY_SRC_ATTRIBUTES = [
     'data-original',
     'data-srcset',
 ];
+function isDataUri(value) {
+    return value.startsWith('data:');
+}
 function extractNonDataSrcsetUrl(value) {
     const url = extractFirstSrcsetUrl(value);
-    return url && !url.startsWith('data:') ? url : undefined;
+    return url && !isDataUri(url) ? url : undefined;
 }
 function resolveLazySrc(getAttribute) {
     for (const attr of LAZY_SRC_ATTRIBUTES) {
         const lazy = getAttribute(attr);
-        if (!lazy || lazy.startsWith('data:'))
+        if (!lazy || isDataUri(lazy))
             continue;
         if (attr === 'data-srcset') {
             const url = extractNonDataSrcsetUrl(lazy);
@@ -736,7 +741,7 @@ function resolveImageSrc(getAttribute) {
     if (!getAttribute)
         return '';
     const srcRaw = getAttribute('src') ?? '';
-    if (srcRaw && !srcRaw.startsWith('data:'))
+    if (srcRaw && !isDataUri(srcRaw))
         return srcRaw;
     // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
     const lazySrc = resolveLazySrc(getAttribute);
@@ -750,7 +755,7 @@ function resolveImageSrc(getAttribute) {
             return url;
     }
     // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
-    if (srcRaw.startsWith('data:'))
+    if (isDataUri(srcRaw))
         return '[data URI removed]';
     return '';
 }
@@ -1099,7 +1104,7 @@ function resolveRelativeUrlsInSegment(markdown, baseUrl, origin) {
     }
     return output;
 }
-function resolveRelativeUrls(markdown, baseUrl) {
+function resolveRelativeUrls(markdown, baseUrl, signal) {
     let origin;
     try {
         ({ origin } = new URL(baseUrl));
@@ -1109,7 +1114,6 @@ function resolveRelativeUrls(markdown, baseUrl) {
     }
     if (!markdown)
         return markdown;
-    const lines = markdown.split('\n');
     let output = '';
     let buffer = '';
     let fenceMarker = null;
@@ -1119,26 +1123,51 @@ function resolveRelativeUrls(markdown, baseUrl) {
         output += resolveRelativeUrlsInSegment(buffer, baseUrl, origin);
         buffer = '';
     };
-    for (let i = 0; i < lines.length; i += 1) {
-        const line = lines[i] ?? '';
+    const len = markdown.length;
+    let lastIndex = 0;
+    let lineCount = 0;
+    while (lastIndex < len) {
+        if (++lineCount % 500 === 0 && signal?.aborted) {
+            throw new Error('Transform aborted during URL resolution');
+        }
+        let nextIndex = markdown.indexOf('\n', lastIndex);
+        let line;
+        let lineWithNewline;
+        if (nextIndex === -1) {
+            line = markdown.slice(lastIndex);
+            lineWithNewline = line;
+            nextIndex = len;
+        }
+        else {
+            if (nextIndex > lastIndex && markdown.charCodeAt(nextIndex - 1) === 13) {
+                line = markdown.slice(lastIndex, nextIndex - 1);
+            }
+            else {
+                line = markdown.slice(lastIndex, nextIndex);
+            }
+            lineWithNewline = markdown.slice(lastIndex, nextIndex + 1);
+            nextIndex++; // Skip \n
+        }
         const trimmed = line.trimStart();
-        const lineWithNewline = i < lines.length - 1 ? `${line}\n` : line;
         if (fenceMarker) {
             output += lineWithNewline;
             if (trimmed.startsWith(fenceMarker) &&
                 trimmed.slice(fenceMarker.length).trim() === '') {
                 fenceMarker = null;
             }
-            continue;
         }
-        const fenceMatch = FENCE_LINE_PATTERN.exec(line);
-        if (fenceMatch?.[1]) {
-            flushBuffer();
-            output += lineWithNewline;
-            fenceMarker = fenceMatch[1];
-            continue;
+        else {
+            const fenceMatch = FENCE_LINE_PATTERN.exec(line);
+            if (fenceMatch?.[1]) {
+                flushBuffer();
+                output += lineWithNewline;
+                fenceMarker = fenceMatch[1];
+            }
+            else {
+                buffer += lineWithNewline;
+            }
         }
-        buffer += lineWithNewline;
+        lastIndex = nextIndex;
     }
     flushBuffer();
     return output;
@@ -1148,12 +1177,12 @@ function translateHtmlToMarkdown(params) {
     abortPolicy.throwIfAborted(signal, url, 'markdown:begin');
     const cleanedHtml = skipNoiseRemoval
         ? html
-        : stageTracker.run(url, 'markdown:noise', () => removeNoiseFromHtml(html, document, url));
+        : stageTracker.run(url, 'markdown:noise', () => removeNoiseFromHtml(html, document, url, signal));
     abortPolicy.throwIfAborted(signal, url, 'markdown:cleaned');
     const content = stageTracker.run(url, 'markdown:translate', () => translateHtmlFragmentToMarkdown(cleanedHtml));
     abortPolicy.throwIfAborted(signal, url, 'markdown:translated');
     const cleaned = cleanupMarkdownArtifacts(content, signal ? { signal, url } : { url });
-    return url ? resolveRelativeUrls(cleaned, url) : cleaned;
+    return url ? resolveRelativeUrls(cleaned, url, signal) : cleaned;
 }
 function appendMetadataFooter(content, metadata, url) {
     const footer = buildMetadataFooter(metadata, url);
@@ -1448,13 +1477,13 @@ function shouldUseArticleContent(article, originalHtmlOrDocument) {
     return !hasTruncatedSentences(article.textContent);
 }
 function buildContentSource(params) {
-    const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, truncated, skipNoiseRemoval, } = params;
+    const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, truncated, skipNoiseRemoval, signal, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
     if (useArticleContent && article) {
         // Readability output can still be noisy (unless user requested skip).
         const cleanedArticleHtml = skipNoiseRemoval
             ? article.content
-            : removeNoiseFromHtml(article.content, undefined, url);
+            : removeNoiseFromHtml(article.content, undefined, url, signal);
         return {
             sourceHtml: cleanedArticleHtml,
             title: article.title,
@@ -1468,7 +1497,7 @@ function buildContentSource(params) {
     if (document) {
         const cleanedHtml = skipNoiseRemoval
             ? html
-            : removeNoiseFromHtml(html, document, url);
+            : removeNoiseFromHtml(html, document, url, signal);
         const contentRoot = findContentRoot(document);
         if (contentRoot) {
             return {
@@ -1521,6 +1550,7 @@ function resolveContentSource(params) {
         document,
         truncated: truncated ?? false,
         ...(params.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
+        ...(params.signal ? { signal: params.signal } : {}),
     });
 }
 function buildMarkdownFromContext(context, url, signal) {
@@ -1603,727 +1633,6 @@ export function transformHtmlToMarkdownInProcess(html, url, options) {
         endTotalTransformStage(totalStage, completed);
     }
 }
-function isWorkerResultPayload(value) {
-    if (!isObject(value))
-        return false;
-    const { markdown, metadata, title, truncated } = value;
-    const isMetadataObject = metadata === undefined || isObject(metadata);
-    if (!isMetadataObject)
-        return false;
-    if (metadata && !isExtractedMetadataPayload(metadata)) {
-        return false;
-    }
-    return (typeof markdown === 'string' &&
-        typeof truncated === 'boolean' &&
-        (title === undefined || typeof title === 'string'));
-}
-function isExtractedMetadataPayload(value) {
-    if (!isObject(value))
-        return false;
-    const { author, description, favicon, image, modifiedAt, publishedAt, title, } = value;
-    return ((title === undefined || typeof title === 'string') &&
-        (description === undefined || typeof description === 'string') &&
-        (author === undefined || typeof author === 'string') &&
-        (image === undefined || typeof image === 'string') &&
-        (favicon === undefined || typeof favicon === 'string') &&
-        (publishedAt === undefined || typeof publishedAt === 'string') &&
-        (modifiedAt === undefined || typeof modifiedAt === 'string'));
-}
-function isWorkerErrorPayload(value) {
-    if (!isObject(value))
-        return false;
-    const { details, message, name, statusCode, url } = value;
-    return (typeof name === 'string' &&
-        typeof message === 'string' &&
-        typeof url === 'string' &&
-        (statusCode === undefined || typeof statusCode === 'number') &&
-        (details === undefined || isObject(details)));
-}
-function isWorkerResponse(raw) {
-    if (!isObject(raw))
-        return false;
-    if (typeof raw['id'] !== 'string')
-        return false;
-    if (raw['type'] === 'result') {
-        return isWorkerResultPayload(raw['result']);
-    }
-    if (raw['type'] === 'error') {
-        return isWorkerErrorPayload(raw['error']);
-    }
-    if (raw['type'] === 'cancelled') {
-        return true;
-    }
-    return false;
-}
-function createTaskContext() {
-    const runWithStore = AsyncLocalStorage.snapshot();
-    const asyncResource = new AsyncResource('fetch-url-mcp.transform.task');
-    let disposed = false;
-    return {
-        run: (fn) => {
-            runWithStore(() => {
-                asyncResource.runInAsyncScope(fn);
-            });
-        },
-        dispose: () => {
-            if (disposed)
-                return;
-            disposed = true;
-            asyncResource.emitDestroy();
-        },
-    };
-}
-function buildWorkerDispatchPayload(task, supportsTransferList) {
-    const message = {
-        type: 'transform',
-        id: task.id,
-        url: task.url,
-        includeMetadata: task.includeMetadata,
-        ...(task.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
-        ...(task.inputTruncated ? { inputTruncated: true } : {}),
-    };
-    if (!task.htmlBuffer) {
-        message.html = task.html;
-        return { message };
-    }
-    const htmlBuffer = ensureTightBuffer(task.htmlBuffer);
-    if (!supportsTransferList) {
-        message.htmlBuffer = htmlBuffer;
-        if (task.encoding)
-            message.encoding = task.encoding;
-        return { message };
-    }
-    const transferableHtmlBuffer = Uint8Array.from(htmlBuffer);
-    message.htmlBuffer = transferableHtmlBuffer;
-    if (task.encoding)
-        message.encoding = task.encoding;
-    const backingBuffer = transferableHtmlBuffer.buffer;
-    if (isSharedArrayBuffer(backingBuffer))
-        return { message };
-    return { message, transferList: [backingBuffer] };
-}
-/**
- * Worker Pool Sizing Configuration
- *
- * Default: min(4, floor(availableParallelism() / 2)), constrained to [2, N]
- *
- * Tuning Guidance:
- * - **Default behavior**: Appropriate for most deployments. Uses half of available
- *   CPU threads (capped at 4) to balance throughput with system resource availability.
- *
- * - **CPU-limited containers**: If running in a container with strict CPU limits
- *   (e.g., Docker with --cpus=2), the default may over-subscribe. Consider setting
- *   maxWorkerScale to match the container's CPU limit.
- *
- * - **High-concurrency workloads**: For dedicated servers handling many concurrent
- *   fetch requests, increasing maxWorkerScale to (availableParallelism() + 2) may
- *   improve throughput by overlapping I/O wait with computation.
- *
- * - **Memory-constrained environments**: Each worker allocates ~50-100MB for DOM
- *   parsing. If memory is limited, reduce maxWorkerScale to (availableParallelism() / 2)
- *   or lower to prevent OOM errors.
- *
- * - **Shared hosting**: On shared systems where CPU is contested, reducing the pool
- *   size prevents starving other processes. Consider maxWorkerScale = 2 or using
- *   process-based workers (TRANSFORM_WORKER_MODE=process) for better isolation.
- *
- * Configuration:
- * - TRANSFORM_MAX_WORKER_SCALE env var (default: availableParallelism())
- * - TRANSFORM_WORKER_MODE env var: 'threads' (default) or 'process'
- *
- * See config.ts for full worker configuration options.
- */
-const POOL_MIN_WORKERS = Math.max(2, Math.min(4, Math.floor(availableParallelism() / 2)));
-const POOL_MAX_WORKERS = config.transform.maxWorkerScale;
-const POOL_SCALE_THRESHOLD = 0.5;
-const WORKER_NAME_PREFIX = 'fetch-url-mcp-transform';
-const DEFAULT_TIMEOUT_MS = config.transform.timeoutMs;
-const TRANSFORM_CHILD_PATH = fileURLToPath(new URL('./workers/transform-child.js', import.meta.url));
-function ensureTightBuffer(buffer) {
-    if (buffer.byteOffset === 0 &&
-        buffer.byteLength === buffer.buffer.byteLength) {
-        return buffer;
-    }
-    return Buffer.from(buffer);
-}
-function createThreadWorkerHost(_workerIndex, name) {
-    const resourceLimits = config.transform.workerResourceLimits;
-    const worker = new Worker(new URL('./workers/transform-worker.js', import.meta.url), {
-        name,
-        ...(resourceLimits ? { resourceLimits } : {}),
-    });
-    return {
-        kind: 'thread',
-        supportsTransferList: true,
-        threadId: worker.threadId,
-        postMessage: (message, transferList) => {
-            worker.postMessage(message, transferList);
-        },
-        terminate: async () => {
-            await worker.terminate();
-        },
-        unref: () => {
-            worker.unref();
-        },
-        onMessage: (handler) => {
-            worker.on('message', handler);
-        },
-        onError: (handler) => {
-            worker.on('error', handler);
-            worker.on('messageerror', handler);
-        },
-        onExit: (handler) => {
-            worker.on('exit', (code) => {
-                handler(code, null);
-            });
-        },
-    };
-}
-function createProcessWorkerHost(workerIndex, name) {
-    const child = fork(TRANSFORM_CHILD_PATH, [], {
-        stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
-        serialization: 'advanced',
-        env: {
-            ...process.env,
-            FETCH_URL_MCP_WORKER_INDEX: String(workerIndex),
-            FETCH_URL_MCP_WORKER_NAME: name,
-        },
-    });
-    if (child.pid === undefined) {
-        throw new Error('Failed to fork process');
-    }
-    return {
-        kind: 'process',
-        supportsTransferList: false,
-        pid: child.pid,
-        postMessage: (message) => {
-            if (!child.connected) {
-                throw new Error('Transform worker IPC channel is closed');
-            }
-            child.send(message);
-        },
-        terminate: () => new Promise((resolve) => {
-            if (child.exitCode !== null || child.killed) {
-                resolve();
-                return;
-            }
-            child.once('exit', () => {
-                resolve();
-            });
-            try {
-                child.kill();
-            }
-            catch {
-                resolve();
-            }
-        }),
-        unref: () => {
-            child.unref();
-        },
-        onMessage: (handler) => {
-            child.on('message', handler);
-        },
-        onError: (handler) => {
-            child.on('error', handler);
-        },
-        onExit: (handler) => {
-            child.on('exit', (code, signal) => {
-                handler(code, signal);
-            });
-        },
-    };
-}
-class WorkerPool {
-    static CLOSED_MESSAGE = 'Transform worker pool closed';
-    workers = [];
-    capacity;
-    minCapacity = POOL_MIN_WORKERS;
-    maxCapacity = POOL_MAX_WORKERS;
-    queue = [];
-    queueHead = 0;
-    inflight = new Map();
-    cancelAcks = new Map();
-    timeoutMs;
-    queueMax;
-    spawnWorkerImpl;
-    closed = false;
-    taskIdSeq = 0;
-    constructor(size, timeoutMs, spawnWorker) {
-        if (size === 0) {
-            this.capacity = 0;
-        }
-        else {
-            this.capacity = Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
-        }
-        this.timeoutMs = timeoutMs;
-        this.queueMax = this.maxCapacity * 32;
-        this.spawnWorkerImpl = spawnWorker;
-    }
-    async transform(htmlOrBuffer, url, options) {
-        this.ensureOpen();
-        if (options.signal?.aborted)
-            throw abortPolicy.createAbortError(url, 'transform:enqueue');
-        if (this.getQueueDepth() >= this.queueMax) {
-            throw new FetchError('Transform worker queue is full', url, 503, {
-                reason: 'queue_full',
-                stage: 'transform:enqueue',
-            });
-        }
-        return new Promise((resolve, reject) => {
-            const task = this.createPendingTask(htmlOrBuffer, url, options, resolve, reject);
-            this.queue.push(task);
-            this.drainQueue();
-        });
-    }
-    getQueueDepth() {
-        const depth = this.queue.length - this.queueHead;
-        return depth > 0 ? depth : 0;
-    }
-    getActiveWorkers() {
-        return this.workers.filter((s) => s?.busy).length;
-    }
-    getCapacity() {
-        return this.capacity;
-    }
-    resize(size) {
-        const newCapacity = Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
-        if (newCapacity === this.capacity)
-            return;
-        this.capacity = newCapacity;
-        this.drainQueue();
-    }
-    async close() {
-        if (this.closed)
-            return;
-        this.closed = true;
-        const terminations = this.workers
-            .map((slot) => slot?.host.terminate())
-            .filter((p) => p !== undefined);
-        this.workers.fill(undefined);
-        this.workers.length = 0;
-        for (const id of Array.from(this.inflight.keys())) {
-            const inflight = this.takeInflight(id);
-            if (!inflight)
-                continue;
-            this.finalizeTask(inflight.context, () => {
-                inflight.reject(new Error(WorkerPool.CLOSED_MESSAGE));
-            });
-        }
-        for (let i = this.queueHead; i < this.queue.length; i += 1) {
-            const task = this.queue[i];
-            if (!task)
-                continue;
-            this.clearAbortListener(task.signal, task.abortListener);
-            this.finalizeTask(task.context, () => {
-                task.reject(new Error(WorkerPool.CLOSED_MESSAGE));
-            });
-        }
-        this.queue.length = 0;
-        this.queueHead = 0;
-        await Promise.allSettled(terminations);
-    }
-    ensureOpen() {
-        if (this.closed)
-            throw new Error(WorkerPool.CLOSED_MESSAGE);
-    }
-    createPendingTask(htmlOrBuffer, url, options, resolve, reject) {
-        const id = (this.taskIdSeq++).toString(36);
-        // Preserve request context for resolve/reject even when callbacks fire
-        // from worker thread events.
-        const context = createTaskContext();
-        let abortListener;
-        if (options.signal) {
-            abortListener = () => {
-                this.onAbortSignal(id, url, context, reject);
-            };
-            options.signal.addEventListener('abort', abortListener, { once: true });
-        }
-        const task = {
-            id,
-            url,
-            includeMetadata: options.includeMetadata,
-            ...(options.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
-            ...(options.inputTruncated ? { inputTruncated: true } : {}),
-            signal: options.signal,
-            abortListener,
-            context,
-            resolve,
-            reject,
-        };
-        if (typeof htmlOrBuffer === 'string') {
-            task.html = htmlOrBuffer;
-        }
-        else {
-            task.htmlBuffer = htmlOrBuffer;
-            if (options.encoding) {
-                task.encoding = options.encoding;
-            }
-        }
-        return task;
-    }
-    onAbortSignal(id, url, context, reject) {
-        if (this.closed) {
-            this.finalizeTask(context, () => {
-                reject(new Error(WorkerPool.CLOSED_MESSAGE));
-            });
-            return;
-        }
-        const inflight = this.inflight.get(id);
-        if (inflight) {
-            void this.abortInflight(id, url, inflight.workerIndex);
-            return;
-        }
-        const queuedIndex = this.findQueuedIndex(id);
-        if (queuedIndex !== null) {
-            const task = this.queue[queuedIndex];
-            if (task)
-                this.clearAbortListener(task.signal, task.abortListener);
-            this.queue.splice(queuedIndex, 1);
-            if (task) {
-                this.finalizeTask(task.context, () => {
-                    task.reject(abortPolicy.createAbortError(url, 'transform:queued-abort'));
-                });
-            }
-            else {
-                this.finalizeTask(context, () => {
-                    reject(abortPolicy.createAbortError(url, 'transform:queued-abort'));
-                });
-            }
-            this.maybeCompactQueue();
-        }
-    }
-    resolveCancelAck(id) {
-        const pending = this.cancelAcks.get(id);
-        if (!pending)
-            return;
-        pending.timeout.cancel();
-        pending.resolve();
-    }
-    waitForCancelAck(id) {
-        const existing = this.cancelAcks.get(id);
-        if (existing) {
-            return existing.promise;
-        }
-        let resolve = () => { };
-        const timeout = createUnrefTimeout(200, undefined);
-        const racePromise = new Promise((finish) => {
-            resolve = finish;
-        });
-        const promise = Promise.race([racePromise, timeout.promise]).finally(() => {
-            this.cancelAcks.delete(id);
-            timeout.cancel();
-        });
-        this.cancelAcks.set(id, { promise, resolve, timeout });
-        return promise;
-    }
-    async abortInflight(id, url, workerIndex) {
-        const slot = this.workers[workerIndex];
-        const inflight = this.inflight.get(id);
-        if (inflight) {
-            inflight.cancelPending = true;
-        }
-        if (slot) {
-            try {
-                slot.host.postMessage({ type: 'cancel', id });
-            }
-            catch {
-                // Worker may be unavailable; failure is acceptable during abort
-            }
-        }
-        await this.waitForCancelAck(id);
-        this.failTask(id, abortPolicy.createAbortError(url, 'transform:signal-abort'));
-        if (slot)
-            this.restartWorker(workerIndex, slot);
-    }
-    clearAbortListener(signal, listener) {
-        if (!signal || !listener)
-            return;
-        try {
-            signal.removeEventListener('abort', listener);
-        }
-        catch {
-            // Defensive: removeEventListener should not throw, but handle edge cases
-        }
-    }
-    spawnWorker(workerIndex) {
-        const name = `${WORKER_NAME_PREFIX}-${workerIndex + 1}`;
-        const host = this.spawnWorkerImpl(workerIndex, name);
-        host.unref();
-        host.onMessage((raw) => {
-            this.onWorkerMessage(workerIndex, raw);
-        });
-        host.onError((error) => {
-            this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
-        });
-        host.onExit((code, signal) => {
-            const suffix = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`;
-            this.onWorkerBroken(workerIndex, `Transform worker exited (${suffix})`);
-        });
-        return { host, busy: false, currentTaskId: null, name };
-    }
-    onWorkerBroken(workerIndex, message) {
-        if (this.closed)
-            return;
-        const slot = this.workers[workerIndex];
-        if (!slot)
-            return;
-        logWarn('Transform worker unavailable; restarting', {
-            reason: message,
-            workerIndex,
-            workerKind: slot.host.kind,
-            workerName: slot.name,
-            ...(slot.host.kind === 'process'
-                ? { pid: slot.host.pid }
-                : { threadId: slot.host.threadId }),
-        });
-        if (slot.busy && slot.currentTaskId) {
-            this.failTask(slot.currentTaskId, new Error(message));
-        }
-        this.restartWorker(workerIndex, slot);
-    }
-    restartWorker(workerIndex, slot) {
-        if (this.closed)
-            return;
-        const target = slot ?? this.workers[workerIndex];
-        if (target) {
-            target.host.terminate().catch(() => undefined);
-        }
-        this.workers[workerIndex] = this.spawnWorker(workerIndex);
-        this.drainQueue();
-    }
-    onWorkerMessage(workerIndex, raw) {
-        if (!isWorkerResponse(raw))
-            return;
-        const message = raw;
-        if (message.type === 'cancelled') {
-            this.resolveCancelAck(message.id);
-            return;
-        }
-        const inflightPeek = this.inflight.get(message.id);
-        if (inflightPeek?.cancelPending) {
-            this.resolveCancelAck(message.id);
-            return;
-        }
-        const inflight = this.takeInflight(message.id);
-        if (!inflight)
-            return;
-        this.markIdle(workerIndex);
-        if (message.type === 'result') {
-            this.finalizeTask(inflight.context, () => {
-                inflight.resolve({
-                    markdown: message.result.markdown,
-                    truncated: message.result.truncated,
-                    title: message.result.title,
-                    ...(message.result.metadata
-                        ? { metadata: message.result.metadata }
-                        : {}),
-                });
-            });
-        }
-        else {
-            const err = message.error;
-            if (err.name === 'FetchError') {
-                this.finalizeTask(inflight.context, () => {
-                    inflight.reject(new FetchError(err.message, err.url, err.statusCode, err.details ?? {}));
-                });
-            }
-            else {
-                this.finalizeTask(inflight.context, () => {
-                    inflight.reject(new Error(err.message));
-                });
-            }
-        }
-        this.drainQueue();
-    }
-    takeInflight(id) {
-        const inflight = this.inflight.get(id);
-        if (!inflight)
-            return null;
-        inflight.timeout.cancel();
-        this.clearAbortListener(inflight.signal, inflight.abortListener);
-        this.inflight.delete(id);
-        return inflight;
-    }
-    markIdle(workerIndex) {
-        const slot = this.workers[workerIndex];
-        if (!slot)
-            return;
-        slot.busy = false;
-        slot.currentTaskId = null;
-    }
-    failTask(id, error) {
-        const inflight = this.takeInflight(id);
-        if (!inflight)
-            return;
-        this.finalizeTask(inflight.context, () => {
-            inflight.reject(error);
-        });
-        this.markIdle(inflight.workerIndex);
-    }
-    maybeScaleUp() {
-        if (this.getQueueDepth() > this.capacity * POOL_SCALE_THRESHOLD &&
-            this.capacity < this.maxCapacity) {
-            this.capacity += 1;
-        }
-    }
-    drainQueue() {
-        if (this.closed || this.getQueueDepth() === 0)
-            return;
-        this.maybeScaleUp();
-        for (let i = 0; i < this.workers.length; i += 1) {
-            const slot = this.workers[i];
-            if (slot && !slot.busy) {
-                this.dispatchFromQueue(i, slot);
-                if (this.getQueueDepth() === 0)
-                    return;
-            }
-        }
-        if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
-            const workerIndex = this.workers.length;
-            const slot = this.spawnWorker(workerIndex);
-            this.workers.push(slot);
-            this.dispatchFromQueue(workerIndex, slot);
-            if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
-                setImmediate(() => {
-                    this.drainQueue();
-                });
-            }
-        }
-    }
-    takeNextQueuedTask() {
-        while (this.queueHead < this.queue.length) {
-            const task = this.queue[this.queueHead];
-            this.queueHead += 1;
-            if (task) {
-                this.maybeCompactQueue();
-                return task;
-            }
-        }
-        this.maybeCompactQueue();
-        return null;
-    }
-    dispatchFromQueue(workerIndex, slot) {
-        const task = this.takeNextQueuedTask();
-        if (!task)
-            return;
-        if (this.closed) {
-            this.clearAbortListener(task.signal, task.abortListener);
-            this.finalizeTask(task.context, () => {
-                task.reject(new Error(WorkerPool.CLOSED_MESSAGE));
-            });
-            return;
-        }
-        if (task.signal?.aborted) {
-            this.clearAbortListener(task.signal, task.abortListener);
-            this.finalizeTask(task.context, () => {
-                task.reject(abortPolicy.createAbortError(task.url, 'transform:dispatch'));
-            });
-            return;
-        }
-        slot.busy = true;
-        slot.currentTaskId = task.id;
-        const timeout = createUnrefTimeout(this.timeoutMs, null);
-        void timeout.promise
-            .then(() => {
-            try {
-                slot.host.postMessage({ type: 'cancel', id: task.id });
-            }
-            catch {
-                // Worker may be unavailable; proceed with timeout handling
-            }
-            const inflight = this.takeInflight(task.id);
-            if (!inflight)
-                return;
-            this.finalizeTask(inflight.context, () => {
-                inflight.reject(new FetchError('Request timeout', task.url, 504, {
-                    reason: 'timeout',
-                    stage: 'transform:worker-timeout',
-                }));
-            });
-            this.restartWorker(workerIndex, slot);
-        })
-            .catch((error) => {
-            this.failTask(task.id, error);
-        });
-        this.inflight.set(task.id, {
-            resolve: task.resolve,
-            reject: task.reject,
-            timeout,
-            signal: task.signal,
-            abortListener: task.abortListener,
-            workerIndex,
-            context: task.context,
-            cancelPending: false,
-        });
-        try {
-            const { message, transferList } = buildWorkerDispatchPayload(task, slot.host.supportsTransferList);
-            slot.host.postMessage(message, transferList);
-        }
-        catch (error) {
-            timeout.cancel();
-            this.clearAbortListener(task.signal, task.abortListener);
-            this.inflight.delete(task.id);
-            this.markIdle(workerIndex);
-            this.finalizeTask(task.context, () => {
-                task.reject(error instanceof Error
-                    ? error
-                    : new Error('Failed to dispatch transform worker message'));
-            });
-            this.restartWorker(workerIndex, slot);
-        }
-    }
-    finalizeTask(context, fn) {
-        try {
-            context.run(fn);
-        }
-        finally {
-            context.dispose();
-        }
-    }
-    findQueuedIndex(id) {
-        for (let i = this.queueHead; i < this.queue.length; i += 1) {
-            const task = this.queue[i];
-            if (task?.id === id)
-                return i;
-        }
-        return null;
-    }
-    maybeCompactQueue() {
-        if (this.queueHead === 0)
-            return;
-        if (this.queueHead >= this.queue.length ||
-            (this.queueHead > 1024 && this.queueHead > this.queue.length / 2)) {
-            this.queue.splice(0, this.queueHead);
-            this.queueHead = 0;
-        }
-    }
-}
-let workerPool = null;
-function resolveWorkerSpawner() {
-    return config.transform.workerMode === 'process'
-        ? createProcessWorkerHost
-        : createThreadWorkerHost;
-}
-function getOrCreateWorkerPool() {
-    const size = config.transform.maxWorkerScale === 0 ? 0 : POOL_MIN_WORKERS;
-    workerPool ??= new WorkerPool(size, DEFAULT_TIMEOUT_MS, resolveWorkerSpawner());
-    return workerPool;
-}
-function getWorkerPoolStats() {
-    if (!workerPool)
-        return null;
-    return {
-        queueDepth: workerPool.getQueueDepth(),
-        activeWorkers: workerPool.getActiveWorkers(),
-        capacity: workerPool.getCapacity(),
-    };
-}
-async function shutdownWorkerPool() {
-    if (!workerPool)
-        return;
-    await workerPool.close();
-    workerPool = null;
-}
 export function getTransformPoolStats() {
     return getWorkerPoolStats();
 }
@@ -2410,3 +1719,4 @@ export async function transformHtmlToMarkdown(html, url, options) {
 export async function transformBufferToMarkdown(htmlBuffer, url, options) {
     return transformInputToMarkdown(htmlBuffer, url, options);
 }
+//# sourceMappingURL=transform.js.map