@j0hanz/superfetch 2.0.1 → 2.1.1

package/dist/services/telemetry.js

@@ -0,0 +1,43 @@
+import diagnosticsChannel from 'node:diagnostics_channel';
+import { performance } from 'node:perf_hooks';
+import { redactUrl } from '../utils/url-redactor.js';
+import { getOperationId, getRequestId } from './context.js';
+const transformChannel = diagnosticsChannel.channel('superfetch.transform');
+function publishTransformEvent(event) {
+    if (!transformChannel.hasSubscribers)
+        return;
+    try {
+        transformChannel.publish(event);
+    }
+    catch {
+        // Avoid crashing the publisher if a subscriber throws.
+    }
+}
+export function startTransformStage(url, stage) {
+    if (!transformChannel.hasSubscribers)
+        return null;
+    return {
+        stage,
+        startTime: performance.now(),
+        url: redactUrl(url),
+    };
+}
+export function endTransformStage(context, options) {
+    if (!context)
+        return;
+    const requestId = getRequestId();
+    const operationId = getOperationId();
+    const event = {
+        v: 1,
+        type: 'stage',
+        stage: context.stage,
+        durationMs: performance.now() - context.startTime,
+        url: context.url,
+        ...(requestId ? { requestId } : {}),
+        ...(operationId ? { operationId } : {}),
+        ...(options?.truncated !== undefined
+            ? { truncated: options.truncated }
+            : {}),
+    };
+    publishTransformEvent(event);
+}

package/dist/services/transform-worker-pool.d.ts

@@ -1,4 +1,11 @@
 import type { MarkdownTransformResult } from '../config/types/content.js';
-import type { WorkerTransformRequest } from './transform-worker-types.js';
-export declare function transformInWorker(request: Omit<WorkerTransformRequest, 'id'>, signal?: AbortSignal): Promise<MarkdownTransformResult>;
-export declare function destroyTransformWorkers(): Promise<void>;
+interface TransformWorkerPool {
+    transform(html: string, url: string, options: {
+        includeMetadata: boolean;
+        signal?: AbortSignal;
+    }): Promise<MarkdownTransformResult>;
+    close(): Promise<void>;
+}
+export declare function getOrCreateTransformWorkerPool(): TransformWorkerPool;
+export declare function shutdownTransformWorkerPool(): Promise<void>;
+export {};

package/dist/services/transform-worker-pool.js

@@ -1,215 +1,244 @@
+import { randomUUID } from 'node:crypto';
+import os from 'node:os';
 import { Worker } from 'node:worker_threads';
-import { config } from '../config/index.js';
-import { getErrorMessage } from '../utils/error-utils.js';
-import { logWarn } from './logger.js';
-import { isWorkerResponse } from './transform-worker-types.js';
-class TransformWorkerPool {
-    workerUrl;
-    slots = [];
+import { FetchError } from '../errors/app-error.js';
+import { getErrorMessage } from '../utils/error-details.js';
+let pool = null;
+function resolveDefaultWorkerCount() {
+    const parallelism = typeof os.availableParallelism === 'function'
+        ? os.availableParallelism()
+        : os.cpus().length;
+    // Leave 1 core for the event loop; cap to avoid runaway memory.
+    return Math.min(16, Math.max(1, parallelism - 1));
+}
+const DEFAULT_TIMEOUT_MS = 30000;
+export function getOrCreateTransformWorkerPool() {
+    pool ??= new WorkerPool(resolveDefaultWorkerCount(), DEFAULT_TIMEOUT_MS);
+    return pool;
+}
+export async function shutdownTransformWorkerPool() {
+    if (!pool)
+        return;
+    await pool.close();
+    pool = null;
+}
+class WorkerPool {
+    workers = [];
     queue = [];
-    nextId = 1;
-    destroyed = false;
-    constructor(workerUrl, size) {
-        this.workerUrl = workerUrl;
-        for (let i = 0; i < size; i += 1) {
-            this.slots.push(this.spawnWorker());
-        }
-    }
-    run(request, signal) {
-        if (this.destroyed) {
-            return Promise.reject(new Error('Worker pool is shut down'));
-        }
-        return new Promise((resolve, reject) => {
-            if (signal?.aborted) {
-                reject(new Error('Aborted'));
-                return;
-            }
-            const task = this.createTask(request, resolve, reject, signal);
-            this.attachAbortHandler(task, signal);
-            this.enqueueTask(task);
-        });
-    }
-    async destroy() {
-        if (this.destroyed)
-            return;
-        this.destroyed = true;
-        const pending = this.queue.splice(0);
-        for (const task of pending) {
-            this.cleanupTask(task);
-            task.reject(new Error('Worker pool shutting down'));
-        }
-        for (const slot of this.slots) {
-            if (slot.current) {
-                const task = slot.current;
-                slot.current = undefined;
-                slot.busy = false;
-                this.cleanupTask(task);
-                task.reject(new Error('Worker pool shutting down'));
-            }
-        }
-        await Promise.allSettled(this.slots.map((slot) => slot.worker.terminate()));
-        this.slots = [];
-    }
-    dispatch() {
-        if (this.destroyed)
-            return;
-        const idle = this.slots.find((slot) => !slot.busy);
-        if (!idle)
-            return;
-        const task = this.queue.shift();
-        if (!task)
-            return;
-        task.status = 'running';
-        idle.busy = true;
-        idle.current = task;
-        try {
-            idle.worker.postMessage(task.request);
-        }
-        catch (error) {
-            this.failTask(idle, error);
+    inflight = new Map();
+    timeoutMs;
+    queueMax;
+    closed = false;
+    constructor(size, timeoutMs) {
+        const safeSize = Math.max(1, size);
+        this.timeoutMs = timeoutMs;
+        this.queueMax = safeSize * 2;
+        for (let index = 0; index < safeSize; index += 1) {
+            this.workers.push(this.spawnWorker(index));
         }
     }
-    createTask(request, resolve, reject, signal) {
-        const id = this.nextId;
-        this.nextId += 1;
-        return {
-            id,
-            request: { ...request, id },
-            resolve,
-            reject,
-            signal,
-            abortHandler: undefined,
-            status: 'queued',
-        };
-    }
-    attachAbortHandler(task, signal) {
-        if (!signal)
-            return;
-        const onAbort = () => {
-            if (task.status === 'queued') {
-                this.removeQueuedTask(task);
-                task.reject(new Error('Aborted'));
-                return;
-            }
-            this.abortRunningTask(task);
+    spawnWorker(workerIndex) {
+        const worker = new Worker(new URL('../workers/transform-worker.js', import.meta.url));
+        // Workers must not keep the process alive by themselves.
+        worker.unref();
+        const slot = {
+            worker,
+            busy: false,
+            currentTaskId: null,
         };
-        task.abortHandler = onAbort;
-        signal.addEventListener('abort', onAbort, { once: true });
-    }
-    enqueueTask(task) {
-        this.queue.push(task);
-        this.dispatch();
-    }
-    attachWorker(slot) {
-        slot.worker.on('message', (message) => {
-            this.handleMessage(slot, message);
+        worker.on('message', (raw) => {
+            this.onWorkerMessage(workerIndex, raw);
         });
-        slot.worker.on('error', (error) => {
-            this.handleWorkerFailure(slot, error);
+        worker.on('error', (error) => {
+            this.onWorkerBroken(workerIndex, `Transform worker error: ${getErrorMessage(error)}`);
         });
-        slot.worker.on('exit', (code) => {
-            if (code !== 0) {
-                this.handleWorkerFailure(slot, new Error(`Worker exited with code ${code}`));
-            }
+        worker.on('exit', (code) => {
+            this.onWorkerBroken(workerIndex, `Transform worker exited (code ${code})`);
         });
-    }
-    spawnWorker() {
-        const slot = {
-            worker: new Worker(this.workerUrl),
-            busy: false,
-            current: undefined,
-        };
-        this.attachWorker(slot);
         return slot;
     }
-    handleMessage(slot, message) {
-        const task = slot.current;
-        if (!task)
+    onWorkerBroken(workerIndex, message) {
+        if (this.closed)
             return;
-        if (!isWorkerResponse(message) || message.id !== task.id) {
-            this.handleWorkerFailure(slot, new Error('Unexpected worker response'));
+        const slot = this.workers[workerIndex];
+        if (!slot)
+            return;
+        if (slot.busy && slot.currentTaskId) {
+            this.failTask(slot.currentTaskId, new Error(message));
+        }
+        void slot.worker.terminate();
+        this.workers[workerIndex] = this.spawnWorker(workerIndex);
+        this.drainQueue();
+    }
+    onWorkerMessage(workerIndex, raw) {
+        if (!raw ||
+            typeof raw !== 'object' ||
+            !('type' in raw) ||
+            !('id' in raw) ||
+            typeof raw.id !== 'string' ||
+            typeof raw.type !== 'string') {
+            return;
+        }
+        const message = raw;
+        const inflight = this.inflight.get(message.id);
+        if (!inflight)
             return;
+        clearTimeout(inflight.timer);
+        if (inflight.signal && inflight.abortListener) {
+            inflight.signal.removeEventListener('abort', inflight.abortListener);
         }
-        slot.current = undefined;
-        slot.busy = false;
-        this.cleanupTask(task);
-        if (message.ok) {
-            task.resolve(message.result);
+        this.inflight.delete(message.id);
+        const slot = this.workers[workerIndex];
+        if (slot) {
+            slot.busy = false;
+            slot.currentTaskId = null;
+        }
+        if (message.type === 'result') {
+            inflight.resolve(message.result);
         }
         else {
-            task.reject(new Error(message.error));
+            const { error } = message;
+            if (error.name === 'FetchError') {
+                inflight.reject(new FetchError(error.message, error.url, error.statusCode, error.details ?? {}));
+            }
+            else {
+                inflight.reject(new Error(error.message));
+            }
         }
-        this.dispatch();
+        this.drainQueue();
     }
-    handleWorkerFailure(slot, error) {
-        const task = slot.current;
-        if (task) {
-            slot.current = undefined;
+    failTask(id, error) {
+        const inflight = this.inflight.get(id);
+        if (!inflight)
+            return;
+        clearTimeout(inflight.timer);
+        if (inflight.signal && inflight.abortListener) {
+            inflight.signal.removeEventListener('abort', inflight.abortListener);
+        }
+        this.inflight.delete(id);
+        inflight.reject(error);
+        const slot = this.workers[inflight.workerIndex];
+        if (slot) {
             slot.busy = false;
-            this.cleanupTask(task);
-            task.reject(error instanceof Error ? error : new Error(getErrorMessage(error)));
+            slot.currentTaskId = null;
         }
-        logWarn('Worker thread failure', {
-            error: getErrorMessage(error),
-        });
-        this.replaceWorker(slot);
-        this.dispatch();
     }
-    replaceWorker(slot) {
-        try {
-            void slot.worker.terminate();
+    async transform(html, url, options) {
+        if (this.closed) {
+            throw new Error('Transform worker pool closed');
         }
-        catch {
-            // Best-effort cleanup.
+        if (this.queue.length >= this.queueMax) {
+            throw new Error('Transform worker queue is full');
         }
-        slot.worker = new Worker(this.workerUrl);
-        slot.busy = false;
-        slot.current = undefined;
-        this.attachWorker(slot);
+        return new Promise((resolve, reject) => {
+            this.queue.push({
+                id: randomUUID(),
+                html,
+                url,
+                includeMetadata: options.includeMetadata,
+                signal: options.signal,
+                resolve,
+                reject,
+            });
+            this.drainQueue();
+        });
     }
-    failTask(slot, error) {
-        const task = slot.current;
-        if (!task)
+    drainQueue() {
+        if (this.queue.length === 0)
             return;
-        slot.current = undefined;
-        slot.busy = false;
-        this.cleanupTask(task);
-        task.reject(error instanceof Error ? error : new Error(String(error)));
-        this.dispatch();
+        for (let workerIndex = 0; workerIndex < this.workers.length; workerIndex += 1) {
+            const slot = this.workers[workerIndex];
+            if (!slot || slot.busy)
+                continue;
+            const task = this.queue.shift();
+            if (!task)
+                return;
+            this.dispatch(workerIndex, slot, task);
+            if (this.queue.length === 0)
+                return;
+        }
     }
-    abortRunningTask(task) {
-        const slot = this.slots.find((s) => s.current?.id === task.id);
-        if (!slot)
+    dispatch(workerIndex, slot, task) {
+        if (task.signal?.aborted) {
+            task.reject(new FetchError('Request was canceled', task.url, 499, {
+                reason: 'aborted',
+                stage: 'transform:dispatch',
+            }));
             return;
-        this.handleWorkerFailure(slot, new Error('Aborted'));
-    }
-    removeQueuedTask(task) {
-        const index = this.queue.findIndex((queued) => queued.id === task.id);
-        if (index >= 0) {
-            this.queue.splice(index, 1);
         }
-        this.cleanupTask(task);
+        slot.busy = true;
+        slot.currentTaskId = task.id;
+        const timer = setTimeout(() => {
+            try {
+                slot.worker.postMessage({ type: 'cancel', id: task.id });
+            }
+            catch {
+                // ignore
+            }
+            const inflight = this.inflight.get(task.id);
+            if (!inflight)
+                return;
+            clearTimeout(inflight.timer);
+            if (inflight.signal && inflight.abortListener) {
+                inflight.signal.removeEventListener('abort', inflight.abortListener);
+            }
+            this.inflight.delete(task.id);
+            inflight.reject(new FetchError('Request timeout', task.url, 504, {
+                reason: 'timeout',
+                stage: 'transform:worker-timeout',
+            }));
+            if (!this.closed) {
+                void slot.worker.terminate();
+                this.workers[workerIndex] = this.spawnWorker(workerIndex);
+                this.drainQueue();
+            }
+        }, this.timeoutMs).unref();
+        let abortListener;
+        if (task.signal) {
+            abortListener = () => {
+                try {
+                    slot.worker.postMessage({ type: 'cancel', id: task.id });
+                }
+                catch {
+                    // ignore
+                }
+            };
+            task.signal.addEventListener('abort', abortListener, { once: true });
+        }
+        this.inflight.set(task.id, {
+            resolve: task.resolve,
+            reject: task.reject,
+            timer,
+            signal: task.signal,
+            abortListener,
+            workerIndex,
+        });
+        slot.worker.postMessage({
+            type: 'transform',
+            id: task.id,
+            html: task.html,
+            url: task.url,
+            includeMetadata: task.includeMetadata,
+        });
     }
-    cleanupTask(task) {
-        if (task.signal && task.abortHandler) {
-            task.signal.removeEventListener('abort', task.abortHandler);
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        const terminations = this.workers.map((slot) => slot.worker.terminate());
+        this.workers.length = 0;
+        for (const [id, inflight] of this.inflight.entries()) {
+            clearTimeout(inflight.timer);
+            if (inflight.signal && inflight.abortListener) {
+                inflight.signal.removeEventListener('abort', inflight.abortListener);
+            }
+            inflight.reject(new Error('Transform worker pool closed'));
+            this.inflight.delete(id);
         }
+        for (const task of this.queue) {
+            task.reject(new Error('Transform worker pool closed'));
+        }
+        this.queue.length = 0;
+        await Promise.allSettled(terminations);
     }
 }
-let pool = null;
-function getPool() {
-    if (pool)
-        return pool;
-    pool = new TransformWorkerPool(new URL('../workers/content-transform.worker.js', import.meta.url), config.workers.poolSize);
-    return pool;
-}
-export async function transformInWorker(request, signal) {
-    return getPool().run(request, signal);
-}
-export async function destroyTransformWorkers() {
-    if (!pool)
-        return;
-    const current = pool;
-    pool = null;
-    await current.destroy();
-}

package/dist/tools/handlers/fetch-url.tool.js

@@ -54,8 +54,8 @@ function deserializeMarkdownResult(cached) {
     return parseCachedMarkdownResult(cached);
 }
 function buildMarkdownTransform() {
-    return (html, url) => {
-        const result = transformHtmlToMarkdown(html, url, {
+    return async (html, url) => {
+        const result = await transformHtmlToMarkdown(html, url, {
             includeMetadata: true,
         });
         return { ...result, content: result.markdown };
@@ -68,9 +68,11 @@ function serializeMarkdownResult(result) {
         truncated: result.truncated,
     });
 }
-function buildStructuredContent(pipeline, inlineResult) {
+function buildStructuredContent(pipeline, inlineResult, inputUrl) {
     return {
         url: pipeline.url,
+        resolvedUrl: pipeline.url,
+        inputUrl,
         title: pipeline.data.title,
         markdown: inlineResult.content,
     };
@@ -89,8 +91,8 @@ async function fetchPipeline(url) {
         deserialize: deserializeMarkdownResult,
     });
 }
-function buildResponse(pipeline, inlineResult) {
-    const structuredContent = buildStructuredContent(pipeline, inlineResult);
+function buildResponse(pipeline, inlineResult, inputUrl) {
+    const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
     const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
     return {
         content,
@@ -113,5 +115,5 @@ async function executeFetch(input) {
     if (inlineResult.error) {
         return createToolErrorResponse(inlineResult.error, url);
     }
-    return buildResponse(pipeline, inlineResult);
+    return buildResponse(pipeline, inlineResult, url);
 }

package/dist/tools/index.d.ts

@@ -1,2 +1,3 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function withRequestContextIfMissing<TParams, TResult>(handler: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;
 export declare function registerTools(server: McpServer): void;

package/dist/tools/index.js

@@ -1,3 +1,5 @@
+import { randomUUID } from 'node:crypto';
+import { getRequestId, runWithRequestContext } from '../services/context.js';
 import { FETCH_URL_TOOL_DESCRIPTION, FETCH_URL_TOOL_NAME, fetchUrlToolHandler, } from './handlers/fetch-url.tool.js';
 import { fetchUrlInputSchema, fetchUrlOutputSchema } from './schemas.js';
 const TOOL_DEFINITION = {
@@ -14,6 +16,16 @@ const TOOL_DEFINITION = {
         openWorldHint: true,
     },
 };
+export function withRequestContextIfMissing(handler) {
+    return async (params) => {
+        const existingRequestId = getRequestId();
+        if (existingRequestId) {
+            return handler(params);
+        }
+        const requestId = randomUUID();
+        return runWithRequestContext({ requestId, operationId: requestId }, () => handler(params));
+    };
+}
 export function registerTools(server) {
     server.registerTool(TOOL_DEFINITION.name, {
         title: TOOL_DEFINITION.title,
@@ -21,5 +33,5 @@ export function registerTools(server) {
         inputSchema: TOOL_DEFINITION.inputSchema,
         outputSchema: TOOL_DEFINITION.outputSchema,
         annotations: TOOL_DEFINITION.annotations,
-    }, TOOL_DEFINITION.handler);
+    }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
 }

package/dist/tools/schemas.d.ts

@@ -4,6 +4,8 @@ export declare const fetchUrlInputSchema: z.ZodObject<{
 }, z.core.$strict>;
 export declare const fetchUrlOutputSchema: z.ZodObject<{
     url: z.ZodString;
+    inputUrl: z.ZodOptional<z.ZodString>;
+    resolvedUrl: z.ZodOptional<z.ZodString>;
     title: z.ZodOptional<z.ZodString>;
     markdown: z.ZodOptional<z.ZodString>;
     error: z.ZodOptional<z.ZodString>;

package/dist/tools/schemas.js

@@ -4,6 +4,14 @@ export const fetchUrlInputSchema = z.strictObject({
 });
 export const fetchUrlOutputSchema = z.strictObject({
     url: z.string().describe('The fetched URL'),
+    inputUrl: z
+        .string()
+        .optional()
+        .describe('The original URL provided by the caller'),
+    resolvedUrl: z
+        .string()
+        .optional()
+        .describe('The normalized or transformed URL that was fetched'),
     title: z.string().optional().describe('Page title'),
     markdown: z
         .string()

package/dist/tools/utils/content-transform-core.d.ts

@@ -0,0 +1,5 @@
+import type { ExtractedArticle, ExtractedMetadata, MarkdownTransformResult, MetadataBlock, TransformOptions } from '../../config/types/content.js';
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
+export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
+export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;