@j0hanz/superfetch 2.4.4 → 2.4.6

package/dist/tasks.js

@@ -1,13 +1,26 @@
 import { randomUUID } from 'node:crypto';
+import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
 const DEFAULT_TTL_MS = 60000;
 const DEFAULT_POLL_INTERVAL_MS = 1000;
+const DEFAULT_OWNER_KEY = 'default';
+const DEFAULT_PAGE_SIZE = 50;
+const TERMINAL_STATUSES = new Set([
+    'completed',
+    'failed',
+    'cancelled',
+]);
+function isTerminalStatus(status) {
+    return TERMINAL_STATUSES.has(status);
+}
 export class TaskManager {
     tasks = new Map();
-    createTask(options, statusMessage = 'Task started') {
+    waiters = new Map();
+    createTask(options, statusMessage = 'Task started', ownerKey = DEFAULT_OWNER_KEY) {
         const taskId = randomUUID();
         const now = new Date().toISOString();
         const task = {
             taskId,
+            ownerKey,
             status: 'working',
             statusMessage,
             createdAt: now,
@@ -18,26 +31,40 @@ export class TaskManager {
         this.tasks.set(taskId, task);
         return task;
     }
-    getTask(taskId) {
-        return this.tasks.get(taskId);
+    getTask(taskId, ownerKey) {
+        const task = this.tasks.get(taskId);
+        if (!task)
+            return undefined;
+        if (ownerKey && task.ownerKey !== ownerKey)
+            return undefined;
+        if (this.isExpired(task)) {
+            this.tasks.delete(taskId);
+            return undefined;
+        }
+        return task;
     }
     updateTask(taskId, updates) {
         const task = this.tasks.get(taskId);
         if (!task)
             return;
+        if (updates.status && task.status !== updates.status) {
+            if (isTerminalStatus(task.status))
+                return;
+        }
         Object.assign(task, {
             ...updates,
             lastUpdatedAt: new Date().toISOString(),
         });
+        this.notifyWaiters(task);
     }
-    cancelTask(taskId) {
-        const task = this.tasks.get(taskId);
+    cancelTask(taskId, ownerKey) {
+        const task = this.getTask(taskId, ownerKey);
         if (!task)
             return undefined;
         if (task.status === 'completed' ||
             task.status === 'failed' ||
             task.status === 'cancelled') {
-            throw new Error(`Cannot cancel task: already in terminal status '${task.status}'`);
+            throw new McpError(ErrorCode.InvalidParams, `Cannot cancel task: already in terminal status '${task.status}'`);
         }
         this.updateTask(taskId, {
             status: 'cancelled',
@@ -45,8 +72,26 @@ export class TaskManager {
         });
         return this.tasks.get(taskId);
     }
-    listTasks() {
-        return Array.from(this.tasks.values());
+    listTasks(options) {
+        const { ownerKey, cursor, limit } = options;
+        const pageSize = limit && limit > 0 ? limit : DEFAULT_PAGE_SIZE;
+        const startIndex = cursor ? this.decodeCursor(cursor) : 0;
+        if (startIndex === null) {
+            throw new McpError(ErrorCode.InvalidParams, 'Invalid cursor');
+        }
+        const allTasks = Array.from(this.tasks.values()).filter((task) => {
+            if (task.ownerKey !== ownerKey)
+                return false;
+            if (this.isExpired(task)) {
+                this.tasks.delete(task.taskId);
+                return false;
+            }
+            return true;
+        });
+        const page = allTasks.slice(startIndex, startIndex + pageSize);
+        const nextIndex = startIndex + page.length;
+        const nextCursor = nextIndex < allTasks.length ? this.encodeCursor(nextIndex) : undefined;
+        return nextCursor ? { tasks: page, nextCursor } : { tasks: page };
     }
     // Helper to check if task is expired and could be cleaned up
     // In a real implementation, this would be called by a periodic job
@@ -62,5 +107,77 @@ export class TaskManager {
         }
         return count;
     }
+    async waitForTerminalTask(taskId, ownerKey, signal) {
+        const task = this.getTask(taskId, ownerKey);
+        if (!task)
+            return undefined;
+        if (isTerminalStatus(task.status))
+            return task;
+        return new Promise((resolve, reject) => {
+            const onAbort = () => {
+                cleanup();
+                removeWaiter();
+                reject(new McpError(ErrorCode.ConnectionClosed, 'Request was cancelled'));
+            };
+            const cleanup = () => {
+                if (signal) {
+                    signal.removeEventListener('abort', onAbort);
+                }
+            };
+            const removeWaiter = () => {
+                const waiters = this.waiters.get(taskId);
+                if (!waiters)
+                    return;
+                waiters.delete(waiter);
+                if (waiters.size === 0)
+                    this.waiters.delete(taskId);
+            };
+            const waiter = (updated) => {
+                cleanup();
+                resolve(updated);
+            };
+            if (signal?.aborted) {
+                onAbort();
+                return;
+            }
+            const waiters = this.waiters.get(taskId) ?? new Set();
+            waiters.add(waiter);
+            this.waiters.set(taskId, waiters);
+            if (signal) {
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
+        });
+    }
+    notifyWaiters(task) {
+        if (!isTerminalStatus(task.status))
+            return;
+        const waiters = this.waiters.get(task.taskId);
+        if (!waiters)
+            return;
+        this.waiters.delete(task.taskId);
+        for (const waiter of waiters)
+            waiter(task);
+    }
+    isExpired(task) {
+        const createdAt = Date.parse(task.createdAt);
+        if (!Number.isFinite(createdAt))
+            return false;
+        return Date.now() - createdAt > task.ttl;
+    }
+    encodeCursor(index) {
+        return Buffer.from(String(index)).toString('base64');
+    }
+    decodeCursor(cursor) {
+        try {
+            const decoded = Buffer.from(cursor, 'base64').toString('utf8');
+            const value = Number.parseInt(decoded, 10);
+            if (!Number.isFinite(value) || value < 0)
+                return null;
+            return value;
+        }
+        catch {
+            return null;
+        }
+    }
 }
 export const taskManager = new TaskManager();

package/dist/tools.d.ts

@@ -1,6 +1,5 @@
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { CallToolResult, ContentBlock } from '@modelcontextprotocol/sdk/types.js';
-import * as cache from './cache.js';
 import type { MarkdownTransformResult } from './transform-types.js';
 export interface FetchUrlInput {
     url: string;
@@ -73,7 +72,7 @@ export interface ToolHandlerExtra {
     sendNotification?: (notification: ProgressNotification) => Promise<void>;
 }
 export declare const FETCH_URL_TOOL_NAME = "fetch-url";
-export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
+export declare const FETCH_URL_TOOL_DESCRIPTION: string;
 interface ProgressReporter {
     report: (progress: number, message: string) => Promise<void>;
 }
@@ -119,5 +118,5 @@ type MarkdownPipelineResult = MarkdownTransformResult & {
 export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
 export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHandlerExtra): Promise<ToolResponseBase>;
 export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
-export declare function registerTools(server: McpServer, serverIcons?: cache.McpIcon[]): void;
+export declare function registerTools(server: McpServer): void;
 export {};

package/dist/tools.js

@@ -15,7 +15,7 @@ const fetchUrlInputSchema = z.strictObject({
         .url({ protocol: /^https?$/i })
         .min(1)
         .max(config.constants.maxUrlLength)
-        .describe('The URL to fetch'),
+        .describe('The URL of the webpage to fetch and convert to Markdown'),
 });
 const fetchUrlOutputSchema = z.strictObject({
     url: z
@@ -46,24 +46,52 @@ const fetchUrlOutputSchema = z.strictObject({
         .describe('Error message if the request failed'),
 });
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
-export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
+export const FETCH_URL_TOOL_DESCRIPTION = `
+Fetches a webpage and converts it to clean Markdown format optimized for LLM context.
+
+This tool is useful for:
+- Reading documentation, blog posts, or articles.
+- Extracting main content while removing navigation and ads (noise removal).
+- Caching content to speed up repeated queries.
+
+Limitations:
+- Returns truncated content if it exceeds ${config.constants.maxInlineContentChars} characters.
+- Does not execute complex client-side JavaScript interactions.
+`.trim();
+// Specific icon for the fetch-url tool (download cloud / web)
+const TOOL_ICON = {
+    src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
+    mimeType: 'image/svg+xml',
+};
 /* -------------------------------------------------------------------------------------------------
  * Progress reporting
  * ------------------------------------------------------------------------------------------------- */
+function resolveRelatedTaskMeta(meta) {
+    if (!meta)
+        return undefined;
+    const related = meta['io.modelcontextprotocol/related-task'];
+    if (!isObject(related))
+        return undefined;
+    const { taskId } = related;
+    return typeof taskId === 'string' ? { taskId } : undefined;
+}
 class ToolProgressReporter {
     token;
     sendNotification;
-    constructor(token, sendNotification) {
+    relatedTaskMeta;
+    constructor(token, sendNotification, relatedTaskMeta) {
         this.token = token;
         this.sendNotification = sendNotification;
+        this.relatedTaskMeta = relatedTaskMeta;
     }
     static create(extra) {
         const token = extra?._meta?.progressToken ?? null;
         const sendNotification = extra?.sendNotification;
+        const relatedTaskMeta = resolveRelatedTaskMeta(extra?._meta);
         if (token === null || !sendNotification) {
             return { report: async () => { } };
         }
-        return new ToolProgressReporter(token, sendNotification);
+        return new ToolProgressReporter(token, sendNotification, relatedTaskMeta);
     }
     async report(progress, message) {
         try {
@@ -75,6 +103,13 @@ class ToolProgressReporter {
                         progress,
                         total: FETCH_PROGRESS_TOTAL,
                         message,
+                        ...(this.relatedTaskMeta
+                            ? {
+                                _meta: {
+                                    'io.modelcontextprotocol/related-task': this.relatedTaskMeta,
+                                },
+                            }
+                            : {}),
                     },
                 }),
                 new Promise((_, reject) => {
@@ -461,7 +496,6 @@ async function executeFetch(input, extra) {
     await progress.report(1, 'Validating URL');
     logDebug('Fetching URL', { url });
     await progress.report(2, 'Fetching content');
-    await progress.report(2, 'Fetching content'); // preserve existing behavior
     const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress);
     if (pipeline.fromCache) {
         await progress.report(3, 'Using cached content');
@@ -485,6 +519,9 @@ const TOOL_DEFINITION = {
     inputSchema: fetchUrlInputSchema,
     outputSchema: fetchUrlOutputSchema,
     handler: fetchUrlToolHandler,
+    execution: {
+        taskSupport: 'optional',
+    },
     annotations: {
         readOnlyHint: true,
         destructiveHint: false,
@@ -499,7 +536,12 @@ export function withRequestContextIfMissing(handler) {
             return handler(params, extra);
         }
         const derivedRequestId = resolveRequestIdFromExtra(extra) ?? randomUUID();
-        return runWithRequestContext({ requestId: derivedRequestId, operationId: derivedRequestId }, () => handler(params, extra));
+        const derivedSessionId = resolveSessionIdFromExtra(extra);
+        return runWithRequestContext({
+            requestId: derivedRequestId,
+            operationId: derivedRequestId,
+            ...(derivedSessionId ? { sessionId: derivedSessionId } : {}),
+        }, () => handler(params, extra));
     };
 }
 function resolveRequestIdFromExtra(extra) {
@@ -512,13 +554,32 @@ function resolveRequestIdFromExtra(extra) {
         return String(requestId);
     return undefined;
 }
-export function registerTools(server, serverIcons) {
-    server.registerTool(TOOL_DEFINITION.name, {
-        title: TOOL_DEFINITION.title,
-        description: TOOL_DEFINITION.description,
-        inputSchema: TOOL_DEFINITION.inputSchema,
-        outputSchema: TOOL_DEFINITION.outputSchema,
-        annotations: TOOL_DEFINITION.annotations,
-        ...(serverIcons ? { icons: serverIcons } : {}),
-    }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
+function resolveSessionIdFromExtra(extra) {
+    if (!isObject(extra))
+        return undefined;
+    const { sessionId } = extra;
+    if (typeof sessionId === 'string')
+        return sessionId;
+    const { requestInfo } = extra;
+    if (!isObject(requestInfo))
+        return undefined;
+    const { headers } = requestInfo;
+    if (!isObject(headers))
+        return undefined;
+    const headerValue = headers['mcp-session-id'];
+    return typeof headerValue === 'string' ? headerValue : undefined;
+}
+export function registerTools(server) {
+    if (config.tools.enabled.includes(FETCH_URL_TOOL_NAME)) {
+        server.registerTool(TOOL_DEFINITION.name, {
+            title: TOOL_DEFINITION.title,
+            description: TOOL_DEFINITION.description,
+            inputSchema: TOOL_DEFINITION.inputSchema,
+            outputSchema: TOOL_DEFINITION.outputSchema,
+            annotations: TOOL_DEFINITION.annotations,
+            execution: TOOL_DEFINITION.execution,
+            // Use specific tool icon here
+            icons: [TOOL_ICON],
+        }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
+    }
 }

package/dist/transform.js

@@ -876,8 +876,8 @@ function buildContentSource(params) {
         return { sourceHtml: article.content, title: article.title, metadata };
     }
     if (document) {
-        const cleanedHtml = removeNoiseFromHtml(html, undefined, url);
-        const { document: cleanedDoc } = parseHTML(cleanedHtml);
+        removeNoiseFromHtml(html, document, url);
+        const cleanedDoc = document;
         const contentRoot = findContentRoot(cleanedDoc);
         if (contentRoot) {
             logDebug('Using content root fallback instead of full HTML', {
@@ -993,6 +993,7 @@ class WorkerPool {
     minCapacity = POOL_MIN_WORKERS;
     maxCapacity = POOL_MAX_WORKERS;
     queue = [];
+    queueHead = 0;
     inflight = new Map();
     timeoutMs;
     queueMax;
@@ -1006,7 +1007,7 @@ class WorkerPool {
         this.ensureOpen();
         if (options.signal?.aborted)
             throw abortPolicy.createAbortError(url, 'transform:enqueue');
-        if (this.queue.length >= this.queueMax) {
+        if (this.getQueueDepth() >= this.queueMax) {
             throw new FetchError('Transform worker queue is full', url, 503, {
                 reason: 'queue_full',
                 stage: 'transform:enqueue',
@@ -1019,7 +1020,8 @@ class WorkerPool {
         });
     }
     getQueueDepth() {
-        return this.queue.length;
+        const depth = this.queue.length - this.queueHead;
+        return depth > 0 ? depth : 0;
     }
     getActiveWorkers() {
         return this.workers.filter((s) => s?.busy).length;
@@ -1042,9 +1044,13 @@ class WorkerPool {
             inflight.reject(new Error('Transform worker pool closed'));
             this.inflight.delete(id);
         }
-        for (const task of this.queue)
-            task.reject(new Error('Transform worker pool closed'));
+        for (let i = this.queueHead; i < this.queue.length; i += 1) {
+            const task = this.queue[i];
+            if (task)
+                task.reject(new Error('Transform worker pool closed'));
+        }
         this.queue.length = 0;
+        this.queueHead = 0;
         await Promise.allSettled(terminations);
     }
     ensureOpen() {
@@ -1081,10 +1087,11 @@ class WorkerPool {
             this.abortInflight(id, url, inflight.workerIndex);
             return;
         }
-        const queuedIndex = this.queue.findIndex((t) => t.id === id);
-        if (queuedIndex !== -1) {
+        const queuedIndex = this.findQueuedIndex(id);
+        if (queuedIndex !== null) {
             this.queue.splice(queuedIndex, 1);
             reject(abortPolicy.createAbortError(url, 'transform:queued-abort'));
+            this.maybeCompactQueue();
         }
     }
     abortInflight(id, url, workerIndex) {
@@ -1196,29 +1203,29 @@ class WorkerPool {
         this.markIdle(inflight.workerIndex);
     }
     maybeScaleUp() {
-        if (this.queue.length > this.capacity * POOL_SCALE_THRESHOLD &&
+        if (this.getQueueDepth() > this.capacity * POOL_SCALE_THRESHOLD &&
             this.capacity < this.maxCapacity) {
             this.capacity += 1;
         }
     }
     drainQueue() {
-        if (this.closed || this.queue.length === 0)
+        if (this.closed || this.getQueueDepth() === 0)
             return;
         this.maybeScaleUp();
         for (let i = 0; i < this.workers.length; i += 1) {
             const slot = this.workers[i];
             if (slot && !slot.busy) {
                 this.dispatchFromQueue(i, slot);
-                if (this.queue.length === 0)
+                if (this.getQueueDepth() === 0)
                     return;
             }
         }
-        if (this.workers.length < this.capacity && this.queue.length > 0) {
+        if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
             const workerIndex = this.workers.length;
             const slot = this.spawnWorker(workerIndex);
             this.workers.push(slot);
             this.dispatchFromQueue(workerIndex, slot);
-            if (this.workers.length < this.capacity && this.queue.length > 0) {
+            if (this.workers.length < this.capacity && this.getQueueDepth() > 0) {
                 setImmediate(() => {
                     this.drainQueue();
                 });
@@ -1226,9 +1233,11 @@ class WorkerPool {
         }
     }
     dispatchFromQueue(workerIndex, slot) {
-        const task = this.queue.shift();
+        const task = this.queue[this.queueHead];
         if (!task)
             return;
+        this.queueHead += 1;
+        this.maybeCompactQueue();
         if (this.closed) {
             task.reject(new Error('Transform worker pool closed'));
             return;
@@ -1285,6 +1294,23 @@ class WorkerPool {
             this.restartWorker(workerIndex, slot);
         }
     }
+    findQueuedIndex(id) {
+        for (let i = this.queueHead; i < this.queue.length; i += 1) {
+            const task = this.queue[i];
+            if (task?.id === id)
+                return i;
+        }
+        return null;
+    }
+    maybeCompactQueue() {
+        if (this.queueHead === 0)
+            return;
+        if (this.queueHead >= this.queue.length ||
+            (this.queueHead > 1024 && this.queueHead > this.queue.length / 2)) {
+            this.queue.splice(0, this.queueHead);
+            this.queueHead = 0;
+        }
+    }
 }
 class TransformWorkerPoolManager {
     pool = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.4.4",
+  "version": "2.4.6",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",