@j0hanz/superfetch 2.4.2 → 2.4.4

package/dist/markdown-cleanup.js

@@ -1,7 +1,7 @@
 import { config } from './config.js';
-// ─────────────────────────────────────────────────────────────────────────────
-// Fence state helpers
-// ─────────────────────────────────────────────────────────────────────────────
+/* -------------------------------------------------------------------------------------------------
+ * Fences
+ * ------------------------------------------------------------------------------------------------- */
 function isFenceStart(line) {
     const trimmed = line.trimStart();
     return trimmed.startsWith('```') || trimmed.startsWith('~~~');
@@ -29,49 +29,121 @@ function advanceFenceState(line, state) {
         state.marker = '';
     }
 }
-// ─────────────────────────────────────────────────────────────────────────────
-// Segment utilities
-// ─────────────────────────────────────────────────────────────────────────────
-/**
- * Split markdown into segments where each segment is either fully inside
- * a fenced block (including the fence lines), or fully outside.
- */
-function splitByFences(content) {
-    const lines = content.split('\n');
-    const segments = [];
-    const state = initialFenceState();
-    let current = [];
-    let currentIsFence = false;
-    for (const line of lines) {
-        // Transition into fence: flush outside segment first.
-        if (!state.inFence && isFenceStart(line)) {
-            if (current.length > 0) {
-                segments.push({ content: current.join('\n'), inFence: currentIsFence });
-                current = [];
+class FencedSegmenter {
+    split(content) {
+        const lines = content.split('\n');
+        const segments = [];
+        const state = initialFenceState();
+        let current = [];
+        let currentIsFence = false;
+        for (const line of lines) {
+            // Transition into fence: flush outside segment first.
+            if (!state.inFence && isFenceStart(line)) {
+                if (current.length > 0) {
+                    segments.push({
+                        content: current.join('\n'),
+                        inFence: currentIsFence,
+                    });
+                    current = [];
+                }
+                currentIsFence = true;
+                current.push(line);
+                advanceFenceState(line, state);
+                continue;
             }
-            currentIsFence = true;
             current.push(line);
+            const wasInFence = state.inFence;
             advanceFenceState(line, state);
-            continue;
+            // Transition out of fence: flush fence segment.
+            if (wasInFence && !state.inFence) {
+                segments.push({ content: current.join('\n'), inFence: true });
+                current = [];
+                currentIsFence = false;
+            }
         }
-        current.push(line);
-        const wasInFence = state.inFence;
-        advanceFenceState(line, state);
-        // Transition out of fence: flush fence segment.
-        if (wasInFence && !state.inFence) {
-            segments.push({ content: current.join('\n'), inFence: true });
-            current = [];
-            currentIsFence = false;
+        if (current.length > 0) {
+            segments.push({ content: current.join('\n'), inFence: currentIsFence });
+        }
+        return segments;
+    }
+}
+const fencedSegmenter = new FencedSegmenter();
+/* -------------------------------------------------------------------------------------------------
+ * Orphan heading promotion
+ * ------------------------------------------------------------------------------------------------- */
+const HEADING_KEYWORDS = new Set([
+    'overview',
+    'introduction',
+    'summary',
+    'conclusion',
+    'prerequisites',
+    'requirements',
+    'installation',
+    'configuration',
+    'usage',
+    'features',
+    'limitations',
+    'troubleshooting',
+    'faq',
+    'resources',
+    'references',
+    'changelog',
+    'license',
+    'acknowledgments',
+    'appendix',
+]);
+class OrphanHeadingPromoter {
+    shouldPromote(line, prevLine) {
+        const isPrecededByBlank = prevLine.trim() === '';
+        if (!isPrecededByBlank)
+            return false;
+        return this.isLikelyHeadingLine(line);
+    }
+    format(line) {
+        const trimmed = line.trim();
+        const isExample = /^example:\s/i.test(trimmed);
+        const prefix = isExample ? '### ' : '## ';
+        return prefix + trimmed;
+    }
+    processLine(line, prevLine) {
+        if (this.shouldPromote(line, prevLine)) {
+            return this.format(line);
         }
+        return line;
     }
-    if (current.length > 0) {
-        segments.push({ content: current.join('\n'), inFence: currentIsFence });
+    isLikelyHeadingLine(line) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.length > 80)
+            return false;
+        if (/^#{1,6}\s/.test(trimmed))
+            return false;
+        if (/^[-*+•]\s/.test(trimmed) || /^\d+\.\s/.test(trimmed))
+            return false;
+        if (/[.!?]$/.test(trimmed))
+            return false;
+        if (/^\[.*\]\(.*\)$/.test(trimmed))
+            return false;
+        if (/^(?:example|note|tip|warning|important|caution):\s+\S/i.test(trimmed)) {
+            return true;
+        }
+        const words = trimmed.split(/\s+/);
+        if (words.length >= 2 && words.length <= 6) {
+            const isTitleCase = words.every((w) => /^[A-Z][a-z]*$/.test(w) || /^(?:and|or|the|of|in|for|to|a)$/i.test(w));
+            if (isTitleCase)
+                return true;
+        }
+        if (words.length === 1) {
+            const lower = trimmed.toLowerCase();
+            if (HEADING_KEYWORDS.has(lower) && /^[A-Z]/.test(trimmed))
+                return true;
+        }
+        return false;
     }
-    return segments;
 }
-// ─────────────────────────────────────────────────────────────────────────────
-// Cleanup rules (OUTSIDE fences only)
-// ─────────────────────────────────────────────────────────────────────────────
+const orphanHeadingPromoter = new OrphanHeadingPromoter();
+/* -------------------------------------------------------------------------------------------------
+ * Cleanup rules (OUTSIDE fences only)
+ * ------------------------------------------------------------------------------------------------- */
 function removeEmptyHeadings(text) {
     return text.replace(/^#{1,6}[ \t\u00A0]*$\r?\n?/gm, '');
 }
@@ -88,7 +160,7 @@ function fixOrphanHeadings(text) {
     });
 }
 function removeSkipLinksAndEmptyAnchors(text) {
-    const zeroWidthAnchorLink = /\[(?:\s|\u200B)*\]\(#[^)]*\)\s*/g;
+    const zeroWidthAnchorLink = /\[(?:\s|\u200B)*\]\(#[^)]*\)[ \t]*/g;
     return text
         .replace(zeroWidthAnchorLink, '')
         .replace(/^\[Skip to (?:main )?content\]\(#[^)]*\)\s*$/gim, '')
@@ -174,39 +246,43 @@ const CLEANUP_STEPS = [
     normalizeListsAndSpacing,
     fixConcatenatedProperties,
 ];
-// ─────────────────────────────────────────────────────────────────────────────
-// Public API
-// ─────────────────────────────────────────────────────────────────────────────
 function getLastLine(text) {
     const index = text.lastIndexOf('\n');
     return index === -1 ? text : text.slice(index + 1);
 }
+class MarkdownCleanupPipeline {
+    cleanup(markdown) {
+        if (!markdown)
+            return '';
+        const segments = fencedSegmenter.split(markdown);
+        const cleaned = segments
+            .map((seg, index) => {
+            if (seg.inFence)
+                return seg.content;
+            const prevSeg = segments[index - 1];
+            const prevLineContext = prevSeg ? getLastLine(prevSeg.content) : '';
+            const lines = seg.content.split('\n');
+            const promotedLines = [];
+            for (let i = 0; i < lines.length; i += 1) {
+                const line = lines[i] ?? '';
+                const prevLine = i > 0 ? (lines[i - 1] ?? '') : prevLineContext;
+                promotedLines.push(orphanHeadingPromoter.processLine(line, prevLine));
+            }
+            const promoted = promotedLines.join('\n');
+            return CLEANUP_STEPS.reduce((text, step) => step(text), promoted);
+        })
+            .join('\n')
+            .trim();
+        return cleaned;
+    }
+}
+const markdownCleanupPipeline = new MarkdownCleanupPipeline();
 export function cleanupMarkdownArtifacts(content) {
-    if (!content)
-        return '';
-    const segments = splitByFences(content);
-    return segments
-        .map((seg, index) => {
-        if (seg.inFence)
-            return seg.content;
-        const prevSeg = segments[index - 1];
-        const prevLineContext = prevSeg ? getLastLine(prevSeg.content) : '';
-        const lines = seg.content.split('\n');
-        const promotedLines = [];
-        for (let i = 0; i < lines.length; i += 1) {
-            const line = lines[i] ?? '';
-            const prevLine = i > 0 ? (lines[i - 1] ?? '') : prevLineContext;
-            promotedLines.push(processNonFencedLine(line, prevLine));
-        }
-        const promoted = promotedLines.join('\n');
-        return CLEANUP_STEPS.reduce((text, step) => step(text), promoted);
-    })
-        .join('\n')
-        .trim();
-}
-// ─────────────────────────────────────────────────────────────────────────────
-// Raw markdown handling + metadata footer
-// ─────────────────────────────────────────────────────────────────────────────
+    return markdownCleanupPipeline.cleanup(content);
+}
+/* -------------------------------------------------------------------------------------------------
+ * Raw markdown handling + metadata footer
+ * ------------------------------------------------------------------------------------------------- */
 const HEADING_PATTERN = /^#{1,6}\s/m;
 const LIST_PATTERN = /^(?:[-*+])\s/m;
 const HTML_DOCUMENT_PATTERN = /^(<!doctype|<html)/i;
@@ -231,16 +307,22 @@ function detectLineEnding(content) {
     return content.includes('\r\n') ? '\r\n' : '\n';
 }
 const FRONTMATTER_DELIMITER = '---';
-function findFrontmatterLines(content) {
-    const lineEnding = detectLineEnding(content);
-    const lines = content.split(lineEnding);
-    if (lines[0] !== FRONTMATTER_DELIMITER)
-        return null;
-    const endIndex = lines.indexOf(FRONTMATTER_DELIMITER, 1);
-    if (endIndex === -1)
-        return null;
-    return { lineEnding, lines, endIndex };
+class RawMarkdownFrontmatter {
+    find(content) {
+        const lineEnding = detectLineEnding(content);
+        const lines = content.split(lineEnding);
+        if (lines[0] !== FRONTMATTER_DELIMITER)
+            return null;
+        const endIndex = lines.indexOf(FRONTMATTER_DELIMITER, 1);
+        if (endIndex === -1)
+            return null;
+        return { lineEnding, lines, endIndex };
+    }
+    hasFrontmatter(trimmed) {
+        return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+    }
 }
+const frontmatter = new RawMarkdownFrontmatter();
 function stripOptionalQuotes(value) {
     const trimmed = value.trim();
     if (trimmed.length < 2)
@@ -288,11 +370,11 @@ function extractTitleFromHeading(content) {
     return undefined;
 }
 export function extractTitleFromRawMarkdown(content) {
-    const frontmatter = findFrontmatterLines(content);
-    if (!frontmatter) {
+    const fm = frontmatter.find(content);
+    if (!fm) {
         return extractTitleFromHeading(content);
     }
-    const { lines, endIndex } = frontmatter;
+    const { lines, endIndex } = fm;
     const entry = lines
         .slice(1, endIndex)
         .map((line) => parseFrontmatterEntry(line))
@@ -339,14 +421,15 @@ function addSourceToMarkdownMarkdownFormat(content, url) {
     return [`Source: ${url}`, '', content].join(lineEnding);
 }
 export function addSourceToMarkdown(content, url) {
-    const frontmatter = findFrontmatterLines(content);
-    if (config.transform.metadataFormat === 'markdown' && !frontmatter) {
+    const fm = frontmatter.find(content);
+    if (config.transform.metadataFormat === 'markdown' && !fm) {
         return addSourceToMarkdownMarkdownFormat(content, url);
     }
-    if (!frontmatter) {
+    if (!fm) {
+        // Preserve existing behavior: always uses LF even if content uses CRLF.
         return `---\nsource: "${url}"\n---\n\n${content}`;
     }
-    const { lineEnding, lines, endIndex } = frontmatter;
+    const { lineEnding, lines, endIndex } = fm;
     const bodyLines = lines.slice(1, endIndex);
     const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
     if (hasSource)
@@ -359,9 +442,6 @@ export function addSourceToMarkdown(content, url) {
     ];
     return updatedLines.join(lineEnding);
 }
-function hasFrontmatter(trimmed) {
-    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
-}
 function looksLikeHtmlDocument(trimmed) {
     return HTML_DOCUMENT_PATTERN.test(trimmed);
 }
@@ -373,7 +453,7 @@ function countCommonHtmlTags(content) {
 export function isRawTextContent(content) {
     const trimmed = content.trim();
     const isHtmlDocument = looksLikeHtmlDocument(trimmed);
-    const hasMarkdownFrontmatter = hasFrontmatter(trimmed);
+    const hasMarkdownFrontmatter = frontmatter.hasFrontmatter(trimmed);
     const hasTooManyHtmlTags = countCommonHtmlTags(content) > 2;
     const isMarkdown = looksLikeMarkdown(content);
     return (!isHtmlDocument &&
@@ -423,76 +503,9 @@ export function buildMetadataFooter(metadata, fallbackUrl) {
     }
     return lines.join('\n');
 }
-// ─────────────────────────────────────────────────────────────────────────────
-// Heading Promotion (fence-aware)
-// ─────────────────────────────────────────────────────────────────────────────
-const HEADING_KEYWORDS = new Set([
-    'overview',
-    'introduction',
-    'summary',
-    'conclusion',
-    'prerequisites',
-    'requirements',
-    'installation',
-    'configuration',
-    'usage',
-    'features',
-    'limitations',
-    'troubleshooting',
-    'faq',
-    'resources',
-    'references',
-    'changelog',
-    'license',
-    'acknowledgments',
-    'appendix',
-]);
-function isLikelyHeadingLine(line) {
-    const trimmed = line.trim();
-    if (!trimmed || trimmed.length > 80)
-        return false;
-    if (/^#{1,6}\s/.test(trimmed))
-        return false;
-    if (/^[-*+•]\s/.test(trimmed) || /^\d+\.\s/.test(trimmed))
-        return false;
-    if (/[.!?]$/.test(trimmed))
-        return false;
-    if (/^\[.*\]\(.*\)$/.test(trimmed))
-        return false;
-    if (/^(?:example|note|tip|warning|important|caution):\s+\S/i.test(trimmed)) {
-        return true;
-    }
-    const words = trimmed.split(/\s+/);
-    if (words.length >= 2 && words.length <= 6) {
-        const isTitleCase = words.every((w) => /^[A-Z][a-z]*$/.test(w) || /^(?:and|or|the|of|in|for|to|a)$/i.test(w));
-        if (isTitleCase)
-            return true;
-    }
-    if (words.length === 1) {
-        const lower = trimmed.toLowerCase();
-        if (HEADING_KEYWORDS.has(lower) && /^[A-Z]/.test(trimmed))
-            return true;
-    }
-    return false;
-}
-function shouldPromoteToHeading(line, prevLine) {
-    const isPrecededByBlank = prevLine.trim() === '';
-    if (!isPrecededByBlank)
-        return false;
-    return isLikelyHeadingLine(line);
-}
-function formatAsHeading(line) {
-    const trimmed = line.trim();
-    const isExample = /^example:\s/i.test(trimmed);
-    const prefix = isExample ? '### ' : '## ';
-    return prefix + trimmed;
-}
-function processNonFencedLine(line, prevLine) {
-    if (shouldPromoteToHeading(line, prevLine)) {
-        return formatAsHeading(line);
-    }
-    return line;
-}
+/* -------------------------------------------------------------------------------------------------
+ * Heading promotion (fence-aware)
+ * ------------------------------------------------------------------------------------------------- */
 /**
  * Promote standalone lines that look like headings to proper markdown headings.
  * Fence-aware: never modifies content inside fenced code blocks.
@@ -511,7 +524,7 @@ export function promoteOrphanHeadings(markdown) {
             advanceFenceState(line, state);
             continue;
         }
-        result.push(processNonFencedLine(line, prevLine));
+        result.push(orphanHeadingPromoter.processLine(line, prevLine));
     }
     return result.join('\n');
 }

package/dist/mcp.js

@@ -1,34 +1,39 @@
 import { readFileSync } from 'node:fs';
+import { z } from 'zod';
 import { McpServer, ResourceTemplate, } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
 import { registerCachedContentResource } from './cache.js';
 import { config } from './config.js';
 import { destroyAgents } from './fetch.js';
 import { logError, logInfo, setMcpServer } from './observability.js';
-import { registerTools } from './tools.js';
+import { registerConfigResource } from './resources.js';
+import { taskManager } from './tasks.js';
+import { FETCH_URL_TOOL_NAME, fetchUrlToolHandler, registerTools, } from './tools.js';
 import { shutdownTransformWorkerPool } from './transform.js';
-function getLocalIconData() {
+import { isObject } from './type-guards.js';
+function getLocalIcons() {
     try {
         const iconPath = new URL('../assets/logo.svg', import.meta.url);
         const buffer = readFileSync(iconPath);
-        return `data:image/svg+xml;base64,${buffer.toString('base64')}`;
+        return [
+            {
+                src: `data:image/svg+xml;base64,${buffer.toString('base64')}`,
+                mimeType: 'image/svg+xml',
+                sizes: ['any'],
+            },
+        ];
     }
     catch {
         return undefined;
     }
 }
 function createServerInfo() {
-    const localIcon = getLocalIconData();
+    const localIcons = getLocalIcons();
     return {
         name: config.server.name,
         version: config.server.version,
-        ...(localIcon
-            ? {
-                icons: [
-                    { src: localIcon, mimeType: 'image/svg+xml', sizes: ['any'] },
-                ],
-            }
-            : {}),
+        ...(localIcons ? { icons: localIcons } : {}),
     };
 }
 function createServerCapabilities() {
@@ -36,6 +41,15 @@ function createServerCapabilities() {
         tools: { listChanged: true },
         resources: { listChanged: true, subscribe: true },
         logging: {},
+        tasks: {
+            list: {},
+            cancel: {},
+            requests: {
+                tools: {
+                    call: {},
+                },
+            },
+        },
     };
 }
 function createServerInstructions(serverVersion) {
@@ -62,6 +76,150 @@ function registerInstructionsResource(server, instructions) {
         ],
     }));
 }
+// Schemas based on methods strings
+const TaskGetSchema = z.object({ method: z.literal('tasks/get') });
+const TaskListSchema = z.object({ method: z.literal('tasks/list') });
+const TaskCancelSchema = z.object({ method: z.literal('tasks/cancel') });
+const TaskResultSchema = z.object({ method: z.literal('tasks/result') });
+function registerTaskHandlers(server) {
+    server.server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const extendedParams = request
+            .params;
+        const taskOptions = extendedParams.task;
+        if (taskOptions) {
+            // Validate tool support
+            if (extendedParams.name !== FETCH_URL_TOOL_NAME) {
+                throw new Error(`Tool '${extendedParams.name}' does not support task execution`);
+            }
+            // Create Task
+            const task = taskManager.createTask(taskOptions.ttl !== undefined ? { ttl: taskOptions.ttl } : undefined);
+            // Start Async Execution
+            void (async () => {
+                try {
+                    const args = extendedParams.arguments;
+                    if (!isObject(args) ||
+                        typeof args.url !== 'string') {
+                        throw new Error('Invalid arguments for fetch-url');
+                    }
+                    const validArgs = args;
+                    const controller = new AbortController();
+                    const result = await fetchUrlToolHandler(validArgs, {
+                        signal: controller.signal,
+                        requestId: task.taskId, // Correlation
+                        ...(extendedParams._meta ? { _meta: extendedParams._meta } : {}),
+                    });
+                    // Update Task on Success
+                    taskManager.updateTask(task.taskId, {
+                        status: 'completed',
+                        result,
+                    });
+                }
+                catch (error) {
+                    // Update Task on Failure
+                    taskManager.updateTask(task.taskId, {
+                        status: 'failed',
+                        statusMessage: error instanceof Error ? error.message : String(error),
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            })();
+            // Return Immediate CreateTaskResult
+            const response = {
+                task: {
+                    taskId: task.taskId,
+                    status: task.status,
+                    ...(task.statusMessage ? { statusMessage: task.statusMessage } : {}),
+                    createdAt: task.createdAt,
+                    lastUpdatedAt: task.lastUpdatedAt,
+                    ttl: task.ttl,
+                    pollInterval: task.pollInterval,
+                },
+            };
+            return response;
+        }
+        if (extendedParams.name === FETCH_URL_TOOL_NAME) {
+            const args = extendedParams.arguments;
+            if (!isObject(args) ||
+                typeof args.url !== 'string') {
+                throw new Error('Invalid arguments for fetch-url');
+            }
+            return fetchUrlToolHandler({ url: args.url }, {
+                ...(extendedParams._meta ? { _meta: extendedParams._meta } : {}),
+            });
+        }
+        throw new Error(`Tool not found: ${extendedParams.name}`);
+    });
+    server.server.setRequestHandler(TaskGetSchema, async (request) => {
+        const { taskId } = request.params;
+        const task = taskManager.getTask(taskId);
+        if (!task) {
+            throw new Error('Task not found');
+        }
+        return Promise.resolve({
+            taskId: task.taskId,
+            status: task.status,
+            statusMessage: task.statusMessage,
+            createdAt: task.createdAt,
+            lastUpdatedAt: task.lastUpdatedAt,
+            ttl: task.ttl,
+            pollInterval: task.pollInterval,
+        });
+    });
+    server.server.setRequestHandler(TaskResultSchema, async (request) => {
+        const { taskId } = request.params;
+        const task = taskManager.getTask(taskId);
+        if (!task) {
+            throw new Error('Task not found');
+        }
+        if (task.status === 'working' || task.status === 'input_required') {
+            throw new Error('Task execution in progress');
+        }
+        if (task.status === 'failed') {
+            return Promise.resolve(task.result ?? { isError: true, content: [] });
+        }
+        if (task.status === 'cancelled') {
+            throw new Error('Task was cancelled');
+        }
+        const result = (task.result ?? { content: [] });
+        return Promise.resolve({
+            ...result,
+            _meta: {
+                ...result._meta,
+                'io.modelcontextprotocol/related-task': { taskId: task.taskId },
+            },
+        });
+    });
+    server.server.setRequestHandler(TaskListSchema, async () => {
+        const tasks = taskManager.listTasks();
+        return Promise.resolve({
+            tasks: tasks.map((t) => ({
+                taskId: t.taskId,
+                status: t.status,
+                createdAt: t.createdAt,
+                lastUpdatedAt: t.lastUpdatedAt,
+                ttl: t.ttl,
+                pollInterval: t.pollInterval,
+            })),
+            nextCursor: undefined,
+        });
+    });
+    server.server.setRequestHandler(TaskCancelSchema, async (request) => {
+        const { taskId } = request.params;
+        const task = taskManager.cancelTask(taskId);
+        if (!task) {
+            throw new Error('Task not found');
+        }
+        return Promise.resolve({
+            taskId: task.taskId,
+            status: task.status,
+            statusMessage: task.statusMessage,
+            createdAt: task.createdAt,
+            lastUpdatedAt: task.lastUpdatedAt,
+            ttl: task.ttl,
+            pollInterval: task.pollInterval,
+        });
+    });
+}
 export function createMcpServer() {
     const instructions = createServerInstructions(config.server.version);
     const server = new McpServer(createServerInfo(), {
@@ -69,10 +227,12 @@ export function createMcpServer() {
         instructions,
     });
     setMcpServer(server);
-    const localIcon = getLocalIconData();
-    registerTools(server, localIcon);
-    registerCachedContentResource(server, localIcon);
+    const localIcons = getLocalIcons();
+    registerTools(server, localIcons);
+    registerCachedContentResource(server, localIcons);
     registerInstructionsResource(server, instructions);
+    registerConfigResource(server);
+    registerTaskHandlers(server);
     return server;
 }
 function attachServerErrorHandler(server) {

package/dist/resources.d.ts

@@ -0,0 +1,2 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function registerConfigResource(server: McpServer): void;