@j0hanz/superfetch 2.4.13 → 2.5.1

package/dist/fetch.d.ts

@@ -1,4 +1,3 @@
-import type { Dispatcher } from 'undici';
 export interface FetchOptions {
     signal?: AbortSignal;
 }
@@ -18,8 +17,6 @@ export declare function validateAndNormalizeUrl(urlString: string): string;
 /** Backwards-compatible exports */
 export declare function transformToRawUrl(url: string): TransformResult;
 export declare function isRawTextContentUrl(url: string): boolean;
-export declare const dispatcher: Dispatcher;
-export declare function destroyAgents(): void;
 export interface FetchTelemetryContext {
     requestId: string;
     startTime: number;

package/dist/fetch.js

@@ -2,9 +2,7 @@ import { randomUUID } from 'node:crypto';
 import diagnosticsChannel from 'node:diagnostics_channel';
 import dns from 'node:dns';
 import { BlockList, isIP } from 'node:net';
-import os from 'node:os';
 import { performance } from 'node:perf_hooks';
-import { Agent } from 'undici';
 import { config } from './config.js';
 import { createErrorWithCode, FetchError, isSystemError } from './errors.js';
 import { getOperationId, getRequestId, logDebug, logError, logWarn, redactUrl, } from './observability.js';
@@ -435,21 +433,16 @@ class SafeDnsLookup {
     }
 }
 const safeDns = new SafeDnsLookup();
-/* -------------------------------------------------------------------------------------------------
- * Dispatcher / Agent lifecycle
- * ------------------------------------------------------------------------------------------------- */
-function getAgentOptions() {
-    const cpuCount = os.availableParallelism();
-    return {
-        keepAliveTimeout: 60000,
-        connections: Math.max(cpuCount * 2, 25),
-        pipelining: 1,
-        connect: { lookup: safeDns.lookup.bind(safeDns) },
-    };
-}
-export const dispatcher = new Agent(getAgentOptions());
-export function destroyAgents() {
-    void dispatcher.close();
+async function assertSafeDnsLookup(hostname) {
+    await new Promise((resolve, reject) => {
+        safeDns.lookup(hostname, { all: true }, (err) => {
+            if (err) {
+                reject(err);
+                return;
+            }
+            resolve();
+        });
+    });
 }
 /* -------------------------------------------------------------------------------------------------
  * Fetch error mapping (request-level)
@@ -834,7 +827,7 @@ function buildRequestSignal(timeoutMs, external) {
     return external ? AbortSignal.any([external, timeoutSignal]) : timeoutSignal;
 }
 function buildRequestInit(headers, signal) {
-    return { method: 'GET', headers, signal, dispatcher };
+    return { method: 'GET', headers, signal };
 }
 function resolveResponseError(response, finalUrl) {
     if (response.status === 429) {
@@ -856,6 +849,8 @@ async function handleFetchResponse(response, finalUrl, ctx, signal) {
 }
 class HttpFetcher {
     async fetchNormalizedUrl(normalizedUrl, options) {
+        const { hostname } = new URL(normalizedUrl);
+        await assertSafeDnsLookup(hostname);
         const timeoutMs = config.fetcher.timeout;
         const headers = buildHeaders();
         const signal = buildRequestSignal(timeoutMs, options?.signal);

package/dist/language-detection.js

@@ -319,6 +319,8 @@ const detector = new LanguageDetector();
  * Detect programming language from code content using heuristics.
  */
 export function detectLanguageFromCode(code) {
+    if (!code || code.trim().length === 0)
+        return undefined;
     return detector.detect(code);
 }
 /**

package/dist/markdown-cleanup.js

@@ -398,7 +398,7 @@ function hasMarkdownSourceLine(content) {
     }
     return false;
 }
-function addSourceToMarkdownMarkdownFormat(content, url) {
+function addSourceToMarkdownAsMarkdown(content, url) {
     if (hasMarkdownSourceLine(content))
         return content;
     const lineEnding = detectLineEnding(content);
@@ -423,7 +423,7 @@ function addSourceToMarkdownMarkdownFormat(content, url) {
 export function addSourceToMarkdown(content, url) {
     const fm = frontmatter.find(content);
     if (config.transform.metadataFormat === 'markdown' && !fm) {
-        return addSourceToMarkdownMarkdownFormat(content, url);
+        return addSourceToMarkdownAsMarkdown(content, url);
     }
     if (!fm) {
         // Preserve existing behavior: always uses LF even if content uses CRLF.

package/dist/mcp.js

@@ -5,7 +5,6 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
 import { CallToolRequestSchema, ErrorCode, McpError, } from '@modelcontextprotocol/sdk/types.js';
 import { registerCachedContentResource } from './cache.js';
 import { config } from './config.js';
-import { destroyAgents } from './fetch.js';
 import { logError, logInfo, setMcpServer } from './observability.js';
 import { registerConfigResource } from './resources.js';
 import { taskManager } from './tasks.js';
@@ -415,7 +414,6 @@ function handleShutdownSignal(server, signal) {
     process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
     Promise.resolve()
         .then(async () => {
-        destroyAgents();
         await shutdownTransformWorkerPool();
         await server.close();
     })

package/dist/tools.js

@@ -95,29 +95,33 @@ class ToolProgressReporter {
     }
     async report(progress, message) {
         try {
-            await Promise.race([
-                this.sendNotification({
-                    method: 'notifications/progress',
-                    params: {
-                        progressToken: this.token,
-                        progress,
-                        total: FETCH_PROGRESS_TOTAL,
-                        message,
-                        ...(this.relatedTaskMeta
-                            ? {
-                                _meta: {
-                                    'io.modelcontextprotocol/related-task': this.relatedTaskMeta,
-                                },
-                            }
-                            : {}),
-                    },
-                }),
-                new Promise((_, reject) => {
-                    setTimeout(() => {
-                        reject(new Error('Progress notification timeout'));
-                    }, PROGRESS_NOTIFICATION_TIMEOUT_MS);
-                }),
-            ]);
+            let timeoutId;
+            const timeoutPromise = new Promise((_, reject) => {
+                timeoutId = setTimeout(() => {
+                    reject(new Error('Progress notification timeout'));
+                }, PROGRESS_NOTIFICATION_TIMEOUT_MS);
+                timeoutId.unref();
+            });
+            const sendPromise = this.sendNotification({
+                method: 'notifications/progress',
+                params: {
+                    progressToken: this.token,
+                    progress,
+                    total: FETCH_PROGRESS_TOTAL,
+                    message,
+                    ...(this.relatedTaskMeta
+                        ? {
+                            _meta: {
+                                'io.modelcontextprotocol/related-task': this.relatedTaskMeta,
+                            },
+                        }
+                        : {}),
+                },
+            }).finally(() => {
+                if (timeoutId)
+                    clearTimeout(timeoutId);
+            });
+            await Promise.race([sendPromise, timeoutPromise]);
         }
         catch (error) {
             const isTimeout = error instanceof Error &&

package/dist/transform.js

@@ -630,6 +630,10 @@ function translateHtmlToMarkdown(params) {
 }
 function appendMetadataFooter(content, metadata, url) {
     const footer = buildMetadataFooter(metadata, url);
+    if (!content.trim() && footer) {
+        const note = '> **Note:** This page contains no readable content. It may require JavaScript to render.\n\n';
+        return `${note}${footer}`;
+    }
     return footer ? `${content}\n\n${footer}` : content;
 }
 export function htmlToMarkdown(html, metadata, options) {
@@ -684,12 +688,12 @@ function tryTransformRawContent(params) {
 /* -------------------------------------------------------------------------------------------------
  * Quality gates + content source resolution
  * ------------------------------------------------------------------------------------------------- */
-const MIN_CONTENT_RATIO = 0.3;
+const MIN_CONTENT_RATIO = 0.15;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
-const MIN_HEADING_RETENTION_RATIO = 0.7;
-const MIN_CODE_BLOCK_RETENTION_RATIO = 0.5;
+const MIN_HEADING_RETENTION_RATIO = 0.3;
+const MIN_CODE_BLOCK_RETENTION_RATIO = 0.15;
 const MIN_LINE_LENGTH_FOR_TRUNCATION_CHECK = 20;
-const MAX_TRUNCATED_LINE_RATIO = 0.5;
+const MAX_TRUNCATED_LINE_RATIO = 0.95;
 function needsDocumentWrapper(html) {
     const trimmed = html.trim().toLowerCase();
     return (!trimmed.startsWith('<!doctype') &&
@@ -780,8 +784,8 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
     return metadata;
 }
 const CONTENT_ROOT_SELECTORS = [
-    'main',
     'article',
+    'main',
     '[role="main"]',
     '#content',
     '#main-content',
@@ -808,10 +812,9 @@ function findContentRoot(document) {
     }
     return undefined;
 }
-function shouldUseArticleContent(article, originalHtmlOrDocument, url) {
+function shouldUseArticleContent(article, originalHtmlOrDocument) {
     const articleLength = article.textContent.length;
     const originalLength = getVisibleTextLength(originalHtmlOrDocument);
-    const safeUrl = url.substring(0, 80);
     let articleDocument = null;
     const getArticleDocument = () => {
         if (articleDocument)
@@ -821,69 +824,45 @@ function shouldUseArticleContent(article, originalHtmlOrDocument, url) {
     };
     if (originalLength >= MIN_HTML_LENGTH_FOR_GATE) {
         const ratio = articleLength / originalLength;
-        if (ratio < MIN_CONTENT_RATIO) {
-            logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
-                url: safeUrl,
-                articleLength,
-            });
+        if (ratio < MIN_CONTENT_RATIO)
             return false;
-        }
     }
     const originalHeadings = countHeadingsDom(originalHtmlOrDocument);
     if (originalHeadings > 0) {
         const articleHeadings = countHeadingsDom(getArticleDocument());
         const retentionRatio = articleHeadings / originalHeadings;
-        if (retentionRatio < MIN_HEADING_RETENTION_RATIO) {
-            logDebug('Quality gate: Readability broke heading structure, using full HTML', {
-                url: safeUrl,
-                originalHeadings,
-                articleHeadings,
-            });
+        if (retentionRatio < MIN_HEADING_RETENTION_RATIO)
             return false;
-        }
     }
     const originalCodeBlocks = countCodeBlocksDom(originalHtmlOrDocument);
     if (originalCodeBlocks > 0) {
         const articleCodeBlocks = countCodeBlocksDom(getArticleDocument());
         const codeRetentionRatio = articleCodeBlocks / originalCodeBlocks;
-        logDebug('Code block retention check', {
-            url: safeUrl,
-            originalCodeBlocks,
-            articleCodeBlocks,
-            codeRetentionRatio,
-        });
-        if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO) {
-            logDebug('Quality gate: Readability removed code blocks, using full HTML', {
-                url: safeUrl,
-                originalCodeBlocks,
-                articleCodeBlocks,
-            });
+        if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO)
             return false;
-        }
-    }
-    if (hasTruncatedSentences(article.textContent)) {
-        logDebug('Quality gate: Extracted text has many truncated sentences, using full HTML', {
-            url: safeUrl,
-        });
-        return false;
     }
-    return true;
+    return !hasTruncatedSentences(article.textContent);
 }
 function buildContentSource(params) {
     const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
     if (useArticleContent && article) {
-        return { sourceHtml: article.content, title: article.title, metadata };
+        // Apply noise removal to Readability-extracted content to remove
+        // author bylines, social share buttons, and other boilerplate
+        // that Readability may have included in the article content
+        const cleanedArticleHtml = removeNoiseFromHtml(article.content, undefined, url);
+        return {
+            sourceHtml: cleanedArticleHtml,
+            title: article.title,
+            metadata,
+            skipNoiseRemoval: true, // Already cleaned
+        };
     }
     if (document) {
-        removeNoiseFromHtml(html, document, url);
-        const cleanedDoc = document;
+        const cleanedHtml = removeNoiseFromHtml(html, undefined, url);
+        const { document: cleanedDoc } = parseHTML(cleanedHtml);
         const contentRoot = findContentRoot(cleanedDoc);
         if (contentRoot) {
-            logDebug('Using content root fallback instead of full HTML', {
-                url: url.substring(0, 80),
-                contentLength: contentRoot.length,
-            });
             return {
                 sourceHtml: contentRoot,
                 title: extractedMeta.title,
@@ -905,7 +884,7 @@ function resolveContentSource(params) {
         ...(params.signal ? { signal: params.signal } : {}),
     });
     const useArticleContent = article
-        ? shouldUseArticleContent(article, document, params.url)
+        ? shouldUseArticleContent(article, document)
         : false;
     return buildContentSource({
         html: params.html,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.4.13",
+  "version": "2.5.1",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",
@@ -60,7 +60,6 @@
     "@mozilla/readability": "^0.6.0",
     "linkedom": "^0.18.12",
     "node-html-markdown": "^2.0.0",
-    "undici": "^7.19.2",
     "zod": "^4.3.6"
   },
   "devDependencies": {