@j0hanz/superfetch 2.4.12 → 2.5.0

package/dist/cache.d.ts

@@ -41,10 +41,6 @@ export declare function get(cacheKey: string | null): CacheEntry | undefined;
 export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;
 export declare function keys(): readonly string[];
 export declare function isEnabled(): boolean;
-export declare function getRecentCachedUrls(): {
-    url: string;
-    title?: string;
-}[];
 export declare function registerCachedContentResource(server: McpServer, serverIcons?: McpIcon[]): void;
 export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
 export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;

package/dist/cache.js

@@ -273,24 +273,6 @@ export function keys() {
 export function isEnabled() {
     return store.isEnabled();
 }
-export function getRecentCachedUrls() {
-    const cacheKeys = store.keys();
-    const maxResults = 20;
-    const results = [];
-    for (let i = 0; i < Math.min(cacheKeys.length, maxResults); i++) {
-        const key = cacheKeys[i];
-        if (!key)
-            continue;
-        const entry = store.get(key);
-        if (entry) {
-            results.push({
-                url: entry.url,
-                ...(entry.title ? { title: entry.title } : {}),
-            });
-        }
-    }
-    return results;
-}
 /* -------------------------------------------------------------------------------------------------
  * MCP cached content resource (superfetch://cache/markdown/{urlHash})
  * ------------------------------------------------------------------------------------------------- */

package/dist/config.d.ts

@@ -46,6 +46,7 @@ export declare const config: {
         timeoutMs: number;
         stageWarnRatio: number;
         metadataFormat: TransformMetadataFormat;
+        maxWorkerScale: number;
     };
     tools: {
         enabled: string[];

package/dist/config.js

@@ -103,7 +103,7 @@ const SIZE_LIMITS = {
     TEN_MB: 10 * 1024 * 1024,
 };
 const TIMEOUT = {
-    DEFAULT_FETCH_TIMEOUT_MS: 15000,
+    DEFAULT_FETCH_TIMEOUT_MS: parseInteger(process.env.FETCH_TIMEOUT_MS, 15000, 1000, 60000),
     DEFAULT_SESSION_TTL_MS: 30 * 60 * 1000,
     DEFAULT_TRANSFORM_TIMEOUT_MS: parseInteger(process.env.TRANSFORM_TIMEOUT_MS, 30000, 5000, 120000),
 };
@@ -202,6 +202,7 @@ export const config = {
         timeoutMs: TIMEOUT.DEFAULT_TRANSFORM_TIMEOUT_MS,
         stageWarnRatio: parseFloat(process.env.TRANSFORM_STAGE_WARN_RATIO ?? '0.5'),
         metadataFormat: parseTransformMetadataFormat(process.env.TRANSFORM_METADATA_FORMAT),
+        maxWorkerScale: parseInteger(process.env.TRANSFORM_WORKER_MAX_SCALE, 4, 1, 16),
     },
     tools: {
         enabled: parseList(process.env.ENABLED_TOOLS ?? 'fetch-url'),
@@ -268,8 +269,8 @@ export const config = {
     auth: buildAuthConfig(baseUrl),
     rateLimit: {
         enabled: true,
-        maxRequests: 100,
-        windowMs: 60000,
+        maxRequests: parseInteger(process.env.RATE_LIMIT_MAX, 100, 1, 10000),
+        windowMs: parseInteger(process.env.RATE_LIMIT_WINDOW_MS, 60000, 1000, 3600000),
         cleanupIntervalMs: 60000,
     },
     runtime: runtimeState,

package/dist/language-detection.js

@@ -319,6 +319,8 @@ const detector = new LanguageDetector();
  * Detect programming language from code content using heuristics.
  */
 export function detectLanguageFromCode(code) {
+    if (!code || code.trim().length === 0)
+        return undefined;
     return detector.detect(code);
 }
 /**

package/dist/markdown-cleanup.d.ts

@@ -5,8 +5,3 @@ export declare function addSourceToMarkdown(content: string, url: string): strin
 export declare function isRawTextContent(content: string): boolean;
 export declare function isLikelyHtmlContent(content: string): boolean;
 export declare function buildMetadataFooter(metadata?: MetadataBlock, fallbackUrl?: string): string;
-/**
- * Promote standalone lines that look like headings to proper markdown headings.
- * Fence-aware: never modifies content inside fenced code blocks.
- */
-export declare function promoteOrphanHeadings(markdown: string): string;

package/dist/markdown-cleanup.js

@@ -398,7 +398,7 @@ function hasMarkdownSourceLine(content) {
     }
     return false;
 }
-function addSourceToMarkdownMarkdownFormat(content, url) {
+function addSourceToMarkdownAsMarkdown(content, url) {
     if (hasMarkdownSourceLine(content))
         return content;
     const lineEnding = detectLineEnding(content);
@@ -423,7 +423,7 @@ function addSourceToMarkdownMarkdownFormat(content, url) {
 export function addSourceToMarkdown(content, url) {
     const fm = frontmatter.find(content);
     if (config.transform.metadataFormat === 'markdown' && !fm) {
-        return addSourceToMarkdownMarkdownFormat(content, url);
+        return addSourceToMarkdownAsMarkdown(content, url);
     }
     if (!fm) {
         // Preserve existing behavior: always uses LF even if content uses CRLF.
@@ -503,28 +503,3 @@ export function buildMetadataFooter(metadata, fallbackUrl) {
     }
     return lines.join('\n');
 }
-/* -------------------------------------------------------------------------------------------------
- * Heading promotion (fence-aware)
- * ------------------------------------------------------------------------------------------------- */
-/**
- * Promote standalone lines that look like headings to proper markdown headings.
- * Fence-aware: never modifies content inside fenced code blocks.
- */
-export function promoteOrphanHeadings(markdown) {
-    if (!markdown)
-        return '';
-    const lines = markdown.split('\n');
-    const result = [];
-    const state = initialFenceState();
-    for (let i = 0; i < lines.length; i += 1) {
-        const line = lines[i] ?? '';
-        const prevLine = i > 0 ? (lines[i - 1] ?? '') : '';
-        if (state.inFence || isFenceStart(line)) {
-            result.push(line);
-            advanceFenceState(line, state);
-            continue;
-        }
-        result.push(orphanHeadingPromoter.processLine(line, prevLine));
-    }
-    return result.join('\n');
-}

package/dist/observability.d.ts

@@ -8,12 +8,10 @@ interface RequestContext {
 export declare function setMcpServer(server: McpServer): void;
 export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
 export declare function getRequestId(): string | undefined;
-export declare function getSessionId(): string | undefined;
 export declare function getOperationId(): string | undefined;
 export declare function logInfo(message: string, meta?: LogMetadata): void;
 export declare function logDebug(message: string, meta?: LogMetadata): void;
 export declare function logWarn(message: string, meta?: LogMetadata): void;
 export declare function logError(message: string, error?: Error | LogMetadata): void;
 export declare function redactUrl(rawUrl: string): string;
-export declare function redactHeaders(headers: Record<string, unknown>): Record<string, unknown>;
 export {};

package/dist/observability.js

@@ -11,7 +11,7 @@ export function runWithRequestContext(context, fn) {
 export function getRequestId() {
     return requestContext.getStore()?.requestId;
 }
-export function getSessionId() {
+function getSessionId() {
     return requestContext.getStore()?.sessionId;
 }
 export function getOperationId() {
@@ -102,13 +102,3 @@ export function redactUrl(rawUrl) {
         return rawUrl;
     }
 }
-export function redactHeaders(headers) {
-    const redacted = { ...headers };
-    const sensitiveKeys = ['authorization', 'cookie', 'set-cookie', 'x-api-key'];
-    for (const key of Object.keys(redacted)) {
-        if (sensitiveKeys.includes(key.toLowerCase())) {
-            redacted[key] = '[REDACTED]';
-        }
-    }
-    return redacted;
-}

package/dist/tasks.d.ts

@@ -31,7 +31,7 @@ export interface CreateTaskResult {
     };
     _meta?: Record<string, unknown>;
 }
-export declare class TaskManager {
+declare class TaskManager {
     private tasks;
     private waiters;
     createTask(options?: CreateTaskOptions, statusMessage?: string, ownerKey?: string): TaskState;
@@ -46,7 +46,6 @@ export declare class TaskManager {
         tasks: TaskState[];
         nextCursor?: string;
     };
-    cleanupExpiredTasks(): number;
     waitForTerminalTask(taskId: string, ownerKey: string, signal?: AbortSignal): Promise<TaskState | undefined>;
     private notifyWaiters;
     private isExpired;
@@ -54,3 +53,4 @@ export declare class TaskManager {
     private decodeCursor;
 }
 export declare const taskManager: TaskManager;
+export {};

package/dist/tasks.js

@@ -12,7 +12,7 @@ const TERMINAL_STATUSES = new Set([
 function isTerminalStatus(status) {
     return TERMINAL_STATUSES.has(status);
 }
-export class TaskManager {
+class TaskManager {
     tasks = new Map();
     waiters = new Map();
     createTask(options, statusMessage = 'Task started', ownerKey = DEFAULT_OWNER_KEY) {
@@ -93,20 +93,6 @@ export class TaskManager {
         const nextCursor = nextIndex < allTasks.length ? this.encodeCursor(nextIndex) : undefined;
         return nextCursor ? { tasks: page, nextCursor } : { tasks: page };
     }
-    // Helper to check if task is expired and could be cleaned up
-    // In a real implementation, this would be called by a periodic job
-    cleanupExpiredTasks() {
-        const now = Date.now();
-        let count = 0;
-        for (const [id, task] of this.tasks.entries()) {
-            const created = new Date(task.createdAt).getTime();
-            if (now - created > task.ttl) {
-                this.tasks.delete(id);
-                count++;
-            }
-        }
-        return count;
-    }
     async waitForTerminalTask(taskId, ownerKey, signal) {
         const task = this.getTask(taskId, ownerKey);
         if (!task)

package/dist/tools.d.ts

@@ -72,7 +72,6 @@ export interface ToolHandlerExtra {
     sendNotification?: (notification: ProgressNotification) => Promise<void>;
 }
 export declare const FETCH_URL_TOOL_NAME = "fetch-url";
-export declare const FETCH_URL_TOOL_DESCRIPTION: string;
 interface ProgressReporter {
     report: (progress: number, message: string) => Promise<void>;
 }

package/dist/tools.js

@@ -46,7 +46,7 @@ const fetchUrlOutputSchema = z.strictObject({
         .describe('Error message if the request failed'),
 });
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
-export const FETCH_URL_TOOL_DESCRIPTION = `
+const FETCH_URL_TOOL_DESCRIPTION = `
 Fetches a webpage and converts it to clean Markdown format optimized for LLM context.
 
 This tool is useful for:

package/dist/transform.js

@@ -684,12 +684,12 @@ function tryTransformRawContent(params) {
 /* -------------------------------------------------------------------------------------------------
  * Quality gates + content source resolution
  * ------------------------------------------------------------------------------------------------- */
-const MIN_CONTENT_RATIO = 0.3;
+const MIN_CONTENT_RATIO = 0.15;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
-const MIN_HEADING_RETENTION_RATIO = 0.7;
-const MIN_CODE_BLOCK_RETENTION_RATIO = 0.5;
+const MIN_HEADING_RETENTION_RATIO = 0.3;
+const MIN_CODE_BLOCK_RETENTION_RATIO = 0.15;
 const MIN_LINE_LENGTH_FOR_TRUNCATION_CHECK = 20;
-const MAX_TRUNCATED_LINE_RATIO = 0.5;
+const MAX_TRUNCATED_LINE_RATIO = 0.95;
 function needsDocumentWrapper(html) {
     const trimmed = html.trim().toLowerCase();
     return (!trimmed.startsWith('<!doctype') &&
@@ -780,8 +780,8 @@ export function createContentMetadataBlock(url, article, extractedMeta, shouldEx
     return metadata;
 }
 const CONTENT_ROOT_SELECTORS = [
-    'main',
     'article',
+    'main',
     '[role="main"]',
     '#content',
     '#main-content',
@@ -808,10 +808,9 @@ function findContentRoot(document) {
     }
     return undefined;
 }
-function shouldUseArticleContent(article, originalHtmlOrDocument, url) {
+function shouldUseArticleContent(article, originalHtmlOrDocument) {
     const articleLength = article.textContent.length;
     const originalLength = getVisibleTextLength(originalHtmlOrDocument);
-    const safeUrl = url.substring(0, 80);
     let articleDocument = null;
     const getArticleDocument = () => {
         if (articleDocument)
@@ -821,69 +820,45 @@ function shouldUseArticleContent(article, originalHtmlOrDocument, url) {
     };
     if (originalLength >= MIN_HTML_LENGTH_FOR_GATE) {
         const ratio = articleLength / originalLength;
-        if (ratio < MIN_CONTENT_RATIO) {
-            logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
-                url: safeUrl,
-                articleLength,
-            });
+        if (ratio < MIN_CONTENT_RATIO)
             return false;
-        }
     }
     const originalHeadings = countHeadingsDom(originalHtmlOrDocument);
     if (originalHeadings > 0) {
         const articleHeadings = countHeadingsDom(getArticleDocument());
         const retentionRatio = articleHeadings / originalHeadings;
-        if (retentionRatio < MIN_HEADING_RETENTION_RATIO) {
-            logDebug('Quality gate: Readability broke heading structure, using full HTML', {
-                url: safeUrl,
-                originalHeadings,
-                articleHeadings,
-            });
+        if (retentionRatio < MIN_HEADING_RETENTION_RATIO)
             return false;
-        }
     }
     const originalCodeBlocks = countCodeBlocksDom(originalHtmlOrDocument);
     if (originalCodeBlocks > 0) {
         const articleCodeBlocks = countCodeBlocksDom(getArticleDocument());
         const codeRetentionRatio = articleCodeBlocks / originalCodeBlocks;
-        logDebug('Code block retention check', {
-            url: safeUrl,
-            originalCodeBlocks,
-            articleCodeBlocks,
-            codeRetentionRatio,
-        });
-        if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO) {
-            logDebug('Quality gate: Readability removed code blocks, using full HTML', {
-                url: safeUrl,
-                originalCodeBlocks,
-                articleCodeBlocks,
-            });
+        if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO)
             return false;
-        }
     }
-    if (hasTruncatedSentences(article.textContent)) {
-        logDebug('Quality gate: Extracted text has many truncated sentences, using full HTML', {
-            url: safeUrl,
-        });
-        return false;
-    }
-    return true;
+    return !hasTruncatedSentences(article.textContent);
 }
 function buildContentSource(params) {
     const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
     if (useArticleContent && article) {
-        return { sourceHtml: article.content, title: article.title, metadata };
+        // Apply noise removal to Readability-extracted content to remove
+        // author bylines, social share buttons, and other boilerplate
+        // that Readability may have included in the article content
+        const cleanedArticleHtml = removeNoiseFromHtml(article.content, undefined, url);
+        return {
+            sourceHtml: cleanedArticleHtml,
+            title: article.title,
+            metadata,
+            skipNoiseRemoval: true, // Already cleaned
+        };
     }
     if (document) {
-        removeNoiseFromHtml(html, document, url);
-        const cleanedDoc = document;
+        const cleanedHtml = removeNoiseFromHtml(html, undefined, url);
+        const { document: cleanedDoc } = parseHTML(cleanedHtml);
         const contentRoot = findContentRoot(cleanedDoc);
         if (contentRoot) {
-            logDebug('Using content root fallback instead of full HTML', {
-                url: url.substring(0, 80),
-                contentLength: contentRoot.length,
-            });
             return {
                 sourceHtml: contentRoot,
                 title: extractedMeta.title,
@@ -905,7 +880,7 @@ function resolveContentSource(params) {
         ...(params.signal ? { signal: params.signal } : {}),
     });
     const useArticleContent = article
-        ? shouldUseArticleContent(article, document, params.url)
+        ? shouldUseArticleContent(article, document)
         : false;
     return buildContentSource({
         html: params.html,
@@ -984,7 +959,7 @@ const workerMessageSchema = z.discriminatedUnion('type', [
     }),
 ]);
 const POOL_MIN_WORKERS = 2;
-const POOL_MAX_WORKERS = 4;
+const POOL_MAX_WORKERS = config.transform.maxWorkerScale;
 const POOL_SCALE_THRESHOLD = 0.5;
 const DEFAULT_TIMEOUT_MS = config.transform.timeoutMs;
 class WorkerPool {
@@ -1029,6 +1004,13 @@ class WorkerPool {
     getCapacity() {
         return this.capacity;
     }
+    resize(size) {
+        const newCapacity = Math.max(this.minCapacity, Math.min(size, this.maxCapacity));
+        if (newCapacity === this.capacity)
+            return;
+        this.capacity = newCapacity;
+        this.drainQueue();
+    }
     async close() {
         if (this.closed)
             return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.4.12",
+  "version": "2.5.0",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",