@j0hanz/superfetch 1.2.2 → 1.2.4

package/dist/tools/utils/content-transform.js

@@ -102,7 +102,10 @@ export function transformHtmlToMarkdown(html, url, options) {
     };
 }
 export function transformHtmlToMarkdownWithBlocks(html, url, options) {
-    if (!options.extractMainContent && options.includeMetadata) {
+    const includeContentBlocks = options.includeContentBlocks ?? true;
+    if (includeContentBlocks &&
+        !options.extractMainContent &&
+        options.includeMetadata) {
         const parsed = parseHtmlWithMetadata(html);
         const context = {
             sourceHtml: html,
@@ -118,7 +121,9 @@ export function transformHtmlToMarkdownWithBlocks(html, url, options) {
         };
     }
     const context = resolveContentSource(html, url, options);
-    const contentBlocks = parseHtml(context.sourceHtml);
+    const contentBlocks = includeContentBlocks
+        ? parseHtml(context.sourceHtml)
+        : [];
     const { content, truncated } = buildMarkdownPayload(context, options.maxContentLength);
     return {
         content,

package/dist/tools/utils/fetch-pipeline.js

@@ -51,7 +51,7 @@ export async function executeFetchPipeline(options) {
     const fetchOptions = buildFetchOptions(options);
     logDebug('Fetching URL', { url: normalizedUrl, retries: options.retries });
     const html = await fetchNormalizedUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
-    const data = options.transform(html, normalizedUrl);
+    const data = await options.transform(html, normalizedUrl);
     if (cache.isEnabled()) {
         persistCache(cacheKey, data, options.serialize, normalizedUrl);
     }
@@ -62,20 +62,28 @@ function resolveCacheKey(options, normalizedUrl) {
     return cache.createCacheKey(options.cacheNamespace, normalizedUrl, cacheVary);
 }
 function buildFetchOptions(options) {
-    return {
-        customHeaders: options.customHeaders,
-        signal: options.signal,
-        timeout: options.timeout,
-    };
+    const fetchOptions = {};
+    if (options.customHeaders !== undefined) {
+        fetchOptions.customHeaders = options.customHeaders;
+    }
+    if (options.signal !== undefined) {
+        fetchOptions.signal = options.signal;
+    }
+    if (options.timeout !== undefined) {
+        fetchOptions.timeout = options.timeout;
+    }
+    return fetchOptions;
 }
 function persistCache(cacheKey, data, serialize, normalizedUrl) {
     if (!cacheKey)
         return;
     const serializer = serialize ?? JSON.stringify;
-    cache.set(cacheKey, serializer(data), {
-        url: normalizedUrl,
-        title: extractTitle(data),
-    });
+    const metadata = { url: normalizedUrl };
+    const title = extractTitle(data);
+    if (title !== undefined) {
+        metadata.title = title;
+    }
+    cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
     if (!value || typeof value !== 'object')

package/dist/utils/content-cleaner.d.ts

@@ -1,5 +1,5 @@
 export declare function cleanParagraph(text: string): string | null;
 export declare function cleanHeading(text: string): string | null;
-export declare function cleanListItems(items: string[]): string[];
+export declare function cleanListItems(items: readonly string[]): readonly string[];
 export declare function cleanCodeBlock(code: string): string | null;
 export declare function removeInlineTimestamps(text: string): string;

package/dist/utils/download-url.d.ts

@@ -1,8 +1,16 @@
+import { config } from '../config/index.js';
 import type { FileDownloadInfo } from '../config/types/tools.js';
+import * as cache from '../services/cache.js';
+import { generateSafeFilename } from './filename-generator.js';
 interface DownloadInfoOptions {
     cacheKey: string | null;
     url: string;
     title?: string;
 }
-export declare function buildFileDownloadInfo(options: DownloadInfoOptions): FileDownloadInfo | null;
+interface DownloadInfoDeps {
+    readonly config?: typeof config;
+    readonly cache?: Pick<typeof cache, 'get' | 'parseCacheKey'>;
+    readonly generateSafeFilename?: typeof generateSafeFilename;
+}
+export declare function buildFileDownloadInfo(options: DownloadInfoOptions, deps?: DownloadInfoDeps): FileDownloadInfo | null;
 export {};

package/dist/utils/download-url.js

@@ -1,22 +1,25 @@
 import { config } from '../config/index.js';
 import * as cache from '../services/cache.js';
 import { generateSafeFilename } from './filename-generator.js';
-export function buildFileDownloadInfo(options) {
-    if (!config.runtime.httpMode) {
+export function buildFileDownloadInfo(options, deps = {}) {
+    const resolvedConfig = deps.config ?? config;
+    const resolvedCache = deps.cache ?? cache;
+    const resolveFilename = deps.generateSafeFilename ?? generateSafeFilename;
+    if (!resolvedConfig.runtime.httpMode) {
         return null;
     }
-    if (!config.cache.enabled || !options.cacheKey) {
+    if (!resolvedConfig.cache.enabled || !options.cacheKey) {
         return null;
     }
-    const parts = cache.parseCacheKey(options.cacheKey);
+    const parts = resolvedCache.parseCacheKey(options.cacheKey);
     if (!parts)
         return null;
-    const cacheEntry = cache.get(options.cacheKey);
+    const cacheEntry = resolvedCache.get(options.cacheKey);
     if (!cacheEntry)
         return null;
     const { expiresAt, title, url } = cacheEntry;
     const downloadUrl = buildDownloadUrl(parts.namespace, parts.urlHash);
-    const fileName = generateSafeFilename(url, title ?? options.title, parts.urlHash, resolveExtension(parts.namespace));
+    const fileName = resolveFilename(url, title ?? options.title, parts.urlHash, resolveExtension(parts.namespace));
     return { downloadUrl, fileName, expiresAt };
 }
 function buildDownloadUrl(namespace, hash) {

package/dist/utils/tool-error-handler.d.ts

@@ -1,3 +1,3 @@
 import type { ToolErrorResponse } from '../config/types/tools.js';
-export declare function createToolErrorResponse(message: string, url: string, code: string): ToolErrorResponse;
-export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
+export declare function createToolErrorResponse(message: string, url: string, code: string, details?: Record<string, unknown>): ToolErrorResponse;
+export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string, details?: Record<string, unknown>): ToolErrorResponse;

package/dist/utils/tool-error-handler.js

@@ -22,12 +22,12 @@ function normalizeToolErrorCode(code) {
         return String(ErrorCode.InternalError);
     return MCP_ERROR_CODE_MAP[code] ?? code;
 }
-export function createToolErrorResponse(message, url, code) {
+export function createToolErrorResponse(message, url, code, details = {}) {
     const structuredContent = {
+        ...details,
         error: message,
         url,
         errorCode: normalizeToolErrorCode(code),
-        errorType: code,
     };
     return {
         content: [{ type: 'text', text: JSON.stringify(structuredContent) }],
@@ -42,19 +42,19 @@ function formatErrorMessage(baseMessage, error, fallback) {
     }
     return message;
 }
-export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
+export function handleToolError(error, url, fallbackMessage = 'Operation failed', details = {}) {
     if (isValidationError(error)) {
-        return createToolErrorResponse(error.message, url, 'VALIDATION_ERROR');
+        return createToolErrorResponse(error.message, url, 'VALIDATION_ERROR', details);
     }
     if (error instanceof FetchError) {
         const message = formatErrorMessage(error.message, error);
-        return createToolErrorResponse(message, url, error.code);
+        return createToolErrorResponse(message, url, error.code, details);
     }
     if (error instanceof Error) {
         const message = formatErrorMessage(error.message, error, fallbackMessage);
-        return createToolErrorResponse(message, url, 'UNKNOWN_ERROR');
+        return createToolErrorResponse(message, url, 'UNKNOWN_ERROR', details);
     }
-    return createToolErrorResponse(`${fallbackMessage}: Unknown error`, url, 'UNKNOWN_ERROR');
+    return createToolErrorResponse(`${fallbackMessage}: Unknown error`, url, 'UNKNOWN_ERROR', details);
 }
 function isValidationError(error) {
     return (error instanceof Error &&

package/dist/utils/url-validator.js

@@ -32,6 +32,35 @@ for (const entry of BLOCKED_IPV6_SUBNETS) {
     BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
 }
 const DNS_LOOKUP_TIMEOUT_MS = 5000;
+const DNS_DECISION_TTL_MS = 60000;
+const DNS_DECISION_MAX = 1000;
+const dnsDecisionCache = new Map();
+function getCachedDnsDecision(hostname) {
+    const cached = dnsDecisionCache.get(hostname);
+    if (!cached)
+        return null;
+    if (cached.expiresAt <= Date.now()) {
+        dnsDecisionCache.delete(hostname);
+        return null;
+    }
+    return cached;
+}
+function setCachedDnsDecision(hostname, ok) {
+    dnsDecisionCache.set(hostname, {
+        ok,
+        expiresAt: Date.now() + DNS_DECISION_TTL_MS,
+    });
+    if (dnsDecisionCache.size <= DNS_DECISION_MAX)
+        return;
+    const evictCount = Math.ceil(DNS_DECISION_MAX * 0.05);
+    const iterator = dnsDecisionCache.keys();
+    for (let i = 0; i < evictCount; i++) {
+        const { value, done } = iterator.next();
+        if (done)
+            break;
+        dnsDecisionCache.delete(value);
+    }
+}
 function matchesBlockedIpPatterns(resolvedIp) {
     for (const pattern of config.security.blockedIpPatterns) {
         if (pattern.test(resolvedIp)) {
@@ -79,6 +108,13 @@ function lookupWithTimeout(hostname) {
     });
 }
 export async function assertResolvedAddressesAllowed(hostname) {
+    const cached = getCachedDnsDecision(hostname);
+    if (cached) {
+        if (!cached.ok) {
+            throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
+        }
+        return;
+    }
     try {
         const result = await lookupWithTimeout(hostname);
         const addresses = Array.isArray(result) ? result : [result];
@@ -87,9 +123,11 @@ export async function assertResolvedAddressesAllowed(hostname) {
         }
         for (const { address } of addresses) {
             if (isBlockedIp(address.toLowerCase())) {
+                setCachedDnsDecision(hostname, false);
                 throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
             }
         }
+        setCachedDnsDecision(hostname, true);
     }
     catch (error) {
         const code = error?.code;

package/dist/workers/transform-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/workers/transform-worker.js

@@ -0,0 +1,50 @@
+import { parentPort } from 'node:worker_threads';
+import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from '../tools/utils/content-transform.js';
+function isTransformJob(value) {
+    if (!value || typeof value !== 'object')
+        return false;
+    const record = value;
+    return (typeof record.id === 'number' &&
+        typeof record.mode === 'string' &&
+        typeof record.html === 'string' &&
+        typeof record.url === 'string');
+}
+function resolveTransform(job) {
+    if (job.mode === 'markdown') {
+        return transformHtmlToMarkdown(job.html, job.url, job.options);
+    }
+    if (job.mode === 'markdown-blocks') {
+        return transformHtmlToMarkdownWithBlocks(job.html, job.url, {
+            ...job.options,
+            includeContentBlocks: job.options.includeContentBlocks ?? true,
+        });
+    }
+    return transformHtmlToJsonl(job.html, job.url, job.options);
+}
+function sendResponse(response) {
+    if (!parentPort)
+        return;
+    parentPort.postMessage(response);
+}
+function handleMessage(message) {
+    if (!isTransformJob(message)) {
+        sendResponse({
+            id: -1,
+            ok: false,
+            error: 'Invalid transform job payload',
+        });
+        return;
+    }
+    try {
+        const result = resolveTransform(message);
+        sendResponse({ id: message.id, ok: true, result });
+    }
+    catch (error) {
+        sendResponse({
+            id: message.id,
+            ok: false,
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+}
+parentPort?.on('message', handleMessage);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "1.2.2",
+  "version": "1.2.4",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
   "type": "module",
@@ -44,8 +44,8 @@
     "type-check": "tsc --noEmit",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
-    "test": "vitest run",
-    "test:coverage": "vitest run --coverage",
+    "test": "npm run build --silent && node --test --experimental-transform-types",
+    "test:coverage": "npm run build --silent && node --test --experimental-transform-types --experimental-test-coverage",
     "bench": "npm run build && node scripts/bench.mjs",
     "knip": "knip",
     "knip:fix": "knip --fix"
@@ -59,7 +59,7 @@
     "linkedom": "^0.18.12",
     "turndown": "^7.2.2",
     "undici": "^6.22.0",
-    "zod": "^3.24.1"
+    "zod": "^4.3.4"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
@@ -67,7 +67,6 @@
     "@types/express": "^5.0.6",
     "@types/node": "^22.19.3",
     "@types/turndown": "^5.0.6",
-    "@vitest/coverage-v8": "^2.1.9",
     "eslint": "^9.23.2",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-unused-imports": "^4.3.0",
@@ -76,8 +75,7 @@
     "shx": "^0.4.0",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.51.0",
-    "vitest": "^2.1.9"
+    "typescript-eslint": "^8.51.0"
   },
   "engines": {
     "node": ">=20.12.0"