@j0hanz/superfetch 2.0.1 → 2.1.0

package/dist/transformers/markdown.d.ts

@@ -1,2 +1,5 @@
 import type { MetadataBlock } from '../config/types/content.js';
-export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock): string;
+export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
+    url?: string;
+    signal?: AbortSignal;
+}): string;

package/dist/transformers/markdown.js

@@ -1,5 +1,9 @@
-import TurndownService from 'turndown';
+import { parseHTML } from 'linkedom';
+import { NodeHtmlMarkdown, } from 'node-html-markdown';
 import { CODE_BLOCK, FRONTMATTER_DELIMITER, joinLines, } from '../config/formatting.js';
+import { FetchError } from '../errors/app-error.js';
+import { endTransformStage, startTransformStage, } from '../services/telemetry.js';
+import { throwIfAborted } from '../utils/cancellation.js';
 import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../utils/code-language.js';
 import { isRecord } from '../utils/guards.js';
 const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
@@ -43,6 +47,9 @@ function buildFrontmatter(metadata) {
     const lines = [FRONTMATTER_DELIMITER];
     appendFrontmatterField(lines, 'title', metadata.title);
     appendFrontmatterField(lines, 'source', metadata.url);
+    appendFrontmatterField(lines, 'author', metadata.author);
+    appendFrontmatterField(lines, 'description', metadata.description);
+    appendFrontmatterField(lines, 'fetchedAt', metadata.fetchedAt);
     lines.push(FRONTMATTER_DELIMITER);
     return joinLines(lines);
 }
@@ -51,36 +58,6 @@ function isElement(node) {
         'getAttribute' in node &&
         typeof node.getAttribute === 'function');
 }
-function isFencedCodeBlock(node, options) {
-    return (options.codeBlockStyle === 'fenced' &&
-        node.nodeName === 'PRE' &&
-        node.firstChild?.nodeName === 'CODE');
-}
-function formatFencedCodeBlock(node) {
-    const codeNode = node.firstChild;
-    if (!isElement(codeNode))
-        return '';
-    const code = codeNode.textContent || '';
-    const language = resolveCodeLanguage(codeNode, code);
-    return CODE_BLOCK.format(code, language);
-}
-function resolveCodeLanguage(codeNode, code) {
-    const { className, dataLanguage } = readCodeAttributes(codeNode);
-    const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
-    return attributeLanguage ?? detectLanguageFromCode(code) ?? '';
-}
-function readCodeAttributes(codeNode) {
-    return {
-        className: codeNode.getAttribute('class') ?? '',
-        dataLanguage: codeNode.getAttribute('data-language') ?? '',
-    };
-}
-function addFencedCodeRule(instance) {
-    instance.addRule('fencedCodeBlockWithLanguage', {
-        filter: (node, options) => isFencedCodeBlock(node, options),
-        replacement: (_content, node) => formatFencedCodeBlock(node),
-    });
-}
 const STRUCTURAL_TAGS = new Set([
     'script',
     'style',
@@ -109,6 +86,67 @@ const PROMO_PATTERN = /banner|promo|announcement|cta|callout|advert|newsletter|s
 const FIXED_PATTERN = /\b(fixed|sticky)\b/;
 const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
 const ISOLATE_PATTERN = /\bisolate\b/;
+const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
+const NOISE_MARKERS = [
+    '<script',
+    '<style',
+    '<noscript',
+    '<iframe',
+    '<nav',
+    '<footer',
+    '<aside',
+    '<header',
+    '<form',
+    '<button',
+    '<input',
+    '<select',
+    '<textarea',
+    '<svg',
+    '<canvas',
+    ' aria-hidden="true"',
+    " aria-hidden='true'",
+    ' hidden',
+    ' role="navigation"',
+    " role='navigation'",
+    ' role="banner"',
+    " role='banner'",
+    ' role="complementary"',
+    " role='complementary'",
+    ' role="contentinfo"',
+    " role='contentinfo'",
+    ' role="tree"',
+    " role='tree'",
+    ' role="menubar"',
+    " role='menubar'",
+    ' role="menu"',
+    " role='menu'",
+    ' banner',
+    ' promo',
+    ' announcement',
+    ' cta',
+    ' callout',
+    ' advert',
+    ' newsletter',
+    ' subscribe',
+    ' cookie',
+    ' consent',
+    ' popup',
+    ' modal',
+    ' overlay',
+    ' toast',
+    ' fixed',
+    ' sticky',
+    ' z-50',
+    ' z-4',
+    ' isolate',
+];
+function mayContainNoise(html) {
+    const haystack = html.toLowerCase();
+    return NOISE_MARKERS.some((marker) => haystack.includes(marker));
+}
+function isFullDocumentHtml(html) {
+    return HTML_DOCUMENT_MARKERS.test(html);
+}
 function isStructuralNoiseTag(tagName) {
     return (STRUCTURAL_TAGS.has(tagName) || tagName === 'svg' || tagName === 'canvas');
 }
@@ -146,40 +184,131 @@ function isNoiseElement(node) {
         matchesFixedOrHighZIsolate(metadata.className) ||
         matchesPromoIdOrClass(metadata.className, metadata.id));
 }
-function isNoiseNode(node) {
-    return isElement(node) && isNoiseElement(node);
+function removeNoiseFromHtml(html) {
+    const shouldParse = isFullDocumentHtml(html) || mayContainNoise(html);
+    if (!shouldParse)
+        return html;
+    const shouldRemove = mayContainNoise(html);
+    try {
+        const { document } = parseHTML(html);
+        if (shouldRemove) {
+            const nodes = Array.from(document.querySelectorAll('*'));
+            for (let index = nodes.length - 1; index >= 0; index -= 1) {
+                const node = nodes[index];
+                if (!node)
+                    continue;
+                if (isElement(node) && isNoiseElement(node)) {
+                    node.remove();
+                }
+            }
+        }
+        const { body } = document;
+        if (body?.innerHTML)
+            return body.innerHTML;
+        if (typeof document.toString ===
+            'function') {
+            return document.toString();
+        }
+        const { documentElement } = document;
+        if (documentElement?.outerHTML)
+            return documentElement.outerHTML;
+        return html;
+    }
+    catch {
+        return html;
+    }
+}
+function buildInlineCode(content) {
+    const runs = content.match(/`+/g);
+    const longest = runs?.sort((a, b) => b.length - a.length)[0] ?? '';
+    const delimiter = `\`${longest}`;
+    const padding = delimiter.length > 1 ? ' ' : '';
+    return `${delimiter}${padding}${content}${padding}${delimiter}`;
 }
-function addNoiseRule(instance) {
-    instance.addRule('removeNoise', {
-        filter: (node) => isNoiseNode(node),
-        replacement: () => '',
-    });
+function isCodeBlock(parent) {
+    if (!isRecord(parent))
+        return false;
+    const tagName = typeof parent.tagName === 'string' ? parent.tagName.toUpperCase() : '';
+    return ['PRE', 'WRAPPED-PRE'].includes(tagName);
 }
-let turndownInstance = null;
-function createTurndownInstance() {
-    const instance = new TurndownService({
-        headingStyle: 'atx',
+function createCodeTranslator() {
+    return {
+        code: (ctx) => {
+            if (!isRecord(ctx)) {
+                return {
+                    spaceIfRepeatingChar: true,
+                    noEscape: true,
+                    postprocess: ({ content }) => buildInlineCode(content),
+                };
+            }
+            const { node, parent, visitor } = ctx;
+            const getAttribute = isRecord(node) && typeof node.getAttribute === 'function'
+                ? node.getAttribute.bind(node)
+                : undefined;
+            if (!isCodeBlock(parent)) {
+                return {
+                    spaceIfRepeatingChar: true,
+                    noEscape: true,
+                    postprocess: ({ content }) => buildInlineCode(content),
+                };
+            }
+            const className = getAttribute?.('class') ?? '';
+            const dataLanguage = getAttribute?.('data-language') ?? '';
+            const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
+            const childTranslators = isRecord(visitor) ? visitor.instance : null;
+            const codeBlockTranslators = isRecord(childTranslators) &&
+                isRecord(childTranslators
+                    .codeBlockTranslators)
+                ? childTranslators.codeBlockTranslators
+                : null;
+            return {
+                noEscape: true,
+                preserveWhitespace: true,
+                ...(codeBlockTranslators
+                    ? { childTranslators: codeBlockTranslators }
+                    : null),
+                postprocess: ({ content }) => {
+                    const language = attributeLanguage ?? detectLanguageFromCode(content) ?? '';
+                    return CODE_BLOCK.format(content, language);
+                },
+            };
+        },
+    };
+}
+let markdownInstance = null;
+function createMarkdownInstance() {
+    return new NodeHtmlMarkdown({
+        codeFence: CODE_BLOCK.fence,
         codeBlockStyle: 'fenced',
         emDelimiter: '_',
-        bulletListMarker: '-',
-    });
-    addNoiseRule(instance);
-    addFencedCodeRule(instance);
-    return instance;
+        bulletMarker: '-',
+    }, createCodeTranslator());
 }
-function getTurndown() {
-    turndownInstance ??= createTurndownInstance();
-    return turndownInstance;
+function getMarkdownConverter() {
+    markdownInstance ??= createMarkdownInstance();
+    return markdownInstance;
 }
-export function htmlToMarkdown(html, metadata) {
+export function htmlToMarkdown(html, metadata, options) {
+    const url = options?.url ?? metadata?.url ?? '';
     const frontmatter = buildFrontmatter(metadata);
     if (!html)
         return frontmatter;
     try {
-        const content = getTurndown().turndown(html).trim();
+        throwIfAborted(options?.signal, url, 'markdown:begin');
+        const noiseStage = startTransformStage(url, 'markdown:noise');
+        const cleanedHtml = removeNoiseFromHtml(html);
+        endTransformStage(noiseStage);
+        throwIfAborted(options?.signal, url, 'markdown:cleaned');
+        const translateStage = startTransformStage(url, 'markdown:translate');
+        const content = getMarkdownConverter().translate(cleanedHtml).trim();
+        endTransformStage(translateStage);
+        throwIfAborted(options?.signal, url, 'markdown:translated');
         return frontmatter ? `${frontmatter}\n${content}` : content;
     }
-    catch {
+    catch (error) {
+        if (error instanceof FetchError) {
+            throw error;
+        }
         return frontmatter;
     }
 }

package/dist/utils/cancellation.d.ts

@@ -0,0 +1 @@
+export declare function throwIfAborted(signal: AbortSignal | undefined, url: string, stage: string): void;

package/dist/utils/cancellation.js

@@ -0,0 +1,18 @@
+import { FetchError } from '../errors/app-error.js';
+function isTimeoutReason(reason) {
+    return reason instanceof Error && reason.name === 'TimeoutError';
+}
+export function throwIfAborted(signal, url, stage) {
+    if (!signal?.aborted)
+        return;
+    if (isTimeoutReason(signal.reason)) {
+        throw new FetchError('Request timeout', url, 504, {
+            reason: 'timeout',
+            stage,
+        });
+    }
+    throw new FetchError('Request was canceled', url, 499, {
+        reason: 'aborted',
+        stage,
+    });
+}

package/dist/utils/code-language.d.ts

@@ -1,11 +1,2 @@
-export declare function containsJsxTag(code: string): boolean;
-export declare function containsWord(source: string, word: string): boolean;
-export declare function splitLines(content: string): string[];
-export declare function extractLanguageFromClassName(className: string): string | undefined;
-export declare function resolveLanguageFromDataAttribute(dataLang: string): string | undefined;
-export interface CodeDetector {
-    language: string;
-    detect: (code: string) => boolean;
-}
 export declare function detectLanguageFromCode(code: string): string | undefined;
 export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;

package/dist/utils/code-language.js

@@ -1,4 +1,4 @@
-export function containsJsxTag(code) {
+function containsJsxTag(code) {
     for (let index = 0; index < code.length - 1; index += 1) {
         if (code[index] !== '<')
             continue;
@@ -10,7 +10,7 @@ export function containsJsxTag(code) {
     }
     return false;
 }
-export function containsWord(source, word) {
+function containsWord(source, word) {
     let startIndex = source.indexOf(word);
     while (startIndex !== -1) {
         const before = startIndex === 0 ? '' : source[startIndex - 1];
@@ -22,10 +22,10 @@ export function containsWord(source, word) {
     }
     return false;
 }
-export function splitLines(content) {
+function splitLines(content) {
     return content.split('\n');
 }
-export function extractLanguageFromClassName(className) {
+function extractLanguageFromClassName(className) {
     const tokens = className.match(/\S+/g);
     if (!tokens)
         return undefined;
@@ -41,7 +41,7 @@ export function extractLanguageFromClassName(className) {
     }
     return undefined;
 }
-export function resolveLanguageFromDataAttribute(dataLang) {
+function resolveLanguageFromDataAttribute(dataLang) {
     const trimmed = dataLang.trim();
     if (!trimmed)
         return undefined;

package/dist/utils/host-normalizer.d.ts

@@ -0,0 +1 @@
+export declare function normalizeHost(value: string): string | null;

package/dist/utils/host-normalizer.js

@@ -0,0 +1,37 @@
+import { isIP } from 'node:net';
+function takeFirstHostValue(value) {
+    const first = value.split(',')[0];
+    if (!first)
+        return null;
+    const trimmed = first.trim();
+    return trimmed ? trimmed : null;
+}
+function stripIpv6Brackets(value) {
+    if (!value.startsWith('['))
+        return null;
+    const end = value.indexOf(']');
+    if (end === -1)
+        return null;
+    return value.slice(1, end);
+}
+function stripPortIfPresent(value) {
+    const colonIndex = value.indexOf(':');
+    if (colonIndex === -1)
+        return value;
+    return value.slice(0, colonIndex);
+}
+export function normalizeHost(value) {
+    const trimmed = value.trim().toLowerCase();
+    if (!trimmed)
+        return null;
+    const first = takeFirstHostValue(trimmed);
+    if (!first)
+        return null;
+    const ipv6 = stripIpv6Brackets(first);
+    if (ipv6)
+        return ipv6;
+    if (isIP(first) === 6) {
+        return first;
+    }
+    return stripPortIfPresent(first);
+}

package/dist/utils/url-redactor.d.ts

@@ -0,0 +1 @@
+export declare function redactUrl(rawUrl: string): string;

package/dist/utils/url-redactor.js

@@ -0,0 +1,13 @@
+export function redactUrl(rawUrl) {
+    try {
+        const url = new URL(rawUrl);
+        url.username = '';
+        url.password = '';
+        url.hash = '';
+        url.search = '';
+        return url.toString();
+    }
+    catch {
+        return rawUrl;
+    }
+}

package/dist/utils/url-validator.js

@@ -78,6 +78,8 @@ export function normalizeUrl(urlString) {
     assertNoCredentials(url);
     const hostname = normalizeHostname(url);
     assertHostnameAllowed(hostname);
+    // Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
+    url.hostname = hostname;
     return { normalizedUrl: url.href, hostname };
 }
 export function validateAndNormalizeUrl(urlString) {
@@ -103,12 +105,10 @@ function assertUrlLength(url) {
     throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
 }
 function parseUrl(urlString) {
-    try {
-        return new URL(urlString);
-    }
-    catch {
+    if (!URL.canParse(urlString)) {
         throw createValidationError('Invalid URL format');
     }
+    return new URL(urlString);
 }
 function assertHttpProtocol(url) {
     if (url.protocol === 'http:' || url.protocol === 'https:')
@@ -121,7 +121,10 @@ function assertNoCredentials(url) {
     throw createValidationError('URLs with embedded credentials are not allowed');
 }
 function normalizeHostname(url) {
-    const hostname = url.hostname.toLowerCase();
+    let hostname = url.hostname.toLowerCase();
+    while (hostname.endsWith('.')) {
+        hostname = hostname.slice(0, -1);
+    }
     if (!hostname) {
         throw createValidationError('URL must have a valid hostname');
     }

package/dist/workers/transform-worker.js

@@ -1,50 +1,94 @@
 import { parentPort } from 'node:worker_threads';
-import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from '../tools/utils/content-transform.js';
-function isTransformJob(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const record = value;
-    return (typeof record.id === 'number' &&
-        typeof record.mode === 'string' &&
-        typeof record.html === 'string' &&
-        typeof record.url === 'string');
+import { FetchError, getErrorMessage } from '../errors.js';
+import { transformHtmlToMarkdownInProcess } from '../transform.js';
+const controllers = new Map();
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
 }
-function resolveTransform(job) {
-    if (job.mode === 'markdown') {
-        return transformHtmlToMarkdown(job.html, job.url, job.options);
+function post(message) {
+    parentPort?.postMessage(message);
+}
+function handleTransform(message) {
+    const controller = new AbortController();
+    controllers.set(message.id, controller);
+    try {
+        const result = transformHtmlToMarkdownInProcess(message.html, message.url, {
+            includeMetadata: message.includeMetadata,
+            signal: controller.signal,
+        });
+        post({
+            type: 'result',
+            id: message.id,
+            result: {
+                markdown: result.markdown,
+                ...(result.title === undefined ? {} : { title: result.title }),
+                truncated: result.truncated,
+            },
+        });
     }
-    if (job.mode === 'markdown-blocks') {
-        return transformHtmlToMarkdownWithBlocks(job.html, job.url, {
-            ...job.options,
-            includeContentBlocks: job.options.includeContentBlocks ?? true,
+    catch (error) {
+        if (error instanceof FetchError) {
+            post({
+                type: 'error',
+                id: message.id,
+                error: {
+                    name: error.name,
+                    message: error.message,
+                    url: error.url,
+                    statusCode: error.statusCode,
+                    details: { ...error.details },
+                },
+            });
+            return;
+        }
+        post({
+            type: 'error',
+            id: message.id,
+            error: {
+                name: error instanceof Error ? error.name : 'Error',
+                message: getErrorMessage(error),
+                url: message.url,
+            },
         });
     }
-    return transformHtmlToJsonl(job.html, job.url, job.options);
+    finally {
+        controllers.delete(message.id);
+    }
 }
-function sendResponse(response) {
-    if (!parentPort)
+function handleCancel(message) {
+    const controller = controllers.get(message.id);
+    if (!controller)
         return;
-    parentPort.postMessage(response);
+    controller.abort(new Error('Canceled'));
 }
-function handleMessage(message) {
-    if (!isTransformJob(message)) {
-        sendResponse({
-            id: -1,
-            ok: false,
-            error: 'Invalid transform job payload',
-        });
+if (!parentPort) {
+    throw new Error('transform-worker started without parentPort');
+}
+parentPort.on('message', (raw) => {
+    if (!isRecord(raw))
+        return;
+    const { type } = raw;
+    if (type === 'cancel') {
+        if (typeof raw.id !== 'string')
+            return;
+        handleCancel({ type: 'cancel', id: raw.id });
         return;
     }
-    try {
-        const result = resolveTransform(message);
-        sendResponse({ id: message.id, ok: true, result });
-    }
-    catch (error) {
-        sendResponse({
-            id: message.id,
-            ok: false,
-            error: error instanceof Error ? error.message : String(error),
+    if (type === 'transform') {
+        if (typeof raw.id !== 'string')
+            return;
+        if (typeof raw.html !== 'string')
+            return;
+        if (typeof raw.url !== 'string')
+            return;
+        if (typeof raw.includeMetadata !== 'boolean')
+            return;
+        handleTransform({
+            type: 'transform',
+            id: raw.id,
+            html: raw.html,
+            url: raw.url,
+            includeMetadata: raw.includeMetadata,
         });
     }
-}
-parentPort?.on('message', handleMessage);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",
@@ -40,6 +40,8 @@
     "start": "node dist/index.js",
     "format": "prettier --write .",
     "type-check": "tsc --noEmit",
+    "type-check:diagnostics": "tsc --noEmit --extendedDiagnostics",
+    "type-check:trace": "node -e \"require('fs').rmSync('.ts-trace',{recursive:true,force:true})\" && tsc --noEmit --generateTrace .ts-trace",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "test": "npm run build --silent && node --test --experimental-transform-types",
@@ -54,29 +56,28 @@
     "@mozilla/readability": "^0.6.0",
     "express": "^5.2.1",
     "linkedom": "^0.18.12",
-    "turndown": "^7.2.2",
-    "undici": "^6.23.0",
+    "node-html-markdown": "^2.0.0",
+    "undici": "^7.18.2",
     "zod": "^4.3.5"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
     "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/express": "^5.0.6",
-    "@types/node": "^22.19.3",
-    "@types/turndown": "^5.0.6",
+    "@types/node": "^22.19.5",
     "eslint": "^9.23.2",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-de-morgan": "^2.0.0",
     "eslint-plugin-depend": "^1.4.0",
     "eslint-plugin-sonarjs": "^3.0.5",
     "eslint-plugin-unused-imports": "^4.3.0",
-    "knip": "^5.80.1",
+    "knip": "^5.80.2",
     "prettier": "^3.7.4",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.52.0"
   },
   "engines": {
-    "node": ">=20.12.0"
+    "node": ">=20.18.1"
   }
 }