@j0hanz/superfetch 2.6.0 → 2.7.0

package/dist/transform.js

@@ -325,26 +325,32 @@ const META_NAME_HANDLERS = new Map([
         },
     ],
 ]);
-function extractMetadata(document) {
+function processMetaTag(ctx, tag) {
+    const content = tag.getAttribute('content')?.trim();
+    if (!content)
+        return;
+    const property = tag.getAttribute('property');
+    if (property)
+        META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
+    const name = tag.getAttribute('name');
+    if (name)
+        META_NAME_HANDLERS.get(name)?.(ctx, content);
+}
+function buildMetaContext(document) {
     const ctx = { title: {}, description: {} };
     for (const tag of document.querySelectorAll('meta')) {
-        const content = tag.getAttribute('content')?.trim();
-        if (!content)
-            continue;
-        const property = tag.getAttribute('property');
-        if (property)
-            META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
-        const name = tag.getAttribute('name');
-        if (name)
-            META_NAME_HANDLERS.get(name)?.(ctx, content);
+        processMetaTag(ctx, tag);
     }
     const titleEl = document.querySelector('title');
     if (!ctx.title.standard && titleEl?.textContent) {
         ctx.title.standard = titleEl.textContent.trim();
     }
+    return ctx;
+}
+function resolveMetadataFromContext(ctx) {
+    const metadata = {};
     const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
     const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
-    const metadata = {};
     if (resolvedTitle)
         metadata.title = resolvedTitle;
     if (resolvedDesc)
@@ -359,6 +365,37 @@ function extractMetadata(document) {
         metadata.modifiedAt = ctx.modifiedAt;
     return metadata;
 }
+function extractMetadata(document, baseUrl) {
+    const ctx = buildMetaContext(document);
+    const metadata = resolveMetadataFromContext(ctx);
+    if (baseUrl) {
+        const icon32 = document.querySelector('link[rel="icon"][sizes="32x32"]');
+        const href = icon32?.getAttribute('href');
+        if (href) {
+            const resolved = resolveFaviconUrl(href, baseUrl);
+            if (resolved)
+                metadata.favicon = resolved;
+        }
+    }
+    return metadata;
+}
+function resolveFaviconUrl(href, baseUrl) {
+    const trimmed = href.trim();
+    if (!trimmed)
+        return undefined;
+    if (trimmed.toLowerCase().startsWith('data:'))
+        return undefined;
+    try {
+        const resolved = new URL(trimmed, baseUrl);
+        if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') {
+            return undefined;
+        }
+        return resolved.toString();
+    }
+    catch {
+        return undefined;
+    }
+}
 function isReadabilityCompatible(doc) {
     if (!isObject(doc))
         return false;
@@ -469,7 +506,7 @@ function extractContentContext(html, url, options) {
         const { document } = stageTracker.run(url, 'extract:parse', () => parseHTML(limitedHtml));
         abortPolicy.throwIfAborted(options.signal, url, 'extract:parsed');
         applyBaseUri(document, url);
-        const lateMetadata = stageTracker.run(url, 'extract:metadata', () => extractMetadata(document));
+        const lateMetadata = stageTracker.run(url, 'extract:metadata', () => extractMetadata(document, url));
         abortPolicy.throwIfAborted(options.signal, url, 'extract:metadata');
         // Merge early (pre-truncation) with late (post-truncation) metadata
         const metadata = mergeMetadata(earlyMetadata, lateMetadata);
@@ -615,6 +652,59 @@ function buildCodeTranslator(ctx) {
         return buildInlineCodeTranslator();
     return { noEscape: true, preserveWhitespace: true };
 }
+function extractFirstSrcsetUrl(srcset) {
+    const first = srcset.split(',')[0];
+    if (!first)
+        return '';
+    return first.trim().split(/\s+/)[0] ?? '';
+}
+const LAZY_SRC_ATTRIBUTES = [
+    'data-src',
+    'data-lazy-src',
+    'data-original',
+    'data-srcset',
+];
+function extractNonDataSrcsetUrl(value) {
+    const url = extractFirstSrcsetUrl(value);
+    return url && !url.startsWith('data:') ? url : undefined;
+}
+function resolveLazySrc(getAttribute) {
+    for (const attr of LAZY_SRC_ATTRIBUTES) {
+        const lazy = getAttribute(attr);
+        if (!lazy || lazy.startsWith('data:'))
+            continue;
+        if (attr === 'data-srcset') {
+            const url = extractNonDataSrcsetUrl(lazy);
+            if (url)
+                return url;
+            continue;
+        }
+        return lazy;
+    }
+    return undefined;
+}
+function resolveImageSrc(getAttribute) {
+    if (!getAttribute)
+        return '';
+    const srcRaw = getAttribute('src') ?? '';
+    if (srcRaw && !srcRaw.startsWith('data:'))
+        return srcRaw;
+    // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
+    const lazySrc = resolveLazySrc(getAttribute);
+    if (lazySrc)
+        return lazySrc;
+    // If the src is a data URI or missing, check srcset for a valid URL. Some sites use srcset with data URIs in src and actual URLs in srcset for responsive images.
+    const srcset = getAttribute('srcset');
+    if (srcset) {
+        const url = extractNonDataSrcsetUrl(srcset);
+        if (url)
+            return url;
+    }
+    // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
+    if (srcRaw.startsWith('data:'))
+        return '[data URI removed]';
+    return '';
+}
 function buildImageTranslator(ctx) {
     if (!isObject(ctx))
         return { content: '' };
@@ -622,8 +712,7 @@ function buildImageTranslator(ctx) {
     const getAttribute = hasGetAttribute(node)
         ? node.getAttribute.bind(node)
         : undefined;
-    const srcRaw = getAttribute?.('src') ?? '';
-    const src = srcRaw.startsWith('data:') ? '[data URI removed]' : srcRaw;
+    const src = resolveImageSrc(getAttribute);
     const existingAlt = getAttribute?.('alt') ?? '';
     const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
     const markdown = `![${alt}](${src})`;
@@ -794,11 +883,11 @@ function createCustomTranslators() {
                 const trimmed = content.trim();
                 if (!trimmed)
                     return '';
-                return `\n\n<details>\n${trimmed}\n</details>\n\n`;
+                return `\n\n${trimmed}\n\n`;
             },
         }),
         summary: () => ({
-            postprocess: ({ content }) => `<summary>${content.trim()}</summary>\n\n`,
+            postprocess: ({ content }) => `${content.trim()}\n\n`,
         }),
         span: (ctx) => {
             if (!isObject(ctx) || !isObject(ctx.node))
@@ -1315,6 +1404,7 @@ function buildContentSource(params) {
         return {
             sourceHtml: cleanedArticleHtml,
             title: article.title,
+            favicon: extractedMeta.favicon,
             metadata,
             skipNoiseRemoval: true,
             truncated,
@@ -1329,6 +1419,7 @@ function buildContentSource(params) {
             return {
                 sourceHtml: contentRoot,
                 title: extractedMeta.title,
+                favicon: extractedMeta.favicon,
                 metadata,
                 skipNoiseRemoval: true,
                 document,
@@ -1338,6 +1429,7 @@ function buildContentSource(params) {
         return {
             sourceHtml: cleanedHtml,
             title: extractedMeta.title,
+            favicon: extractedMeta.favicon,
             metadata,
             skipNoiseRemoval: true,
             document,
@@ -1347,6 +1439,7 @@ function buildContentSource(params) {
     return {
         sourceHtml: html,
         title: extractedMeta.title,
+        favicon: extractedMeta.favicon,
         metadata,
         truncated,
     };
@@ -1379,7 +1472,19 @@ function buildMarkdownFromContext(context, url, signal) {
         ...(context.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
     }));
     if (context.title && !content.trim().startsWith('# ')) {
-        content = `# ${context.title}\n\n${content}`;
+        const icon = context.favicon;
+        let prefix = ' ';
+        if (icon) {
+            let alt = '';
+            try {
+                alt = new URL(url).hostname;
+            }
+            catch {
+                /* skip */
+            }
+            prefix = ` ![${alt}](${icon}) `;
+        }
+        content = `#${prefix}${context.title}\n\n${content}`;
     }
     return {
         markdown: content,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/superfetch",
-  "version": "2.6.0",
+  "version": "2.7.0",
   "mcpName": "io.github.j0hanz/superfetch",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",