@j0hanz/fetch-url-mcp 1.10.14 → 1.10.16

package/dist/lib/dom-prep.d.ts

@@ -1,3 +1,4 @@
+export declare function extractNoscriptImages(document: Document): void;
 export declare function resolveDocumentBody(document: Document): Element;
 export declare function serializeDocumentForMarkdown(document: Document, fallback: string): string;
 /** Surface hidden tab panels, then strip unselected tab triggers. */

package/dist/lib/dom-prep.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AA4lBA,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAY/D;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CASR;AA0CD,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAG5D;AAyTD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAMN;AA4BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR"}
+{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAwnBA,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CA6B9D;AAuBD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAY/D;AAED,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CASR;AA0CD,qEAAqE;AACrE,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAG5D;AAyTD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAON;AA4BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR"}

package/dist/lib/dom-prep.js

@@ -497,10 +497,90 @@ function processUrlElement(el, attr, base, isSrcset) {
             el.setAttribute(attr, resolved.href);
     }
 }
+// Rewrite WordPress Photon CDN image URLs to point to the original host, since srcset URLs are often preserved with the updated domain while src is not.
+// This ensures images are correctly resolved when the page is migrated to a new domain but still references the old domain in img src attributes.
+const WP_PHOTON_HOST_PATTERN = /^i\d\.wp\.com$/;
+function rewritePhotonSrc(document, pageHost) {
+    for (const img of document.querySelectorAll('img[src]')) {
+        const src = img.getAttribute('src');
+        if (!src)
+            continue;
+        const parsed = URL.parse(src);
+        if (!parsed || !WP_PHOTON_HOST_PATTERN.test(parsed.hostname))
+            continue;
+        if (img.getAttribute('srcset'))
+            continue;
+        const segments = parsed.pathname.split('/').filter(Boolean);
+        if (segments.length < 2)
+            continue;
+        const originHost = segments[0];
+        if (!originHost?.includes('.'))
+            continue;
+        const resourcePath = `/${segments.slice(1).join('/')}`;
+        const rewritten = `https://${pageHost}${resourcePath}`;
+        img.setAttribute('src', rewritten);
+    }
+}
+// For images with src URLs pointing to a different domain than the page, check if their srcset contains a same-domain URL and prefer that for the src attribute.
+// This can help preserve image loading when migrating content that references an old domain, as srcset entries are often left unchanged while src attributes are updated or removed.
+function preferSameDomainSrc(document, base) {
+    const pageHost = base.hostname;
+    for (const img of document.querySelectorAll('img[src][srcset]')) {
+        const src = img.getAttribute('src');
+        if (!src)
+            continue;
+        const srcParsed = URL.parse(src);
+        if (!srcParsed || srcParsed.hostname === pageHost)
+            continue;
+        const srcset = img.getAttribute('srcset') ?? '';
+        const entries = srcset.split(',');
+        for (const entry of entries) {
+            const url = entry.trim().split(/\s+/)[0];
+            if (!url)
+                continue;
+            const parsed = URL.parse(url);
+            if (parsed?.hostname === pageHost) {
+                img.setAttribute('src', url);
+                break;
+            }
+        }
+    }
+}
+export function extractNoscriptImages(document) {
+    for (const noscript of document.querySelectorAll('noscript')) {
+        // linkedom may parse noscript children as DOM or raw text — handle both.
+        let imgs = Array.from(noscript.querySelectorAll('img'));
+        if (imgs.length === 0) {
+            const html = noscript.innerHTML || noscript.textContent || '';
+            if (!/<img\b/i.test(html))
+                continue;
+            const { document: fragDoc } = parseHTML(`<body>${html}</body>`);
+            imgs = Array.from(fragDoc.querySelectorAll('img'));
+        }
+        if (imgs.length === 0)
+            continue;
+        // Skip when the previous sibling is (or contains) an <img> — the
+        // lazy-loaded placeholder is already in the DOM and the translators
+        // handle data-src / placeholder detection.
+        const prev = noscript.previousElementSibling;
+        if (prev?.tagName === 'IMG' || prev?.querySelector('img'))
+            continue;
+        for (const img of imgs) {
+            // Skip tracking pixels (commonly 1×1 images placed in noscript by
+            // analytics providers).
+            if (img.getAttribute('width') === '1' ||
+                img.getAttribute('height') === '1')
+                continue;
+            noscript.before(img.cloneNode(true));
+        }
+    }
+}
 function resolveUrls(document, baseUrlStr) {
     const base = URL.parse(baseUrlStr);
     if (!base)
         return;
+    rewritePhotonSrc(document, base.hostname);
+    preferSameDomainSrc(document, base);
     const elements = document.querySelectorAll('a[href],img[src],source[srcset]');
     for (const el of elements) {
         const tag = el.tagName.toLowerCase();
@@ -813,6 +893,7 @@ function runUrlResolutionPass(document, baseUrl) {
 // (post-Readability). Some passes (stripTabTriggers, etc.) are no-ops
 // on Readability output since tabs are already stripped or absent.
 export function prepareDocumentForMarkdown(document, baseUrl, signal) {
+    extractNoscriptImages(document);
     runDocsControlPass(document);
     runStructuralNoisePass(document, signal);
     runCodeExamplePass(document);

package/dist/transform/html-translators.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAwdA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}
+{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAwfA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}

package/dist/transform/html-translators.js

@@ -97,12 +97,23 @@ function extractFirstSrcsetUrl(srcset) {
 const LAZY_SRC_ATTRIBUTES = [
     'data-src',
     'data-lazy-src',
+    'data-lazy',
     'data-original',
+    'data-echo',
     'data-srcset',
 ];
 function isDataUri(value) {
     return value.startsWith('data:');
 }
+const PLACEHOLDER_FILENAME_PATTERN = /(?:^|\/)(?:blank|spacer|placeholder|grey|gray|pixel|loading|lazy|transparent|empty|dummy)\.[a-z]{3,4}$/i;
+function isPlaceholderSrc(value) {
+    if (isDataUri(value))
+        return true;
+    const parsed = URL.parse(value) ?? URL.parse(value, 'http://localhost');
+    if (!parsed)
+        return false;
+    return PLACEHOLDER_FILENAME_PATTERN.test(parsed.pathname);
+}
 function extractNonDataSrcsetUrl(value) {
     const url = extractFirstSrcsetUrl(value);
     return url && !isDataUri(url) ? url : undefined;
@@ -122,11 +133,28 @@ function resolveLazySrc(getAttribute) {
     }
     return undefined;
 }
+// Some sites (notably WordPress with Photon CDN) use a CDN proxy URL in img src while keeping the original same-domain URL in srcset.
+// Since the converter prefers srcset URLs for CDN-hosted images, we need to detect this pattern and extract the canonical URL from srcset to ensure images are correctly resolved, especially when migrating content to a new domain.
+const WP_PHOTON_HOST_PATTERN = /^i\d\.wp\.com$/;
+function isWpPhotonUrl(src) {
+    const parsed = URL.parse(src);
+    return parsed !== null && WP_PHOTON_HOST_PATTERN.test(parsed.hostname);
+}
 function resolveImageSrc(getAttribute) {
     if (!getAttribute)
         return '';
     const srcRaw = getAttribute('src') ?? '';
-    if (srcRaw && !isDataUri(srcRaw))
+    // When src is a CDN proxy URL, prefer srcset which usually has the
+    // canonical same-domain URL that survives domain migrations.
+    if (srcRaw && isWpPhotonUrl(srcRaw)) {
+        const srcset = getAttribute('srcset');
+        if (srcset) {
+            const url = extractNonDataSrcsetUrl(srcset);
+            if (url)
+                return url;
+        }
+    }
+    if (srcRaw && !isPlaceholderSrc(srcRaw))
         return srcRaw;
     // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
     const lazySrc = resolveLazySrc(getAttribute);

package/dist/transform/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AA2DA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAmCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA6ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAicD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA4KD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AA0ID,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAgED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAwjBD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAgBzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA+G1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AA4DA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAmCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA6ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAkcD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA4KD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AA0ID,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAgED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAwjBD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAgBzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA+G1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}

package/dist/transform/transform.js

@@ -5,7 +5,7 @@ import { parseHTML } from 'linkedom';
 import { extractLanguageFromClassName } from '../lib/code-lang.js';
 import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
-import { normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, resolveDocumentBody, serializeDocumentForMarkdown, } from '../lib/dom-prep.js';
+import { extractNoscriptImages, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, resolveDocumentBody, serializeDocumentForMarkdown, } from '../lib/dom-prep.js';
 import { isRawTextContentUrl } from '../lib/http.js';
 import { cleanupMarkdownArtifacts, processFencedContent, } from '../lib/md-cleanup.js';
 import { addSourceToMarkdown, buildMetadataFooter, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/md-metadata.js';
@@ -328,6 +328,7 @@ function preserveCodeLanguageAttributes(doc) {
 }
 const STRUCTURAL_SKIP_TAGS = new Set(['HTML', 'BODY']);
 function prepareReadabilityDocument(readabilityDoc) {
+    extractNoscriptImages(readabilityDoc);
     preserveGalleryImages(readabilityDoc);
     preserveAlertElements(readabilityDoc);
     preserveCodeLanguageAttributes(readabilityDoc);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.10.14",
+  "version": "1.10.16",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
   "type": "module",