mwoffliner 1.17.4 → 2.0.0

package/lib/renderers/abstract.renderer.js

@@ -7,19 +7,24 @@ import RedisStore from '../RedisStore.js';
 import DOMUtils from '../DOMUtils.js';
 import DU from '../DOMUtils.js';
 import { config } from '../config.js';
+import Downloader from '../Downloader.js';
 import { rewriteUrlsOfDoc } from '../util/rewriteUrls.js';
 import { footerTemplate } from '../Templates.js';
-import { getFullUrl, getMediaBase, getRelativeFilePath, interpolateTranslationString, encodeArticleIdForZimHtmlUrl, getStaticFiles } from '../util/misc.js';
+import { getFullUrl, getMediaBase, getRelativeFilePath, encodePageTitleForZimHtmlUrl, makeZimPath } from '../util/misc.js';
 import { processStylesheetContent } from '../util/dump.js';
-import { isMainPage, isSubpage } from '../util/articles.js';
+import { isMainPage, isSubpage } from '../util/pages.js';
+import { buildCategoryTypeItems } from '../util/categories.js';
 export class Renderer {
-    staticFilesListCommon = [];
+    staticFilesListCommon = new Set();
     constructor() {
-        if (this.staticFilesListCommon.length === 0) {
-            this.staticFilesListCommon = getStaticFiles(config.output.jsResourcesCommon, config.output.cssResourcesCommon);
+        if (this.staticFilesListCommon.size === 0) {
+            this.staticFilesListCommon = new Set([...config.output.jsResourcesCommon, ...config.output.cssResourcesCommon]);
         }
     }
-    async treatAudioVideo(dump, srcCache, articleId, audioVideoEl) {
+    addDownloadErrorStaticFiles() {
+        this.staticFilesListCommon = new Set([...this.staticFilesListCommon, ...config.output.downloadErrorResources]);
+    }
+    async treatAudioVideo(dump, srcCache, pagePath, audioVideoEl) {
         const imageDependencies = [];
         const videoDependencies = [];
         const subtitles = [];
@@ -37,17 +42,17 @@ export class Renderer {
         }
         const chosenAudioVideoSourceEl = isVideo ? this.chooseBestVideoSource(audioVideoEl) : this.chooseBestAudioSource(audioVideoEl);
         if (!chosenAudioVideoSourceEl) {
-            logger.warn(`Deleting audio/video HTML node missing an appropriate source in article '${articleId}':\n${audioVideoEl.outerHTML}`);
+            logger.warn(`Deleting audio/video HTML node missing an appropriate source in page at '${pagePath}':\n${audioVideoEl.outerHTML}`);
             DOMUtils.deleteNode(audioVideoEl);
             return { imageDependencies, videoDependencies, subtitles };
         }
         if (isVideo) {
-            this.handleVideoPoster(audioVideoEl, articleId, imageDependencies, srcCache);
+            this.handleVideoPoster(audioVideoEl, pagePath, imageDependencies, srcCache);
         }
-        this.updateAudioVideoSrc(chosenAudioVideoSourceEl, articleId, srcCache, videoDependencies);
+        this.updateAudioVideoSrc(chosenAudioVideoSourceEl, pagePath, srcCache, videoDependencies);
         const trackElements = Array.from(audioVideoEl.querySelectorAll('track'));
         for (const track of trackElements) {
-            subtitles.push(await this.treatSubtitle(track, articleId));
+            subtitles.push(await this.treatSubtitle(track, pagePath));
         }
         return { imageDependencies, videoDependencies, subtitles };
     }
@@ -65,7 +70,7 @@ export class Renderer {
         const videoSourceEls = Array.from(videoEl.children).filter((child) => child.tagName === 'SOURCE');
         const originalSrc = videoEl.getAttribute('src');
         // Take into account the rare edge case where <video> has no <source> child
-        if (videoSourceEls.length == 0) {
+        if (videoSourceEls.length === 0) {
             if (originalSrc && originalSrc.endsWith('.webm')) {
                 // If video has a webm `src` attribute, this is an acceptable src
                 return videoEl;
@@ -142,7 +147,7 @@ export class Renderer {
         audioEl.removeAttribute('src');
         audioEl.removeAttribute('resource');
         // Take into account the standard case where <audio> has no <source> child
-        if (audioSourceEls.length == 0) {
+        if (audioSourceEls.length === 0) {
             if (originalSrc) {
                 // If audio has a single `src` or `resource` attribute, this is an acceptable src
                 // Set this source back in `src`, no matter where it originally was
@@ -175,11 +180,11 @@ export class Renderer {
         });
         return chosenAudioSourceEl;
     }
-    handleVideoPoster(videoEl, articleId, imageDependencies, srcCache) {
+    handleVideoPoster(videoEl, pagePath, imageDependencies, srcCache) {
         const posterUrl = videoEl.getAttribute('poster');
         if (posterUrl) {
             const videoPosterUrl = getFullUrl(posterUrl, MediaWiki.baseUrl);
-            const newVideoPosterUrl = getRelativeFilePath(articleId, getMediaBase(videoPosterUrl, true));
+            const newVideoPosterUrl = getRelativeFilePath(pagePath, getMediaBase(videoPosterUrl, true));
             if (posterUrl) {
                 videoEl.setAttribute('poster', newVideoPosterUrl);
             }
@@ -191,7 +196,7 @@ export class Renderer {
             }
         }
     }
-    updateAudioVideoSrc(chosenVideoSourceEl, articleId, srcCache, videoDependencies) {
+    updateAudioVideoSrc(chosenVideoSourceEl, pagePath, srcCache, videoDependencies) {
         /* Download content, but avoid duplicate calls */
         const sourceUrl = getFullUrl(chosenVideoSourceEl.getAttribute('src'), MediaWiki.baseUrl);
         // eslint-disable-next-line no-prototype-builtins
@@ -201,20 +206,20 @@ export class Renderer {
         }
         /* Set new URL for the video element */
         const fileBase = getMediaBase(sourceUrl, true);
-        chosenVideoSourceEl.setAttribute('src', getRelativeFilePath(articleId, fileBase));
+        chosenVideoSourceEl.setAttribute('src', getRelativeFilePath(pagePath, fileBase));
     }
-    async treatSubtitle(trackEle, articleId) {
+    async treatSubtitle(trackEle, pagePath) {
         const subtitleSourceUrl = getFullUrl(trackEle.getAttribute('src'), MediaWiki.baseUrl);
         const { title, lang } = QueryStringParser.parse(subtitleSourceUrl);
-        // The source URL we get from Mediawiki article is in srt format, so we replace it to vtt which is standard subtitle trackformat for <track> src attribute.
+        // The source URL we get from Mediawiki page is in srt format, so we replace it to vtt which is standard subtitle trackformat for <track> src attribute.
         const vttFormatUrl = new URL(subtitleSourceUrl);
         vttFormatUrl.searchParams.set('trackformat', 'vtt');
-        trackEle.setAttribute('src', `${getRelativeFilePath(articleId, title)}-${lang}.vtt`);
+        trackEle.setAttribute('src', `${getRelativeFilePath(pagePath, title)}-${lang}.vtt`);
         return vttFormatUrl.href;
     }
     treatImageFrames(dump, parsoidDoc, imageNode) {
         const image = imageNode.getElementsByTagName('img')[0] || imageNode.getElementsByTagName('video')[0];
-        if (!this.shouldKeepNode(dump, imageNode, image)) {
+        if (!this.shouldKeepNode(imageNode, image)) {
             DOMUtils.deleteNode(imageNode);
             return;
         }
@@ -240,19 +245,27 @@ export class Renderer {
         thumbDiv.appendChild(thumbinnerDiv);
         imageNode.parentNode.replaceChild(thumbDiv, imageNode);
     }
-    async treatImage(dump, srcCache, articleId, img) {
+    async treatImage(dump, srcCache, zimPath, img) {
         const imageDependencies = [];
-        if (!this.shouldKeepImage(dump, img)) {
+        if (!this.shouldKeepImage(img)) {
             DOMUtils.deleteNode(img);
             return { imageDependencies };
         }
+        /* Preserve inline data: URL images (e.g. from mw.svg) as-is */
+        const imgSrc = img.getAttribute('src');
+        if (imgSrc.startsWith('data:')) {
+            img.setAttribute('loading', 'lazy');
+            return { imageDependencies };
+        }
+        if (dump.nopic && !this.isMathFallbackImage(img)) {
+            this.convertImageToPlaceholder(img);
+            return { imageDependencies };
+        }
         /* Rewrite image src attribute */
         const src = getFullUrl(img.getAttribute('src'), MediaWiki.baseUrl);
         let newSrc;
         try {
-            const slashesInUrl = articleId.split('/').length - 1;
-            const upStr = slashesInUrl ? '../'.repeat(slashesInUrl) : './';
-            newSrc = upStr + getMediaBase(src, true);
+            newSrc = getRelativeFilePath(zimPath, getMediaBase(src, true));
             /* Download image, but avoid duplicate calls */
             // eslint-disable-next-line no-prototype-builtins
             if (!srcCache.hasOwnProperty(src)) {
@@ -273,14 +286,98 @@ export class Renderer {
         img.setAttribute('loading', 'lazy');
         return { imageDependencies };
     }
-    shouldKeepImage(dump, img) {
+    convertImageToPlaceholder(img) {
+        const width = this.getImageDimension(img, 'width');
+        const height = this.getImageDimension(img, 'height');
+        const meaningfulAlt = this.getMeaningfulAltText(img);
+        const svgPayload = this.getPlaceholderSvgPayload(width, height, meaningfulAlt, this.shouldRenderVisibleAlt(width, height));
+        if (!img.getAttribute('width') && width) {
+            img.setAttribute('width', `${width}`);
+        }
+        if (!img.getAttribute('height') && height) {
+            img.setAttribute('height', `${height}`);
+        }
+        img.setAttribute('src', `data:image/svg+xml;charset=UTF-8,${encodeURIComponent(svgPayload)}`);
+        img.removeAttribute('resource');
+        img.removeAttribute('srcset');
+        img.setAttribute('loading', 'lazy');
+    }
+    getImageDimension(img, attrName) {
+        const attrValue = Number(img.getAttribute(attrName));
+        if (Number.isFinite(attrValue) && attrValue > 0) {
+            return attrValue;
+        }
+        const styleValue = this.getImageDimensionFromStyle(img, attrName);
+        if (styleValue) {
+            return styleValue;
+        }
+        const dataAttrName = attrName === 'width' ? 'data-file-width' : 'data-file-height';
+        const dataAttrValue = Number(img.getAttribute(dataAttrName));
+        if (Number.isFinite(dataAttrValue) && dataAttrValue > 0) {
+            return dataAttrValue;
+        }
+        return null;
+    }
+    getImageDimensionFromStyle(img, attrName) {
+        const style = img.getAttribute('style');
+        if (!style) {
+            return null;
+        }
+        const regex = new RegExp(`(?:^|;)\\s*${attrName}\\s*:\\s*([0-9]+(?:\\.[0-9]+)?)\\s*(?:px)?\\s*(?:;|$)`, 'i');
+        const match = style.match(regex);
+        if (!match) {
+            return null;
+        }
+        const value = Math.round(Number(match[1]));
+        return Number.isFinite(value) && value > 0 ? value : null;
+    }
+    shouldRenderVisibleAlt(width, height) {
+        return !!width && !!height && width >= 48 && height >= 24;
+    }
+    getPlaceholderSvgPayload(width, height, meaningfulAlt, shouldRenderVisibleAlt) {
+        const safeWidth = width || 1;
+        const safeHeight = height || 1;
+        const title = meaningfulAlt ? `<title>${this.escapeXml(meaningfulAlt)}</title>` : '';
+        const defs = shouldRenderVisibleAlt ? '<defs><clipPath id="alt-text-clip"><rect x="0" y="0" width="100%" height="100%" /></clipPath></defs>' : '';
+        let text = '';
+        if (meaningfulAlt && shouldRenderVisibleAlt) {
+            const fontSize = Math.max(10, Math.min(16, Math.floor(safeHeight / 3)));
+            text =
+                `<g clip-path="url(#alt-text-clip)">` +
+                    `<text x="50%" y="50%" text-anchor="middle" dominant-baseline="middle" fill="#54595d" font-size="${fontSize}" font-family="sans-serif">${this.escapeXml(meaningfulAlt)}</text>` +
+                    '</g>';
+        }
+        return `<svg xmlns="http://www.w3.org/2000/svg" width="${safeWidth}" height="${safeHeight}" viewBox="0 0 ${safeWidth} ${safeHeight}" preserveAspectRatio="none">${title}${defs}<rect width="100%" height="100%" fill="transparent"/>${text}</svg>`;
+    }
+    getMeaningfulAltText(img) {
+        const alt = img.getAttribute('alt');
+        if (!alt) {
+            return null;
+        }
+        const compactAlt = alt.replace(/\u00a0/g, ' ').trim();
+        return compactAlt.length > 0 ? compactAlt : null;
+    }
+    escapeXml(input) {
+        return input.replace(/[<>&"']/g, (char) => {
+            const entities = {
+                '<': '&lt;',
+                '>': '&gt;',
+                '&': '&amp;',
+                '"': '&quot;',
+                "'": '&apos;',
+            };
+            return entities[char];
+        });
+    }
+    isMathFallbackImage(img) {
         const imageNodeClass = img.getAttribute('class') || '';
+        return imageNodeClass.includes('mwe-math-fallback-image-inline') || img.getAttribute('typeof') === 'mw:Extension/math';
+    }
+    shouldKeepImage(img) {
         const src = img.getAttribute('src');
-        return ((!dump.nopic || imageNodeClass.includes('mwe-math-fallback-image-inline') || img.getAttribute('typeof') === 'mw:Extension/math') &&
-            src &&
-            !src.includes('./Special:FilePath/'));
+        return !!src && !src.includes('./Special:FilePath/');
     }
-    async treatMedias(parsoidDoc, dump, articleId) {
+    async treatMedias(parsoidDoc, dump, pagePath) {
         let imageDependencies = [];
         let videoDependencies = [];
         let subtitles = [];
@@ -290,13 +387,13 @@ export class Renderer {
         const srcCache = {};
         for (const audioVideoEl of audiosAndVideos) {
             // <video /> and <audio />
-            const ret = await this.treatAudioVideo(dump, srcCache, articleId, audioVideoEl);
+            const ret = await this.treatAudioVideo(dump, srcCache, pagePath, audioVideoEl);
             imageDependencies = imageDependencies.concat(ret.imageDependencies);
             videoDependencies = videoDependencies.concat(ret.videoDependencies);
             subtitles = subtitles.concat(ret.subtitles);
         }
         for (const imgEl of imgs) {
-            const ret = await this.treatImage(dump, srcCache, articleId, imgEl);
+            const ret = await this.treatImage(dump, srcCache, pagePath, imgEl);
             imageDependencies = imageDependencies.concat(ret.imageDependencies);
         }
         /* Treat image figures + special spans */
@@ -311,8 +408,8 @@ export class Renderer {
     isStillLinked(image) {
         return image && image.parentNode && image.parentNode.tagName === 'A';
     }
-    shouldKeepNode(dump, imageNode, image) {
-        return !dump.nopic && imageNode && image;
+    shouldKeepNode(imageNode, image) {
+        return !!(imageNode && image);
     }
     makeThumbDiv(dump, parsoidDoc, imageNode) {
         const imageNodeClass = imageNode.getAttribute('class') || '';
@@ -336,16 +433,57 @@ export class Renderer {
         }
         return thumbDiv;
     }
+    getPositiveIntegerAttribute(node, attribute) {
+        const val = Number.parseInt(node.getAttribute(attribute), 10);
+        return Number.isFinite(val) && val > 0 ? val : undefined;
+    }
+    getRequestedImageWidths(doc) {
+        const requestedWidths = {};
+        const setRequestedWidth = (url, width) => {
+            const prevWidth = requestedWidths[url] || 0;
+            requestedWidths[url] = Math.max(prevWidth, width);
+        };
+        for (const image of Array.from(doc.getElementsByTagName('img'))) {
+            const imageSrc = image.getAttribute('src');
+            const width = this.getPositiveIntegerAttribute(image, 'width') || this.getPositiveIntegerAttribute(image, 'data-width');
+            if (!imageSrc || !width) {
+                continue;
+            }
+            try {
+                setRequestedWidth(getFullUrl(imageSrc, MediaWiki.baseUrl), width);
+            }
+            catch {
+                continue;
+            }
+        }
+        for (const video of Array.from(doc.getElementsByTagName('video'))) {
+            const posterSrc = video.getAttribute('poster');
+            const width = this.getPositiveIntegerAttribute(video, 'width');
+            if (!posterSrc || !width) {
+                continue;
+            }
+            try {
+                setRequestedWidth(getFullUrl(posterSrc, MediaWiki.baseUrl), width);
+            }
+            catch {
+                continue;
+            }
+        }
+        return requestedWidths;
+    }
     // TODO: The first part of this method is common for all renders
     async processHtml(processHtmlOpts) {
-        const { html, dump, articleId, articleDetail, displayTitle, moduleDependencies, callback } = processHtmlOpts;
-        let { articleSubtitle } = processHtmlOpts;
+        const { html, dump, pageTitle, pageDetail, displayTitle, categoryMembers, categoriesHtml, moduleDependencies, callback } = processHtmlOpts;
+        let { pageSubtitle } = processHtmlOpts;
         let imageDependencies = [];
         let videoDependencies = [];
         let mediaDependencies = [];
         let subtitles = [];
         let doc = domino.createDocument(html);
-        const ruRet = await rewriteUrlsOfDoc(doc, articleId, dump);
+        // Simplification for now, ZIM path = pageTitle
+        const pagePath = makeZimPath(pageTitle);
+        const pageItems = [];
+        const ruRet = await rewriteUrlsOfDoc(doc, pagePath, dump);
         doc = ruRet.doc;
         mediaDependencies = mediaDependencies.concat(ruRet.mediaDependencies
             .filter((a) => a)
@@ -353,27 +491,81 @@ export class Renderer {
             const path = getMediaBase(url, false);
             return { url, path };
         }));
-        doc = this.applyOtherTreatments(doc, dump, articleId);
-        const tmRet = await this.treatMedias(doc, dump, articleId);
+        // applyOtherTreatments must run before treatMedias so that iframe
+        // placeholders (e.g. YouTube thumbnails) are already in the DOM
+        // when treatMedias processes <img> tags for download into the ZIM.
+        doc = await this.applyOtherTreatments(doc, dump, pagePath);
+        const imageRequestedWidths = this.getRequestedImageWidths(doc);
+        /* The content of a category page, listing all its members */
+        if (pageDetail.categoryinfo) {
+            const categoryinfo = categoryMembers.categoryinfo;
+            const categoryContent = doc.createElement('div');
+            categoryContent.lang = pageDetail.pagelang;
+            categoryContent.dir = pageDetail.pagedir;
+            categoryContent.classList.add('mw-category-generated');
+            if (categoryinfo.size) {
+                const categoryCollation = MediaWiki.metaData.categoryCollation;
+                const numericSorting = categoryCollation === 'numeric' || (categoryCollation.startsWith('uca-') && categoryCollation.endsWith('-u-kn'));
+                moduleDependencies.styleDependenciesList.push('mediawiki.action.styles');
+                await buildCategoryTypeItems('subcats', categoryinfo, categoryMembers, pagePath, pageDetail, doc, dump, numericSorting, moduleDependencies, pageTitle, categoryContent, pageItems);
+                await buildCategoryTypeItems('pages', categoryinfo, categoryMembers, pagePath, pageDetail, doc, dump, numericSorting, moduleDependencies, pageTitle, categoryContent, pageItems);
+                await buildCategoryTypeItems('files', categoryinfo, categoryMembers, pagePath, pageDetail, doc, dump, numericSorting, moduleDependencies, pageTitle, categoryContent, pageItems);
+                if (MediaWiki.getCategories) {
+                    const p = doc.createElement('p');
+                    p.innerHTML = dump.t('categoryLimited');
+                    categoryContent.appendChild(p);
+                }
+            }
+            else {
+                const p = doc.createElement('p');
+                p.innerHTML = dump.t('categoryEmpty');
+                categoryContent.appendChild(p);
+            }
+            const ruRetCategoryContent = await rewriteUrlsOfDoc(categoryContent, pagePath, dump);
+            doc.getElementsByTagName('body')[0].appendChild(ruRetCategoryContent.doc);
+        }
+        const tmRet = await this.treatMedias(doc, dump, pagePath);
         doc = tmRet.doc;
         // Subtitle
-        if (articleSubtitle) {
-            let articleSubtitleDoc = domino.createDocument(articleSubtitle);
-            const ruRetSubtitle = await rewriteUrlsOfDoc(articleSubtitleDoc, articleId, dump);
-            articleSubtitleDoc = ruRetSubtitle.doc;
-            articleSubtitle = articleSubtitleDoc.getElementsByTagName('body')[0].innerHTML;
+        if (pageSubtitle) {
+            let pageSubtitleDoc = domino.createDocument(pageSubtitle);
+            const ruRetSubtitle = await rewriteUrlsOfDoc(pageSubtitleDoc, pagePath, dump);
+            pageSubtitleDoc = ruRetSubtitle.doc;
+            pageSubtitle = pageSubtitleDoc.getElementsByTagName('body')[0].innerHTML;
+        }
+        // List of categories the page is a member of
+        if (categoriesHtml) {
+            let categoriesDoc = domino.createDocument(categoriesHtml);
+            // Remove categories which do not have an associated page
+            const catLinksDiv = categoriesDoc.getElementById('catlinks');
+            const listItems = catLinksDiv.getElementsByTagName('li');
+            for (const listItem of Array.from(listItems)) {
+                const link = listItem.getElementsByTagName('a')[0];
+                if (Array.from(link.classList).includes('new')) {
+                    listItem.parentNode.removeChild(listItem);
+                }
+            }
+            const nbRemainingLinks = catLinksDiv.getElementsByTagName('li').length;
+            // Rewrite and add categories HTML only if there is at least one category remaining in the list
+            if (nbRemainingLinks > 0) {
+                const ruRetCategories = await rewriteUrlsOfDoc(categoriesDoc, pagePath, dump);
+                categoriesDoc = ruRetCategories.doc;
+                doc.getElementsByTagName('body')[0].appendChild(categoriesDoc.getElementById('catlinks'));
+            }
         }
         videoDependencies = videoDependencies.concat(tmRet.videoDependencies
             .filter((a) => a)
             .map((url) => {
             const path = getMediaBase(url, false);
-            return { url, path };
+            const width = imageRequestedWidths[url];
+            return width ? { url, path, width } : { url, path };
         }));
         imageDependencies = imageDependencies.concat(tmRet.imageDependencies
             .filter((a) => a)
             .map((url) => {
             const path = getMediaBase(url, false);
-            return { url, path };
+            const width = imageRequestedWidths[url];
+            return width ? { url, path, width } : { url, path };
         }));
         subtitles = subtitles.concat(tmRet.subtitles
             .filter((a) => a)
@@ -382,15 +574,16 @@ export class Renderer {
             const path = `${title}-${lang}.vtt`;
             return { url, path };
         }));
-        if (!isMainPage(articleId) && dump.customProcessor?.preProcessArticle) {
-            doc = await dump.customProcessor.preProcessArticle(articleId, doc);
+        if (!isMainPage(pageTitle) && dump.customProcessor?.preProcessPage) {
+            doc = await dump.customProcessor.preProcessPage(pageTitle, doc);
         }
-        let templatedDoc = callback(moduleDependencies, articleId);
-        templatedDoc = await this.mergeTemplateDoc(templatedDoc, doc, dump, articleDetail, RedisStore.articleDetailXId, articleId, displayTitle, articleSubtitle);
-        if (dump.customProcessor && dump.customProcessor.postProcessArticle) {
-            templatedDoc = await dump.customProcessor.postProcessArticle(articleId, templatedDoc);
+        const templateDoc = callback(moduleDependencies, pagePath);
+        const pageFinalDoc = domino.createDocument(templateDoc.documentElement.outerHTML);
+        await this.mergeTemplateDoc(pageFinalDoc, doc, dump, pageDetail, pageTitle, displayTitle, pageSubtitle);
+        if (dump.customProcessor && dump.customProcessor.postProcessPage) {
+            await dump.customProcessor.postProcessPage(pageTitle, pageFinalDoc);
         }
-        let outHtml = templatedDoc.documentElement.outerHTML;
+        let outHtml = pageFinalDoc.documentElement.outerHTML;
         if (dump.opts.minifyHtml) {
             outHtml = await htmlMinifier.minify(outHtml, {
                 removeComments: true,
@@ -401,46 +594,70 @@ export class Renderer {
                 minifyCSS: true,
             });
         }
-        const finalHTML = '<!DOCTYPE html>\n' + outHtml;
+        const div = doc.createElement('div');
+        div.innerHTML = displayTitle;
+        const displayTitleText = div.innerText;
+        pageItems.push({
+            pageTitle,
+            zimPath: pagePath,
+            zimTitle: displayTitleText ? displayTitleText : pageTitle,
+            htmlContent: '<!DOCTYPE html>\n' + outHtml,
+        });
         return {
-            finalHTML,
+            items: pageItems,
             mediaDependencies,
             imageDependencies,
             videoDependencies,
+            moduleDependencies,
             subtitles,
+            needsDownloadErrorStaticFiles: false,
         };
     }
-    async mergeTemplateDoc(htmlTemplateDoc, parsoidDoc, dump, articleDetail, articleDetailXId, articleId, displayTitle, articleSubtitle) {
+    async mergeTemplateDoc(htmlTemplateDoc, parsoidDoc, dump, pageDetail, pageTitle, displayTitle, pageSubtitle) {
         /* Create final document by merging template and parsoid documents */
         const mwContentText = htmlTemplateDoc.getElementById('mw-content-text');
-        mwContentText.lang = articleDetail.pagelang;
-        mwContentText.dir = articleDetail.pagedir;
-        mwContentText.classList.add('mw-content-' + articleDetail.pagedir);
+        mwContentText.lang = pageDetail.pagelang;
+        mwContentText.dir = pageDetail.pagedir;
+        mwContentText.classList.add('mw-content-' + pageDetail.pagedir);
         mwContentText.innerHTML = parsoidDoc.getElementsByTagName('body')[0].innerHTML;
         /* Title */
-        const articleTitle = htmlTemplateDoc.getElementById('title_0') ? htmlTemplateDoc.getElementById('title_0').textContent : articleId.replace(/_/g, ' ');
-        htmlTemplateDoc.getElementsByTagName('title')[0].innerHTML = articleTitle;
+        let htmlTitle;
+        if (htmlTemplateDoc.getElementById('title_0')) {
+            htmlTitle = htmlTemplateDoc.getElementById('title_0').innerText;
+        }
+        else if (displayTitle) {
+            // displayTitle may contain HTML markup (e.g. italics), which has no place in the <title> tag
+            const titleDiv = htmlTemplateDoc.createElement('div');
+            titleDiv.innerHTML = displayTitle;
+            htmlTitle = titleDiv.textContent;
+        }
+        else {
+            htmlTitle = pageTitle;
+        }
+        htmlTemplateDoc.getElementsByTagName('title')[0].textContent = htmlTitle;
         // Set inline page title when missing
         const inlineTitle = htmlTemplateDoc.getElementById('firstHeading');
         if (inlineTitle && !inlineTitle.innerHTML) {
-            inlineTitle.innerHTML = displayTitle || articleTitle;
+            inlineTitle.innerHTML = displayTitle || htmlTitle;
         }
         DOMUtils.deleteNode(htmlTemplateDoc.getElementById('titleHeading'));
         /* Subpage */
         if (this.constructor.name === 'ActionParseRenderer') {
             const mwContentSubtitle = htmlTemplateDoc.getElementById('mw-content-subtitle');
-            mwContentSubtitle.innerHTML = articleSubtitle || '';
+            mwContentSubtitle.innerHTML = pageSubtitle || '';
         }
-        else if (isSubpage(articleId) && !isMainPage(articleId)) {
+        else if (isSubpage(pageTitle) && !isMainPage(pageTitle)) {
             const headingNode = htmlTemplateDoc.getElementById('mw-content-text');
             const subpagesNode = htmlTemplateDoc.createElement('span');
-            const parents = articleId.split('/');
+            const parents = pageTitle.split('/');
             parents.pop();
             let subpages = '';
             await Promise.all(parents.map(async (parent) => {
                 const label = parent.replace(/_/g, ' ');
-                const isParentMirrored = await articleDetailXId.exists(`${articleId.split(parent)[0]}${parent}`);
-                subpages += `&lt; ${isParentMirrored ? `<a href="${'../'.repeat(parents.length)}${encodeArticleIdForZimHtmlUrl(`${articleId.split(parent)[0]}${parent}`)}" title="${label}">` : ''}${label}${isParentMirrored ? '</a> ' : ' '}`;
+                const isParentMirrored = await RedisStore.pagesStore.exists(`${pageTitle.split(parent)[0]}${parent}`);
+                subpages += `&lt; ${isParentMirrored
+                    ? `<a href="${getRelativeFilePath(makeZimPath(pageTitle), encodePageTitleForZimHtmlUrl(`${pageTitle.split(parent)[0]}${parent}`))}" title="${label}">`
+                    : ''}${label}${isParentMirrored ? '</a> ' : ' '}`;
             }));
             subpagesNode.innerHTML = subpages;
             subpagesNode.setAttribute('class', 'subpages');
@@ -449,31 +666,23 @@ export class Renderer {
         /* Set footer */
         const div = htmlTemplateDoc.createElement('div');
         /* Revision date */
-        const date = new Date(articleDetail.timestamp);
-        const lastEditedOnString = date
-            ? interpolateTranslationString(dump.strings.LAST_EDITED_ON, {
-                date: date.toISOString().substring(0, 10),
-            })
-            : null;
+        const date = new Date(pageDetail.timestamp);
+        const lastEditedOnString = date ? dump.t('LAST_EDITED_ON', { date: date.toISOString().substring(0, 10) }) : null;
         const creatorLink = '<a class="external text" ' +
             `${lastEditedOnString ? `title="${lastEditedOnString}"` : ''} ` +
-            `href="${MediaWiki.webUrl.href}?title=${encodeURIComponent(articleId)}&oldid=${articleDetail.revisionId}">` +
+            `href="${MediaWiki.webUrl.href}?title=${encodeURIComponent(pageTitle.replace(/ /g, '_'))}&oldid=${pageDetail.revisionId}">` +
             `${dump.mwMetaData.title || dump.mwMetaData.creator}</a>`;
         const licenseLink = (dump.mwMetaData.licenseUrl ? `<a class="external text" href="${dump.mwMetaData.licenseUrl}">` : '') +
-            (dump.mwMetaData.licenseName || dump.mwMetaData.licenseUrl || dump.strings.LICENSE_UNKNOWN).replace(/</g, '&lt;') +
+            (dump.mwMetaData.licenseName || dump.mwMetaData.licenseUrl || dump.t('LICENSE_UNKNOWN')).replace(/</g, '&lt;') +
             (dump.mwMetaData.licenseUrl ? `</a>` : '');
         div.innerHTML = footerTemplate({
-            disclaimer: interpolateTranslationString(dump.strings.DISCLAIMER, {
-                creator: creatorLink,
-                license: licenseLink,
-            }),
-            strings: dump.strings,
+            disclaimer: dump.t('DISCLAIMER', { creator: creatorLink, license: licenseLink }),
         });
-        htmlTemplateDoc.getElementById('mw-content-text').appendChild(div);
+        mwContentText.appendChild(div);
         this.addNoIndexCommentToElement(div);
         /* Geo-coordinates */
-        if (articleDetail.coordinates) {
-            const geoCoordinates = articleDetail.coordinates;
+        if (pageDetail.coordinates) {
+            const geoCoordinates = pageDetail.coordinates;
             const metaNode = htmlTemplateDoc.createElement('meta');
             metaNode.name = 'geo.position';
             metaNode.content = geoCoordinates;
@@ -485,13 +694,81 @@ export class Renderer {
         const slices = element.parentElement.innerHTML.split(element.outerHTML);
         element.parentElement.innerHTML = `${slices[0]}<!--htdig_noindex-->${element.outerHTML}<!--/htdig_noindex-->${slices[1]}`;
     }
-    removeIframeTags(parsoidDoc) {
-        // Remove all <iframe> tags
+    // Rewrite iframe tags into external placeholders
+    processIframeTags(parsoidDoc) {
         const iframes = Array.from(parsoidDoc.getElementsByTagName('iframe'));
         for (const iframe of iframes) {
-            DU.deleteNode(iframe);
+            const src = iframe.getAttribute('src');
+            if (!src) {
+                DU.deleteNode(iframe);
+                continue;
+            }
+            let fullSrc = src;
+            try {
+                fullSrc = getFullUrl(src, MediaWiki.baseUrl);
+            }
+            catch {
+                // Keep original src when URL parsing fails
+            }
+            const placeholder = this.isYouTubeIframeUrl(fullSrc)
+                ? this.createYouTubePlaceholder(parsoidDoc, iframe, fullSrc, this.extractYouTubeVideoId(fullSrc))
+                : this.createExternalIframePlaceholder(parsoidDoc, iframe, fullSrc);
+            if (iframe.parentNode) {
+                iframe.parentNode.replaceChild(placeholder, iframe);
+            }
+            else {
+                DU.deleteNode(iframe);
+            }
+        }
+    }
+    isYouTubeIframeUrl(url) {
+        const lowerUrl = url.toLowerCase();
+        return lowerUrl.includes('youtube.com/embed/') || lowerUrl.includes('youtube-nocookie.com/embed/') || lowerUrl.includes('youtu.be/') || lowerUrl.includes('youtube.com/watch');
+    }
+    extractYouTubeVideoId(url) {
+        const match = url.match(/(?:embed\/|v=|\.be\/)([a-zA-Z0-9_-]+)/);
+        return match ? match[1] : null;
+    }
+    createExternalIframePlaceholder(parsoidDoc, iframe, originalUrl) {
+        const card = parsoidDoc.createElement('div');
+        this.copyNodeAttributes(iframe, card);
+        const iframeClass = card.getAttribute('class');
+        card.setAttribute('class', iframeClass ? `${iframeClass} mwo-external-video-card` : 'mwo-external-video-card');
+        const link = parsoidDoc.createElement('a');
+        link.setAttribute('class', 'external');
+        link.setAttribute('href', originalUrl);
+        link.textContent = 'Open embedded content (external)';
+        card.appendChild(link);
+        return card;
+    }
+    copyNodeAttributes(sourceNode, targetNode) {
+        for (let attributeIndex = 0; attributeIndex < sourceNode.attributes.length; attributeIndex++) {
+            const attribute = sourceNode.attributes.item(attributeIndex);
+            if (!attribute) {
+                continue;
+            }
+            targetNode.setAttribute(attribute.name, attribute.value);
         }
     }
+    createYouTubePlaceholder(parsoidDoc, iframe, originalUrl, videoId) {
+        const link = parsoidDoc.createElement('a');
+        link.setAttribute('class', 'external');
+        link.setAttribute('href', videoId ? `https://www.youtube.com/watch?v=${videoId}` : originalUrl);
+        if (!videoId) {
+            link.textContent = 'View this content externally';
+            link.setAttribute('aria-label', 'View this content externally');
+            return link;
+        }
+        const thumbnail = parsoidDoc.createElement('img');
+        this.copyNodeAttributes(iframe, thumbnail);
+        // Set the online thumbnail URL here; treatMedias will process this
+        // <img> tag later to download the image and rewrite src to a local path in the ZIM.
+        thumbnail.setAttribute('src', `https://img.youtube.com/vi/${videoId}/hqdefault.jpg`);
+        thumbnail.setAttribute('alt', 'Video thumbnail');
+        thumbnail.setAttribute('loading', 'lazy');
+        link.appendChild(thumbnail);
+        return link;
+    }
     removeCitations(parsoidDoc) {
         // Remove all citation-related material
         const sups = Array.from(parsoidDoc.getElementsByTagName('sup'));
@@ -600,8 +877,8 @@ export class Renderer {
             node.textContent = '🌍';
         }
     }
-    applyOtherTreatments(parsoidDoc, dump, articleId) {
-        this.removeIframeTags(parsoidDoc);
+    async applyOtherTreatments(parsoidDoc, dump, pagePath) {
+        this.processIframeTags(parsoidDoc);
         if (dump.nodet) {
             this.removeCitations(parsoidDoc);
         }
@@ -632,7 +909,7 @@ export class Renderer {
             // We use MediaWiki.baseUrl which is an approximation but it is deemed sufficient because
             // all non-absolute URL found in inline CSS are expected to be relative to the root, not to
             // current web URL which is "moving" (could use something like /w/index.php?title=... or /wiki/...)
-            style.textContent = processStylesheetContent(MediaWiki.baseUrl.toString(), '', style.textContent, articleId);
+            style.textContent = await processStylesheetContent(MediaWiki.baseUrl.toString(), '', style.textContent, pagePath);
         }
         /* Remove element with id in the blacklist */
         filtersConfig.idBlackList.forEach((id) => {
@@ -642,11 +919,13 @@ export class Renderer {
             }
         });
         /*
-         * Because of CSP, some ZIM reader environments do not allow inline JS. See issues/2096.
+         * We can't trust the content of inline scripts, only keep when all JavaScript is allowed
          */
-        const scripts = Array.from(parsoidDoc.getElementsByTagName('script'));
-        for (const script of scripts) {
-            script.parentNode.removeChild(script);
+        if (!Downloader.trustedJs || Downloader.trustedJs.length) {
+            const scripts = Array.from(parsoidDoc.getElementsByTagName('script'));
+            for (const script of scripts) {
+                script.parentNode.removeChild(script);
+            }
         }
         /* Force display of element with that CSS class */
         filtersConfig.cssClassDisplayList.map((classname) => {
@@ -674,14 +953,14 @@ export class Renderer {
         return parsoidDoc;
     }
     /**
-     * Add an H1 tag with page title on top of article except main page
+     * Add an H1 tag with page title on top of page except main page
      */
-    injectH1TitleToHtml(content, articleDetail) {
+    injectH1TitleToHtml(content, pageDetail) {
         const doc = domino.createDocument(content);
         const header = doc.createElement('h1');
-        if (articleDetail?.title) {
-            header.appendChild(doc.createTextNode(articleDetail.title));
-            header.classList.add('article-header');
+        if (pageDetail?.title) {
+            header.appendChild(doc.createTextNode(pageDetail.title));
+            header.classList.add('page-header');
             const target = doc.querySelector('body.mw-body-content');
             if (target) {
                 target.insertAdjacentElement('afterbegin', header);