mwoffliner 1.17.4 → 2.0.0

package/lib/Downloader.js

@@ -1,17 +1,11 @@
 import * as backoff from 'backoff';
 import { config } from './config.js';
-import { contains, normalizeMwResponse, DB_ERROR, WEAK_ETAG_REGEX, stripHttpFromUrl, isBitmapImageMimeType, isWebpCandidateImageMimeType } from './util/index.js';
-import deepmerge from 'deepmerge';
-import * as domino from 'domino';
-import { default as imagemin } from 'imagemin';
-import imageminAdvPng from 'imagemin-advpng';
+import { filterRedirects, DB_ERROR, WEAK_ETAG_REGEX, stripHttpFromUrl, isBitmapImageMimeType, isWebpCandidateImageMimeType, makeZimPath } from './util/index.js';
 import axios, { AxiosError } from 'axios';
-import { default as imageminPngquant } from 'imagemin-pngquant';
-import imageminGifsicle from 'imagemin-gifsicle';
-import imageminJpegoptim from 'imagemin-jpegoptim';
-import imageminWebp from 'imagemin-webp';
 import sharp from 'sharp';
-import { fileTypeFromBuffer } from 'file-type';
+import apng from 'sharp-apng';
+import { FileTypeParser } from 'file-type';
+import { detectXml } from '@file-type/xml';
 import { HttpCookieAgent, HttpsCookieAgent } from 'http-cookie-agent/http';
 import { CookieJar } from 'tough-cookie';
 import * as logger from './Logger.js';
@@ -20,24 +14,36 @@ import ApiURLDirector from './util/builders/url/api.director.js';
 import urlHelper from './util/url.helper.js';
 import { findFirstMatchingRule, renderDownloadError } from './error.manager.js';
 import RedisStore from './RedisStore.js';
-const imageminOptions = new Map();
-imageminOptions.set('default', new Map());
-imageminOptions.set('webp', new Map());
-imageminOptions.get('default').set('image/png', {
-    plugins: [imageminPngquant({ speed: 3, strip: true, dithering: 0 }), imageminAdvPng({ optimizationLevel: 4, iterations: 5 })],
-});
-imageminOptions.get('default').set('image/jpeg', {
-    plugins: [imageminJpegoptim({ max: 60, stripAll: true })],
-});
-imageminOptions.get('default').set('image/gif', {
-    plugins: [imageminGifsicle({ optimizationLevel: 3, colors: 64 })],
-});
-imageminOptions.get('webp').set('image/png', {
-    plugins: [imageminWebp({ quality: 50, method: 6 })],
-});
-imageminOptions.get('webp').set('image/jpeg', {
-    plugins: [imageminWebp({ quality: 50, method: 6 })],
-});
+import deepmerge from 'deepmerge';
+import semver from 'semver';
+// create file-type parser with add-on to detect XML documents content (typically SVG)
+// Nota: file-type is capable to detect only binary content mime-type without the custom
+// detector
+const fileTypeParser = new FileTypeParser({ customDetectors: [detectXml] });
+function isJsonContentType(contentType) {
+    return typeof contentType === 'string' && /\b(?:application\/json|[\w.-]+\+json)\b/i.test(contentType);
+}
+function isMalformedJsonResponseError(err) {
+    if (err instanceof SyntaxError) {
+        return true;
+    }
+    if (!axios.isAxiosError(err) || err.code !== AxiosError.ERR_BAD_RESPONSE) {
+        return false;
+    }
+    return err.cause instanceof SyntaxError;
+}
+function parseJsonResponse(url, status, contentType, body) {
+    if (body !== null && typeof body === 'object') {
+        return body;
+    }
+    if (typeof body !== 'string') {
+        return body;
+    }
+    if (!isJsonContentType(contentType)) {
+        throw new DownloadError('Unexpected non-JSON response while calling API', url, status, contentType, body);
+    }
+    return JSON.parse(body);
+}
 export class DownloadError extends Error {
     urlCalled;
     httpReturnCode;
@@ -67,27 +73,35 @@ class Downloader {
         }
         return Downloader.instance;
     }
-    _speed;
-    cssDependenceUrls = {};
+    _workers;
+    _pageRequestInterval;
     _webp = false;
     _requestTimeout;
     _basicRequestOptions;
     _arrayBufferRequestOptions;
     _jsonRequestOptions;
     _streamRequestOptions;
-    wikimediaMobileJsDependenciesList = [];
-    wikimediaMobileStyleDependenciesList = [];
+    trustedJs = [];
+    customCssUrls = [];
+    customJsUrls = [];
+    mathJaxSource = null;
+    mathJaxConfig = null;
+    mathJaxConfigScript = null;
+    mathJaxEntryPoint = 'MathJax.js';
+    mathJaxAllPages = false;
     uaString;
     backoffOptions;
     optimisationCacheUrl;
     s3;
     _apiUrlDirector;
     cookierJar;
-    articleUrlDirector;
-    mainPageUrlDirector;
+    pageUrlDirector;
     insecure = false;
-    get speed() {
-        return this._speed;
+    get workers() {
+        return this._workers;
+    }
+    get pageRequestInterval() {
+        return this._pageRequestInterval;
     }
     get webp() {
         return this._webp;
@@ -110,13 +124,15 @@ class Downloader {
     get apiUrlDirector() {
         return this._apiUrlDirector;
     }
-    set init({ uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions, insecure }) {
+    set init({ uaString, workers, pageRequestInterval, reqTimeout, optimisationCacheUrl, s3, webp, trustedJs = config.output.mw.js_trusted.slice(), backoffOptions, insecure, }) {
         this.reset();
         this.uaString = uaString;
-        this._speed = speed;
+        this._workers = workers;
+        this._pageRequestInterval = pageRequestInterval;
         this._requestTimeout = reqTimeout;
         this.optimisationCacheUrl = optimisationCacheUrl;
         this._webp = webp;
+        this.trustedJs = trustedJs;
         this.s3 = s3;
         this._apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href);
         this.insecure = insecure;
@@ -129,47 +145,52 @@ class Downloader {
             failAfter: 10,
             retryIf: (err) => {
                 const requestedUrl = err.urlCalled || err.config?.url || 'unknown';
+                if (isMalformedJsonResponseError(err)) {
+                    logger.info(`Retrying ${requestedUrl} due to malformed JSON response`);
+                    return true;
+                }
                 if (err instanceof AxiosError && err.code && !['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) {
-                    logger.log(`Retrying ${requestedUrl} URL due to ${err.code} error`);
+                    logger.info(`Retrying ${requestedUrl} URL due to ${err.code} error`);
                     return true; // retry all connection issues
                 }
-                if (err.responseData?.error?.code == 'maxlag') {
-                    logger.log(`Mediawiki server is lagging ${err.responseData?.error?.lag}s; retrying in few seconds`);
+                if (err.responseData?.error?.code === 'maxlag') {
+                    logger.info(`Mediawiki server is lagging ${err.responseData?.error?.lag}s; retrying in few seconds`);
                     return true; // note that we do not honor Retry-After header value because it is not possible in current code architecture
                 }
                 const httpReturnCode = err.response?.status || err.httpReturnCode;
                 if ([429, 500, 502, 503, 504, 524].includes(httpReturnCode)) {
-                    logger.log(`Retrying ${requestedUrl} URL due to HTTP ${httpReturnCode} error`);
+                    logger.info(`Retrying ${requestedUrl} URL due to HTTP ${httpReturnCode} error`);
                     return true; // retry these HTTP status codes
                 }
                 if (err.responseData?.error?.code &&
                     ![
                         'missingtitle',
                         'readapidenied',
+                        'permissiondenied',
                         'internal_api_error_MediaWiki\\Revision\\BadRevisionException',
                         'internal_api_error_Wikimedia\\Assert\\UnreachableException',
                         'internal_api_error_Wikimedia\\Assert\\InvariantException',
                         'internal_api_error_Wikimedia\\Parsoid\\Core\\ResourceLimitExceededException',
                     ].includes(err.responseData?.error?.code)) {
-                    logger.log(`Retrying ${requestedUrl} URL due to ${err.responseData?.error?.code} Mediawiki error`);
+                    logger.info(`Retrying ${requestedUrl} URL due to ${err.responseData?.error?.code} Mediawiki error`);
                     return true; // retry these Mediawiki codes which are known to be transient
                 }
                 return false; // don't retry other errors
             },
             backoffHandler: (number, delay) => {
-                logger.info(`[backoff] #${number} after ${delay} ms`);
+                logger.debug(`[backoff] #${number} after ${delay} ms`);
             },
             ...backoffOptions,
         };
         this._basicRequestOptions = {
             // HTTP agent pools with 'keepAlive' to reuse TCP connections, so it's faster
-            // Set cookie jar and use special Http(s)CookieAgent so that cookies are automatically intercepted and persited across calls
+            // Set cookie jar and use special Http(s)CookieAgent so that cookies are automatically intercepted and persisted across calls
             httpAgent: new HttpCookieAgent({ cookies: { jar: this.cookierJar }, keepAlive: true }),
             httpsAgent: new HttpsCookieAgent({ cookies: { jar: this.cookierJar }, keepAlive: true, rejectUnauthorized: !this.insecure }), // rejectUnauthorized: false disables TLS
             timeout: this.requestTimeout,
             headers: {
                 // Use the base domain of the wiki being scraped as the Referer header, so that we can
-                // successfully scrap WMF map tiles.
+                // successfully scrape WMF map tiles.
                 Referer: MediaWiki.baseUrl.href,
                 'cache-control': 'public, max-stale=86400',
                 'user-agent': this.uaString,
@@ -190,7 +211,7 @@ class Downloader {
                 accept: 'application/json',
                 'accept-encoding': 'gzip, deflate',
             },
-            responseType: 'json',
+            responseType: 'text',
             method: 'GET',
         };
         this._streamRequestOptions = {
@@ -206,10 +227,12 @@ class Downloader {
     }
     reset() {
         this.uaString = undefined;
-        this._speed = undefined;
+        this._workers = undefined;
+        this._pageRequestInterval = undefined;
         this._requestTimeout = undefined;
         this.optimisationCacheUrl = undefined;
         this._webp = false;
+        this.trustedJs = [];
         this.s3 = undefined;
         this._apiUrlDirector = undefined;
         this.insecure = false;
@@ -218,22 +241,33 @@ class Downloader {
         this._arrayBufferRequestOptions = undefined;
         this._jsonRequestOptions = undefined;
         this._streamRequestOptions = undefined;
-        this.cssDependenceUrls = {};
-        this.wikimediaMobileJsDependenciesList = [];
-        this.wikimediaMobileStyleDependenciesList = [];
-        this.articleUrlDirector = undefined;
-        this.mainPageUrlDirector = undefined;
+        this.pageUrlDirector = undefined;
+    }
+    /**
+     * Destroys HTTP agents to properly close all connections.
+     * This is required for tests to prevent "Jest environment has been torn down" errors.
+     * Safe to call multiple times (idempotent).
+     */
+    async destroy() {
+        try {
+            if (this._basicRequestOptions?.httpAgent && typeof this._basicRequestOptions.httpAgent.destroy === 'function') {
+                this._basicRequestOptions.httpAgent.destroy();
+            }
+        }
+        catch {
+            // Ignore errors from destroying agents
+        }
+        try {
+            if (this._basicRequestOptions?.httpsAgent && typeof this._basicRequestOptions.httpsAgent.destroy === 'function') {
+                this._basicRequestOptions.httpsAgent.destroy();
+            }
+        }
+        catch {
+            // Ignore errors from destroying agents
+        }
     }
     getUrlDirector(renderer) {
         switch (renderer.constructor.name) {
-            case 'WikimediaDesktopRenderer':
-                return MediaWiki.wikimediaDesktopUrlDirector;
-            case 'VisualEditorRenderer':
-                return MediaWiki.visualEditorUrlDirector;
-            case 'WikimediaMobileRenderer':
-                return MediaWiki.wikimediaMobileUrlDirector;
-            case 'RestApiRenderer':
-                return MediaWiki.restApiUrlDirector;
             case 'ActionParseRenderer':
                 return MediaWiki.actionParseUrlDirector;
             /* istanbul ignore next */
@@ -241,15 +275,11 @@ class Downloader {
                 throw new Error(`Unknown renderer ${renderer.constructor.name}`);
         }
     }
-    setUrlsDirectors(mainPageRenderer, articlesRenderer) {
-        this.articleUrlDirector = this.getUrlDirector(articlesRenderer);
-        this.mainPageUrlDirector = this.getUrlDirector(mainPageRenderer);
+    setUrlsDirectors(pagesRenderer) {
+        this.pageUrlDirector = this.getUrlDirector(pagesRenderer);
     }
-    getArticleUrl(articleId, articleUrlOpts = {}) {
-        return this.articleUrlDirector.buildArticleURL(articleId, articleUrlOpts);
-    }
-    getMainPageUrl(articleId) {
-        return this.mainPageUrlDirector.buildArticleURL(articleId);
+    getPageUrl(pageTitle, pageUrlOpts = {}) {
+        return this.pageUrlDirector.buildPageUrl(pageTitle, pageUrlOpts);
     }
     removeEtagWeakPrefix(etag) {
         return etag && etag.replace(WEAK_ETAG_REGEX, '');
@@ -257,13 +287,14 @@ class Downloader {
     querySiteInfo() {
         return this.getJSON(this.apiUrlDirector.buildSiteInfoURL());
     }
-    async getArticleDetailsIds(articleIds, shouldGetThumbnail = false) {
+    async getPagesByTitle(pageTitles, shouldGetThumbnail = false, followRedirects = true) {
         let continuation;
-        let finalProcessedResp;
+        let allPages = {};
+        const visitedUrls = new Set();
         while (true) {
             const queryOpts = {
-                ...(await this.getArticleQueryOpts(shouldGetThumbnail, true)),
-                titles: articleIds.join('|'),
+                ...(await this.getPageQueryOpts(shouldGetThumbnail, followRedirects)),
+                titles: pageTitles.join('|'),
                 ...((await MediaWiki.hasCoordinates()) ? { colimit: 'max' } : {}),
                 ...(MediaWiki.getCategories
                     ? {
@@ -274,43 +305,50 @@ class Downloader {
                 ...continuation,
             };
             const reqUrl = this.apiUrlDirector.buildQueryURL(queryOpts);
+            if (visitedUrls.has(reqUrl)) {
+                throw new Error(`Detected continuation cycle while fetching page details by IDs. ` + `visitedUrls=[\n${[...visitedUrls].join('\n')}\n]`);
+            }
+            visitedUrls.add(reqUrl);
             const resp = await this.getJSON(reqUrl);
             Downloader.handleMWWarningsAndErrors(resp);
-            let processedResponse = resp.query?.pages ? normalizeMwResponse(resp.query) : {};
-            if (resp.continue) {
-                continuation = resp.continue;
-                finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse);
-            }
-            else {
-                if (MediaWiki.getCategories) {
-                    processedResponse = await this.setArticleSubCategories(processedResponse);
-                }
-                finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse);
+            const pages = Object.values(resp.query?.pages);
+            pages.forEach((page) => filterRedirects(page));
+            /* deepmerge because a single page data might be split over multiple API results pages */
+            allPages = deepmerge(allPages, pages.reduce((acc, page) => {
+                acc[page.title] = page;
+                return acc;
+            }, {}));
+            continuation = resp.continue;
+            if (!continuation) {
                 break;
             }
         }
-        return finalProcessedResp;
+        return Object.values(allPages);
     }
-    async getArticleDetailsNS(ns, gapcontinue = '') {
-        let queryContinuation;
-        let finalProcessedResp;
-        let gCont = null;
+    async getPagesByNamespace(namespace, requestGapcontinue = '') {
+        let queryContinuation = null;
+        let allPages = {};
+        const visitedUrls = new Set();
         while (true) {
             const queryOpts = {
-                ...(await this.getArticleQueryOpts()),
+                ...(await this.getPageQueryOpts()),
                 ...((await MediaWiki.hasCoordinates()) ? { colimit: 'max' } : {}),
                 ...(MediaWiki.getCategories
-                    ? {
-                        cllimit: 'max',
-                        clshow: '!hidden',
-                    }
+                    ? semver.satisfies(MediaWiki.metaData.versionMw, '>=1.46')
+                        ? {
+                            cllimit: 'max',
+                        }
+                        : {
+                            cllimit: 'max',
+                            clshow: '!hidden',
+                        }
                     : {}),
                 rawcontinue: 'true',
                 generator: 'allpages',
                 gapfilterredir: 'nonredirects',
                 gaplimit: 'max',
-                gapnamespace: String(ns),
-                gapcontinue,
+                gapnamespace: String(namespace),
+                gapcontinue: requestGapcontinue,
             };
             if (queryContinuation) {
                 queryOpts.cocontinue = queryContinuation?.coordinates?.cocontinue ?? queryOpts.cocontinue;
@@ -319,59 +357,77 @@ class Downloader {
                 queryOpts.rdcontinue = queryContinuation?.redirects?.rdcontinue ?? queryOpts.rdcontinue;
             }
             const reqUrl = this.apiUrlDirector.buildQueryURL(queryOpts);
+            if (visitedUrls.has(reqUrl)) {
+                throw new Error(`Detected continuation cycle while fetching pages details in namespace ${namespace}. ` + `visitedUrls=[\n${[...visitedUrls].join('\n')}\n]`);
+            }
+            visitedUrls.add(reqUrl);
             const resp = await this.getJSON(reqUrl);
             Downloader.handleMWWarningsAndErrors(resp);
-            let processedResponse = normalizeMwResponse(resp.query);
-            gCont = resp['query-continue']?.allpages?.gapcontinue ?? gCont;
-            const queryComplete = Object.keys(resp['query-continue'] || {}).filter((key) => key !== 'allpages').length === 0;
-            if (!queryComplete) {
-                queryContinuation = resp['query-continue'];
-                finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse);
-            }
-            else {
-                if (MediaWiki.getCategories) {
-                    processedResponse = await this.setArticleSubCategories(processedResponse);
-                }
-                finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse);
+            const pages = Object.values(resp.query?.pages || {});
+            pages.forEach((page) => filterRedirects(page));
+            /* deepmerge because a single page data might be split over multiple API results pages */
+            allPages = deepmerge(allPages, pages.reduce((acc, page) => {
+                acc[page.title] = page;
+                return acc;
+            }, {}));
+            queryContinuation = resp['query-continue'];
+            const queryComplete = Object.keys(queryContinuation || {}).filter((key) => key !== 'allpages').length === 0;
+            if (queryComplete) {
                 break;
             }
         }
         return {
-            articleDetails: finalProcessedResp,
-            gapContinue: gCont,
+            pages: Object.values(allPages),
+            gapContinue: queryContinuation?.allpages?.gapcontinue || null,
         };
     }
-    async getLogEvents(letype, articleId) {
-        const logEventsData = await this.getJSON(this.apiUrlDirector.buildLogEventsQuery(letype, articleId));
+    async getLogEvents(letype, pageTitle) {
+        const logEventsData = await this.getJSON(this.apiUrlDirector.buildLogEventsQuery(letype, pageTitle));
         return logEventsData.query?.logevents;
     }
-    async getArticle(articleId, articleDetailXId, articleRenderer, articleUrl, dump, articleDetail) {
-        logger.info(`Getting article [${articleId}] from ${articleUrl}`);
+    async getPage(pageTitle, pageRenderer, pageUrl, dump, pageDetail) {
+        logger.debug(`Getting page [${pageTitle}] from ${pageUrl}`);
         try {
-            const { data, moduleDependencies, redirects, displayTitle, articleSubtitle, bodyCssClass, htmlCssClass } = await articleRenderer.download({
-                articleId,
-                articleUrl,
-                articleDetail,
+            const { data, moduleDependencies, redirects, displayTitle, subtitle, categoriesHtml = '', bodyCssClass, htmlCssClass, } = await pageRenderer.download({
+                pageTitle,
+                pageUrl,
+                pageDetail,
+                langVar: dump.langVar,
             });
             // Cope with the fact that the page we are fetching might have been moved and replaced by a redirect
             // In such a case, the download above is expected to follow the redirect so that we have proper original
-            // content at original article path, but we probably need to add a redirect since new article location
-            // probably did not existed when listing articles (might have existed if the move occured during article
+            // content at original page path, but we probably need to add a redirect since new page location
+            // probably did not existed when listing pages (might have existed if the move occured during page
             // listing). The redirect we add in hence in the "opposite" direction than usual, i.e. it will redirect
             // from new location to original location. Note that only ActionParse API gives proper redirects info.
             for (const redirect of redirects) {
-                if (!(await RedisStore.articleDetailXId.exists(redirect.to)) && !(await RedisStore.redirectsXId.exists(redirect.to))) {
-                    RedisStore.redirectsXId.set(redirect.to, { targetId: redirect.from, title: redirect.to, fragment: '' });
+                if (!(await RedisStore.pagesStore.exists(redirect.to)) && !(await RedisStore.redirectsStore.exists(redirect.to))) {
+                    await RedisStore.redirectsStore.set(redirect.to, { to: redirect.from, from: redirect.to, fragment: '' });
+                }
+            }
+            const categoryMembers = {
+                categoryinfo: pageDetail.categoryinfo,
+                subcats: [],
+                pages: [],
+                files: [],
+            };
+            if (pageDetail.categoryinfo?.size) {
+                await this.getCategoryMembers(pageTitle, categoryMembers);
+                if (MediaWiki.getCategories) {
+                    categoryMembers.categoryinfo.subcats = categoryMembers.subcats.length;
+                    categoryMembers.categoryinfo.pages = categoryMembers.pages.length;
+                    categoryMembers.categoryinfo.files = categoryMembers.files.length;
                 }
             }
-            return await articleRenderer.render({
+            return await pageRenderer.render({
                 data,
                 moduleDependencies,
-                articleId,
-                articleDetailXId,
-                articleDetail,
+                pageTitle,
+                pageDetail,
                 displayTitle,
-                articleSubtitle,
+                subtitle,
+                categoryMembers,
+                categoriesHtml,
                 bodyCssClass,
                 htmlCssClass,
                 dump,
@@ -400,7 +456,7 @@ class Downloader {
             if (!downloadErrorContext) {
                 throw err;
             }
-            logger.warn(`Article ${articleId} failed to download from '${downloadErrorContext.urlCalled}' with ` +
+            logger.warn(`Page ${pageTitle} failed to download from '${downloadErrorContext.urlCalled}' with ` +
                 `'${downloadErrorContext.errorCode}' error code, ` +
                 `'${downloadErrorContext.httpReturnCode}' HTTP return code ` +
                 `and '${downloadErrorContext.responseContentType}' content-type ` +
@@ -411,43 +467,45 @@ class Downloader {
                 throw err;
             }
             if (errorRule.isHardFailure) {
-                logger.log(`This is a hard ${errorRule.detailsMessageKey} error which will be replaced by a placeholder`);
-                dump.status.articles.hardFail += 1;
-                dump.status.articles.hardFailedArticleIds.push(articleId);
-                if (dump.maxHardFailedArticles > 0 && dump.status.articles.hardFail > dump.maxHardFailedArticles) {
-                    throw new Error('Too many articles failed to download');
+                logger.info(`This is a hard ${errorRule.detailsMessageKey} error which will be replaced by a placeholder`);
+                dump.status.pages.hardFail += 1;
+                dump.status.pages.hardFailedPages.push(pageTitle);
+                if (dump.maxHardFailedPages > 0 && dump.status.pages.hardFail > dump.maxHardFailedPages) {
+                    throw new Error('Too many pages failed to download'); // eslint-disable-line preserve-caught-error
                 }
             }
             else {
-                logger.log(`This is a soft ${errorRule.detailsMessageKey} error which will be replaced by a placeholder`);
-                dump.status.articles.softFail += 1;
-                dump.status.articles.softFailedArticleIds.push(articleId);
+                logger.info(`This is a soft ${errorRule.detailsMessageKey} error which will be replaced by a placeholder`);
+                dump.status.pages.softFail += 1;
+                dump.status.pages.softFailedPages.push(pageTitle);
             }
-            RedisStore.articleDetailXId.delete(articleId); // Remove article from list so that we stop creating links to this placeholder
-            const articleTitle = articleId.replace(/_/g, ' ');
-            const errorPlaceholderHtml = renderDownloadError(errorRule, dump, articleId, articleTitle);
-            return [
-                {
-                    articleId,
-                    displayTitle: articleTitle,
-                    html: errorPlaceholderHtml,
-                    imageDependencies: [],
-                    videoDependencies: [],
-                    mediaDependencies: [],
-                    moduleDependencies: [],
-                    staticFiles: config.output.downloadErrorResources,
-                    subtitles: [],
-                },
-            ];
+            await RedisStore.pagesStore.delete(pageTitle); // Remove page from list so that we stop creating links to this placeholder
+            const errorPlaceholderHtml = renderDownloadError(errorRule, dump, pageTitle);
+            return {
+                items: [
+                    {
+                        pageTitle,
+                        zimPath: makeZimPath(pageTitle),
+                        zimTitle: '', // we do not want these failed downloads to end-up in suggestion search
+                        htmlContent: errorPlaceholderHtml,
+                    },
+                ],
+                mediaDependencies: [],
+                imageDependencies: [],
+                videoDependencies: [],
+                moduleDependencies: [],
+                subtitles: [],
+                needsDownloadErrorStaticFiles: true,
+            };
         }
     }
     async getJSON(_url) {
         const url = urlHelper.deserializeUrl(_url);
         return new Promise((resolve, reject) => {
-            this.backoffCall(this.getJSONCb, url, 'json', (err, val) => {
+            this.backoffCall(this.getJSONCb, url, 'json', undefined, (err, val) => {
                 if (err) {
                     const httpStatus = (err.response && err.response.status) || err.httpReturnCode;
-                    logger.info(`Failed to get [${url}] [status=${httpStatus}]`);
+                    logger.debug(`Failed to get [${url}] [status=${httpStatus}]`);
                     reject(err);
                 }
                 else {
@@ -457,6 +515,10 @@ class Downloader {
         });
     }
     async request(config) {
+        // axios@1.20's typings resolve the `request` return type through a conditional type keyed on
+        // an internal sentinel default; since our own `R` generic is not that sentinel but TS cannot
+        // prove it at this level, the conditional type doesn't reduce and the (correct) result needs
+        // an explicit cast back to `R`.
         return axios.request({
             ...this._basicRequestOptions,
             ...config,
@@ -473,7 +535,7 @@ class Downloader {
     async post(url, data, config) {
         return this.request({ url, data, method: 'POST', ...config });
     }
-    async downloadContent(_url, kind, retry = true) {
+    async downloadContent(_url, kind, retry = true, requestedWidth) {
         if (!_url) {
             throw new Error(`Parameter [${_url}] is not a valid url`);
         }
@@ -492,15 +554,16 @@ class Downloader {
                     }
                 };
                 if (retry) {
-                    this.backoffCall(this.getContentCb, url, kind, cb);
+                    this.backoffCall(this.getContentCb, url, kind, requestedWidth, cb);
                 }
                 else {
-                    this.getContentCb(url, kind, cb);
+                    this.getContentCb(url, kind, requestedWidth, cb);
                 }
             });
         }
         catch (err) {
-            const httpStatus = err.response && err.response.status;
+            const e = err;
+            const httpStatus = e.response && e.response.status;
             logger.warn(`Failed to get [${url}] [status=${httpStatus}]`);
             throw err;
         }
@@ -520,10 +583,10 @@ class Downloader {
         if (resp.error?.code === DB_ERROR)
             throw new Error(`Got error from MW Query ${JSON.stringify(resp.error, null, '\t')}`);
         if (resp.error)
-            logger.log(`Got error from MW Query ${JSON.stringify(resp.warnings, null, '\t')}`);
+            logger.info(`Got error from MW Query ${JSON.stringify(resp.warnings, null, '\t')}`);
     }
-    async getArticleQueryOpts(includePageimages = false, followRedirects = false) {
-        const prop = `${includePageimages ? '|pageimages' : ''}${(await MediaWiki.hasCoordinates()) ? '|coordinates' : ''}${MediaWiki.getCategories ? '|categories' : ''}`;
+    async getPageQueryOpts(includePageimages = false, followRedirects = false) {
+        const prop = `${includePageimages ? '|pageimages' : ''}${(await MediaWiki.hasCoordinates()) ? '|coordinates' : ''}${MediaWiki.getCategories ? '|categories' : ''}${(await MediaWiki.hasFlaggedRevs()) ? '|flagged' : ''}`;
         return {
             ...MediaWiki.queryOpts,
             prop: MediaWiki.queryOpts.prop.concat(prop),
@@ -531,89 +594,112 @@ class Downloader {
             redirects: followRedirects ? true : undefined,
         };
     }
-    async setArticleSubCategories(articleDetails) {
-        logger.info('Getting subCategories');
-        for (const [articleId, articleDetail] of Object.entries(articleDetails)) {
-            const isCategoryArticle = articleDetail.ns === 14;
-            if (isCategoryArticle) {
-                const categoryMembers = await this.getSubCategories(articleId);
-                articleDetails[articleId].subCategories = categoryMembers.slice();
-            }
-        }
-        return articleDetails;
-    }
-    getJSONCb = (url, kind, handler) => {
-        logger.info(`Getting JSON from [${url}]`);
+    getJSONCb = (url, kind, _requestedWidth, handler) => {
+        logger.debug(`Getting JSON from [${url}]`);
         this.request({ url, method: 'GET', ...this.jsonRequestOptions })
             .then((val) => {
-            if (val.data.error) {
-                handler(new DownloadError(`Error returned while calling API`, url, val.status, val.headers['content-type'].toString(), val.data));
+            const contentType = val.headers['content-type']?.toString() || null;
+            const data = parseJsonResponse(url, val.status, contentType, val.data);
+            if (data.error) {
+                handler(new DownloadError(`Error returned while calling API`, url, val.status, contentType, data));
             }
             else {
-                handler(null, val.data);
+                handler(null, data);
             }
         })
-            .catch((err) => handler(err));
+            .catch((err) => {
+            if (isMalformedJsonResponseError(err)) {
+                logger.warn(`Malformed JSON from [${url}] → ${err.message}`);
+            }
+            handler(err);
+        });
     };
     async getImageMimeType(data) {
-        const fileType = await fileTypeFromBuffer(data);
-        if (fileType && fileType.mime === 'application/xml') {
-            // File type is known to be wrong, might be SVG
-            return null;
+        // get mime-type of a binary file, with XML addon to detect SVGs
+        return (await fileTypeParser.fromBuffer(data)).mime;
+    }
+    async getSharpObject(input, contentType) {
+        if (contentType == 'image/apng') {
+            try {
+                // resolveWithObject is not passed, so sharpFromApng always resolves to a Sharp instance, never ImageData
+                return (await apng.sharpFromApng(input.data));
+            }
+            catch {
+                logger.warn(`Failed to read animated PNG at ${input.context}: static image will be used instead of animation`);
+                return sharp(input.data, { animated: true });
+            }
+        }
+        else {
+            // { animated: true } reads all frames of an animated pictures; it is a no-op for static images
+            return sharp(input.data, { animated: true });
         }
-        return fileType ? fileType.mime : null;
     }
-    async getCompressedBody(input) {
+    async compressDefault(sharpObject, context, contentType) {
+        if (contentType == 'image/png') {
+            return sharpObject.png({ palette: true, quality: 60, effort: 7 }).toBuffer();
+        }
+        else if (contentType == 'image/apng') {
+            logger.warn(`Cannot compress image at ${context}: recompressing APNG to APNG is not supported ATM, using original data`);
+            return sharpObject.toBuffer();
+        }
+        else if (contentType === 'image/jpeg') {
+            return sharpObject.jpeg({ quality: 60, mozjpeg: true }).toBuffer();
+        }
+        else if (contentType === 'image/gif') {
+            return sharpObject.gif({ colours: 64, effort: 7 }).toBuffer();
+        }
+        else {
+            logger.warn(`Cannot compress image at ${context}: unknown content-type ${contentType}, using original data`);
+            return sharpObject.toBuffer();
+        }
+    }
+    async getCompressedBody(input, requestedWidth) {
         const contentType = await this.getImageMimeType(input.data);
         if (isBitmapImageMimeType(contentType)) {
+            // Resize down with sharp before compression when the source is wider than the requested
+            // display width. { animated: true } reads all frames so animated GIFs/APNGs are resized
+            // frame-by-frame and stay animated, instead of only keeping their first frame.
+            let sharpData = await this.getSharpObject(input, contentType);
+            if (requestedWidth) {
+                try {
+                    const metadata = await sharpData.metadata();
+                    if (metadata.width && metadata.width > requestedWidth) {
+                        sharpData = sharpData.resize({ width: requestedWidth, withoutEnlargement: true });
+                    }
+                }
+                catch (err) {
+                    logger.warn(`Failed to resize image from ${input.context} to ${requestedWidth}px, proceeding without resize: ${err.message}`);
+                }
+            }
             if (this.webp && isWebpCandidateImageMimeType(contentType)) {
-                return {
-                    data: await imagemin
-                        .buffer(input.data, imageminOptions.get('webp').get(contentType))
-                        .catch(async (err) => {
-                        if (/Unsupported color conversion request/.test(err.stderr)) {
-                            return imagemin
-                                .buffer(await sharp(input.data).toColorspace('srgb').toBuffer(), imageminOptions.get('webp').get(contentType))
-                                .catch(() => {
-                                return input.data;
-                            })
-                                .then((data) => {
-                                return data;
-                            });
-                        }
-                        else {
-                            return imagemin.buffer(input.data, imageminOptions.get('default').get(contentType)).catch(() => {
-                                return input.data;
-                            });
-                        }
-                    })
-                        .then((data) => {
-                        return data;
-                    }),
-                };
+                try {
+                    return await sharpData.webp({ quality: 50, effort: 6 }).toBuffer();
+                }
+                catch {
+                    logger.warn(`Failed to compress ${input.context} to WEBP as requested, falling back to simple recompression of original format`);
+                    return await this.compressDefault(sharpData, input.context, contentType);
+                }
             }
             else {
-                return {
-                    data: await imagemin.buffer(input.data, imageminOptions.get('default').get(contentType)).catch(() => {
-                        return input.data;
-                    }),
-                };
+                return await this.compressDefault(sharpData, input.context, contentType);
             }
         }
-        return {
-            data: input.data,
-        };
+        return input.data;
     }
-    getContentCb = async (url, kind, handler) => {
-        logger.info(`Downloading [${url}]`);
+    getContentCb = async (url, kind, requestedWidth, handler) => {
+        logger.debug(`Downloading [${url}]`);
         try {
             if (this.optimisationCacheUrl && kind === 'image') {
-                this.downloadImage(url, handler);
+                this.downloadImage(url, handler, requestedWidth);
             }
             else {
                 const resp = await this.request({ url, method: 'GET', ...this.arrayBufferRequestOptions });
                 // If content is an image, we might benefit from compressing it
-                const content = kind === 'image' ? (await this.getCompressedBody({ data: resp.data })).data : resp.data;
+                const content = kind === 'image' ? await this.getCompressedBody({ data: resp.data, context: url }, requestedWidth) : resp.data;
+                // Check content really exists
+                if (!content?.length) {
+                    logger.warn(`Content for ${url} is missing or empty (${typeof content})`);
+                }
                 // compute content-type from content, since getCompressedBody might have modified it
                 const contentType = kind === 'image' ? (await this.getImageMimeType(content)) || resp.headers['content-type'] : resp.headers['content-type'];
                 handler(null, {
@@ -631,62 +717,102 @@ class Downloader {
             }
         }
     };
-    async downloadImage(url, handler) {
+    async downloadImage(url, handler, requestedWidth) {
+        // Build a width-aware cache version token so the same URL at different
+        // requested widths does not reuse an undersized cached image.
+        const cacheVersion = this.webp ? (requestedWidth ? `webp-w${requestedWidth}` : 'webp') : requestedWidth ? `1-w${requestedWidth}` : '1';
         try {
             this.s3
                 // Check first if we have an entry in the (object storage) cache for this URL
-                .downloadBlob(stripHttpFromUrl(url), this.webp ? 'webp' : '1')
+                .downloadBlob(stripHttpFromUrl(url), cacheVersion)
                 // Handle the cache response and act accordingly
                 .then(async (s3Resp) => {
-                // 'Versioning' of image is made via HTTP ETag. We should
-                // check if we have the proper version by requesting proper
-                // ETag from upstream MediaWiki.
-                if (s3Resp?.Metadata?.etag) {
-                    this.arrayBufferRequestOptions.headers['If-None-Match'] = this.removeEtagWeakPrefix(s3Resp.Metadata.etag);
-                }
-                // Use the base domain of the wiki being scraped as the Referer header, so that we can
-                // successfully scrap WMF map tiles.
-                const mwResp = await this.request({ url, method: 'GET', ...this.arrayBufferRequestOptions });
-                // Most of the images, after having been uploaded once to the
-                // cache, will always have 304 status, until modified. If cache
-                // is up to date, return cached image. We always have an s3
-                // response when mwResp is 304, since this can only happen
-                // when we have an eTag coming from s3.
-                if (mwResp.status === 304) {
-                    // Proceed with image
-                    const data = (await this.streamToBuffer(s3Resp.Body));
-                    const contentType = await this.getImageMimeType(data);
-                    logger.info(`Using S3-cached image for ${url} (contentType: ${contentType})`);
-                    handler(null, {
-                        contentType,
-                        content: data,
-                    });
-                    return;
-                }
-                // Destroy the Readable so that socket is freed and returned to the pool
-                if (s3Resp?.Body) {
-                    s3Resp.Body.destroy();
-                }
-                // Compress content because image blob comes from upstream MediaWiki
-                const compressedData = (await this.getCompressedBody({ data: mwResp.data })).data;
-                // Check for the ETag and upload to cache
-                const etag = this.removeEtagWeakPrefix(mwResp.headers.etag);
-                if (etag) {
-                    await this.s3.uploadBlob(stripHttpFromUrl(url), compressedData, etag, this.webp ? 'webp' : '1');
-                }
-                // get contentType from image, with fallback to response headers should the image be unsupported at all (e.g. SVG)
-                const contentType = (await this.getImageMimeType(compressedData)) || mwResp.headers['content-type'];
-                if (s3Resp) {
-                    logger.info(`Using image downloaded from upstream for ${url} (S3-cached image is outdated, contentType: ${contentType})`);
-                }
-                else {
-                    logger.info(`Using image downloaded from upstream for ${url} (no S3-cached image found, contentType: ${contentType})`);
-                }
-                // Proceed with image
-                handler(null, {
-                    contentType,
-                    content: compressedData,
-                });
+                // While we might still need s3Resp.Body (below, or in the 304 branch), forward any
+                // stream error (e.g. ECONNRESET) into this handler's rejection path so it is retried
+                // like any other transient download error, instead of hanging (an errored AWS SDK
+                // stream never emits the 'data'/'end' that a later reader would wait for) or crashing
+                // the process (an AWS SDK stream left without an 'error' listener throws on a late
+                // socket-close error).
+                let onBodyError;
+                const bodyError = s3Resp?.Body
+                    ? new Promise((_resolve, reject) => {
+                        onBodyError = reject;
+                        s3Resp.Body.on('error', onBodyError);
+                    })
+                    : new Promise(() => { });
+                return Promise.race([
+                    bodyError,
+                    (async () => {
+                        // 'Versioning' of image is made via HTTP ETag. We should
+                        // check if we have the proper version by requesting proper
+                        // ETag from upstream MediaWiki.
+                        // Headers are cloned per-request: concurrent downloads share this.arrayBufferRequestOptions,
+                        // so mutating its headers in place would leak one image's If-None-Match onto another's request.
+                        const requestOptions = {
+                            ...this.arrayBufferRequestOptions,
+                            headers: { ...this.arrayBufferRequestOptions.headers },
+                        };
+                        // Display message when ignoring empty S3 objects
+                        if (s3Resp && !s3Resp.ContentLength) {
+                            logger.warn(`Ignoring empty S3 object for ${url}`);
+                        }
+                        // If S3 object has content and etag, check if upstream did changed
+                        if (s3Resp?.ContentLength && s3Resp?.Metadata?.etag) {
+                            requestOptions.headers['If-None-Match'] = this.removeEtagWeakPrefix(s3Resp.Metadata.etag);
+                        }
+                        // Use the base domain of the wiki being scraped as the Referer header, so that we can
+                        // successfully scrape WMF map tiles.
+                        const mwResp = await this.request({ url, method: 'GET', ...requestOptions });
+                        // Most of the images, after having been uploaded once to the
+                        // cache, will always have 304 status, until modified. If cache
+                        // is up to date, return cached image. We always have an s3
+                        // response when mwResp is 304, since this can only happen
+                        // when we have an eTag coming from s3.
+                        if (mwResp.status === 304) {
+                            // Proceed with image
+                            const data = (await this.streamToBuffer(s3Resp.Body));
+                            const contentType = await this.getImageMimeType(data);
+                            logger.debug(`Using S3-cached image for ${url} (contentType: ${contentType})`);
+                            handler(null, {
+                                contentType,
+                                content: data,
+                            });
+                            return;
+                        }
+                        // From here we no longer need s3Resp.Body: stop treating its errors as fatal (we
+                        // already have what we need from upstream MediaWiki), then destroy the Readable so
+                        // the socket is freed and returned to the pool.
+                        if (s3Resp?.Body) {
+                            s3Resp.Body.off('error', onBodyError);
+                            s3Resp.Body.on('error', () => { });
+                            s3Resp.Body.destroy();
+                        }
+                        // Compress content because image blob comes from upstream MediaWiki
+                        const compressedData = await this.getCompressedBody({ data: mwResp.data, context: url }, requestedWidth);
+                        // Check content really exists
+                        if (!compressedData?.length) {
+                            throw new Error(`Content for ${url} is missing or empty (${typeof compressedData})`);
+                        }
+                        // Check for the ETag and upload to cache
+                        const etag = this.removeEtagWeakPrefix(mwResp.headers.etag);
+                        if (etag) {
+                            await this.s3.uploadBlob(stripHttpFromUrl(url), compressedData, etag, cacheVersion);
+                        }
+                        // get contentType from image, with fallback to response headers should the image be unsupported at all (e.g. SVG)
+                        const contentType = (await this.getImageMimeType(compressedData)) || mwResp.headers['content-type'];
+                        if (s3Resp) {
+                            logger.debug(`Using image downloaded from upstream for ${url} (S3-cached image is outdated, contentType: ${contentType})`);
+                        }
+                        else {
+                            logger.debug(`Using image downloaded from upstream for ${url} (no S3-cached image found, contentType: ${contentType})`);
+                        }
+                        // Proceed with image
+                        handler(null, {
+                            contentType,
+                            content: compressedData,
+                        });
+                    })(),
+                ]);
             })
                 .catch((err) => {
                 this.errHandler(err, url, handler);
@@ -697,79 +823,40 @@ class Downloader {
         }
     }
     errHandler(err, url, handler) {
-        logger.log(`Not able to download content for ${url} due to ${err}`);
+        logger.debug(`Error while downloading content for ${url} due to ${err} ; might be retried`);
         handler(err);
     }
-    async getSubCategories(articleId, continueStr = '') {
+    async getCategoryMembers(pageTitle, categoryMembers) {
         const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href);
-        const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr));
-        const items = query.categorymembers.filter((a) => a && a.title);
-        if (cont && cont.cmcontinue) {
-            const nextItems = await this.getSubCategories(articleId, cont.cmcontinue);
-            return items.concat(nextItems);
-        }
-        else {
-            return items;
+        let continueStr = '';
+        while (true) {
+            const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildCategoryMembersURL(pageTitle, continueStr));
+            const items = query.categorymembers.filter((a) => {
+                const sortkey = a.sortkeyprefix + ((a.ns && a.title.split(':').slice(1).join(':')) || a.title);
+                a.sortkeyprefix = [...sortkey][0];
+                return a && a.title;
+            });
+            const pagesInZim = MediaWiki.getCategories ? await RedisStore.pagesStore.existsMany(items.map((a) => a.title)) : null;
+            categoryMembers.subcats.push(...items.filter((a) => a.type === 'subcat' && (pagesInZim ? pagesInZim[a.title] : true)));
+            categoryMembers.pages.push(...items.filter((a) => a.type === 'page' && (pagesInZim ? pagesInZim[a.title] : true)));
+            categoryMembers.files.push(...items.filter((a) => a.type === 'file' && (pagesInZim ? pagesInZim[a.title] : true)));
+            if (cont && cont.cmcontinue) {
+                continueStr = cont.cmcontinue;
+            }
+            else {
+                break;
+            }
         }
     }
-    backoffCall(handler, url, kind, callback) {
+    backoffCall(handler, url, kind, requestedWidth, callback) {
         this.backoffOptions.strategy.reset(); // reset delay to initial one at each call
-        const call = backoff.call(handler, url, kind, callback);
+        const call = backoff.call(handler, url, kind, requestedWidth, callback);
         call.setStrategy(this.backoffOptions.strategy);
         call.retryIf(this.backoffOptions.retryIf);
         call.failAfter(this.backoffOptions.failAfter);
         call.on('backoff', this.backoffOptions.backoffHandler);
         call.start();
     }
-    async getModuleDependencies(title) {
-        const genericJsModules = config.output.mw.js;
-        const genericCssModules = config.output.mw.css;
-        const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href);
-        const articleApiUrl = apiUrlDirector.buildArticleApiURL(title);
-        const articleData = await this.getJSON(articleApiUrl);
-        if (articleData.error) {
-            const errorMessage = `Unable to retrieve js/css dependencies for article '${title}': ${articleData.error.code}`;
-            logger.error(errorMessage);
-            /* If article is missing (for example because it just has been deleted) */
-            if (articleData.error.code === 'missingtitle') {
-                return { jsConfigVars: '', jsDependenciesList: [], styleDependenciesList: [] };
-            }
-            /* Something went wrong in modules retrieval at app level (no HTTP error) */
-            throw new Error(errorMessage);
-        }
-        const { parse: { modules, modulescripts, modulestyles, headhtml }, } = articleData;
-        const jsDependenciesList = genericJsModules.concat(modules, modulescripts).filter((a) => a);
-        const styleDependenciesList = []
-            .concat(modules, modulestyles, genericCssModules)
-            .filter((a) => a)
-            .filter((oneStyleDep) => !contains(config.filters.blackListCssModules, oneStyleDep));
-        logger.info(`Js dependencies of ${title} : ${jsDependenciesList}`);
-        logger.info(`Css dependencies of ${title} : ${styleDependenciesList}`);
-        const jsConfigVars = Downloader.extractJsConfigVars(headhtml);
-        // Download mobile page dependencies only once
-        if ((await MediaWiki.hasWikimediaMobileApi()) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) {
-            try {
-                // TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources
-                const mobileModulesData = await this.getJSON(`${MediaWiki.mobileModulePath}Test`);
-                mobileModulesData.forEach((module) => {
-                    if (module.includes('javascript')) {
-                        this.wikimediaMobileJsDependenciesList.push(module);
-                    }
-                    else if (module.includes('css')) {
-                        this.wikimediaMobileStyleDependenciesList.push(module);
-                    }
-                });
-            }
-            catch (err) {
-                throw new Error(`Error getting mobile modules ${err.message}`);
-            }
-        }
-        return {
-            jsConfigVars,
-            jsDependenciesList: jsDependenciesList.concat(this.wikimediaMobileJsDependenciesList),
-            styleDependenciesList: styleDependenciesList.concat(this.wikimediaMobileStyleDependenciesList),
-        };
-    }
     // Solution to handle aws js sdk v3 from https://github.com/aws/aws-sdk-js-v3/issues/1877
     async streamToBuffer(stream) {
         return new Promise((resolve, reject) => {
@@ -779,24 +866,6 @@ class Downloader {
             stream.on('end', () => resolve(Buffer.concat(chunks)));
         });
     }
-    static extractJsConfigVars(headhtml) {
-        let jsConfigVars = '';
-        // Saving, as a js module, the jsconfigvars that are set in the header of a wikipedia page
-        // the script below extracts the config with a regex executed on the page header returned from the api
-        const scriptTags = domino.createDocument(`${headhtml}</body></html>`).getElementsByTagName('script');
-        const regex = /mw\.config\.set\(\{.*?\}\);/gm;
-        for (let i = 0; i < scriptTags.length; i += 1) {
-            if (scriptTags[i].text.includes('mw.config.set')) {
-                jsConfigVars = regex.exec(scriptTags[i].text)[0] || '';
-                jsConfigVars = `(window.RLQ=window.RLQ||[]).push(function() {${jsConfigVars}});`;
-            }
-            else if (scriptTags[i].text.includes('RLCONF') || scriptTags[i].text.includes('RLSTATE') || scriptTags[i].text.includes('RLPAGEMODULES')) {
-                jsConfigVars = scriptTags[i].text;
-            }
-        }
-        jsConfigVars = jsConfigVars.replace('nosuchaction', 'view'); // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view'
-        return jsConfigVars;
-    }
 }
 export { Downloader as DownloaderClass };
 const dl = Downloader.getInstance();