mwoffliner 1.17.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +355 -62
- package/docs/functional_architecture.md +34 -38
- package/extensions/wiktionary_fr.js +18 -17
- package/jest.config.cjs +2 -2
- package/lib/DOMUtils.js +0 -1
- package/lib/DOMUtils.js.map +1 -1
- package/lib/Downloader.d.ts +39 -31
- package/lib/Downloader.js +418 -349
- package/lib/Downloader.js.map +1 -1
- package/lib/Dump.d.ts +24 -10
- package/lib/Dump.js +95 -50
- package/lib/Dump.js.map +1 -1
- package/lib/Gadgets.d.ts +2 -1
- package/lib/Gadgets.js +7 -5
- package/lib/Gadgets.js.map +1 -1
- package/lib/Logger.d.ts +6 -6
- package/lib/Logger.js +33 -33
- package/lib/Logger.js.map +1 -1
- package/lib/MediaWiki.d.ts +14 -31
- package/lib/MediaWiki.js +119 -172
- package/lib/MediaWiki.js.map +1 -1
- package/lib/RedisStore.d.ts +3 -3
- package/lib/RedisStore.js +21 -23
- package/lib/RedisStore.js.map +1 -1
- package/lib/S3.js +1 -1
- package/lib/S3.js.map +1 -1
- package/lib/Templates.d.ts +3 -10
- package/lib/Templates.js +5 -14
- package/lib/Templates.js.map +1 -1
- package/lib/cli.js +7 -6
- package/lib/cli.js.map +1 -1
- package/lib/config.d.ts +10 -16
- package/lib/config.js +33 -55
- package/lib/config.js.map +1 -1
- package/lib/error.manager.d.ts +2 -2
- package/lib/error.manager.js +42 -49
- package/lib/error.manager.js.map +1 -1
- package/lib/i18n.d.ts +2 -0
- package/lib/i18n.js +53 -0
- package/lib/i18n.js.map +1 -0
- package/lib/mutex.d.ts +2 -1
- package/lib/mutex.js +2 -1
- package/lib/mutex.js.map +1 -1
- package/lib/mwoffliner.lib.js +370 -222
- package/lib/mwoffliner.lib.js.map +1 -1
- package/lib/parameterList.d.ts +20 -8
- package/lib/parameterList.js +30 -18
- package/lib/parameterList.js.map +1 -1
- package/lib/renderers/abstract.renderer.d.ts +48 -55
- package/lib/renderers/abstract.renderer.js +377 -98
- package/lib/renderers/abstract.renderer.js.map +1 -1
- package/lib/renderers/action-parse.renderer.d.ts +22 -3
- package/lib/renderers/action-parse.renderer.js +123 -81
- package/lib/renderers/action-parse.renderer.js.map +1 -1
- package/lib/renderers/renderer.builder.js +6 -69
- package/lib/renderers/renderer.builder.js.map +1 -1
- package/lib/renderers/rendering.context.d.ts +2 -3
- package/lib/renderers/rendering.context.js +6 -14
- package/lib/renderers/rendering.context.js.map +1 -1
- package/lib/sanitize-argument.d.ts +7 -4
- package/lib/sanitize-argument.js +99 -41
- package/lib/sanitize-argument.js.map +1 -1
- package/lib/util/FileManager.d.ts +36 -0
- package/lib/util/FileManager.js +246 -0
- package/lib/util/FileManager.js.map +1 -0
- package/lib/util/RedisKvs.js +4 -4
- package/lib/util/RedisKvs.js.map +1 -1
- package/lib/util/RedisQueue.js +1 -1
- package/lib/util/RedisQueue.js.map +1 -1
- package/lib/util/builders/url/action-parse.director.d.ts +2 -2
- package/lib/util/builders/url/action-parse.director.js +7 -7
- package/lib/util/builders/url/action-parse.director.js.map +1 -1
- package/lib/util/builders/url/api.director.d.ts +2 -4
- package/lib/util/builders/url/api.director.js +10 -17
- package/lib/util/builders/url/api.director.js.map +1 -1
- package/lib/util/builders/url/base.director.d.ts +0 -4
- package/lib/util/builders/url/base.director.js +0 -25
- package/lib/util/builders/url/base.director.js.map +1 -1
- package/lib/util/builders/url/basic.director.d.ts +1 -1
- package/lib/util/builders/url/basic.director.js +1 -1
- package/lib/util/builders/url/web.director.d.ts +1 -1
- package/lib/util/builders/url/web.director.js +2 -2
- package/lib/util/builders/url/web.director.js.map +1 -1
- package/lib/util/categories.d.ts +5 -5
- package/lib/util/categories.js +175 -174
- package/lib/util/categories.js.map +1 -1
- package/lib/util/const.d.ts +4 -8
- package/lib/util/const.js +9 -17
- package/lib/util/const.js.map +1 -1
- package/lib/util/customCssJs.d.ts +6 -0
- package/lib/util/customCssJs.js +70 -0
- package/lib/util/customCssJs.js.map +1 -0
- package/lib/util/dump.d.ts +15 -4
- package/lib/util/dump.js +282 -86
- package/lib/util/dump.js.map +1 -1
- package/lib/util/index.d.ts +1 -1
- package/lib/util/index.js +1 -1
- package/lib/util/index.js.map +1 -1
- package/lib/util/metaData.js.map +1 -1
- package/lib/util/misc.d.ts +23 -13
- package/lib/util/misc.js +90 -59
- package/lib/util/misc.js.map +1 -1
- package/lib/util/mw-api.d.ts +7 -5
- package/lib/util/mw-api.js +167 -221
- package/lib/util/mw-api.js.map +1 -1
- package/lib/util/pageListMainPage.d.ts +3 -0
- package/lib/util/pageListMainPage.js +10 -0
- package/lib/util/pageListMainPage.js.map +1 -0
- package/lib/util/pages.d.ts +30 -0
- package/lib/util/pages.js +146 -0
- package/lib/util/pages.js.map +1 -0
- package/lib/util/rewriteUrls.d.ts +2 -2
- package/lib/util/rewriteUrls.js +33 -31
- package/lib/util/rewriteUrls.js.map +1 -1
- package/lib/util/savePages.d.ts +8 -0
- package/lib/util/savePages.js +182 -0
- package/lib/util/savePages.js.map +1 -0
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/offliner-definition.json +178 -90
- package/package.json +50 -49
- package/res/page_list_home.js +20 -0
- package/res/{article_not_found.svg → page_not_found.svg} +1 -1
- package/res/script.js +30 -45
- package/res/style.css +52 -68
- package/res/templates/download_error_placeholder.html +7 -8
- package/res/templates/javaScript.html +19 -0
- package/res/templates/pageFallback.html +9 -7
- package/res/templates/pageVector2022.html +9 -7
- package/res/templates/pageVectorLegacy.html +9 -7
- package/res/templates/{article_list_home.html → page_list_home.html} +1 -2
- package/translation/ar.json +21 -21
- package/translation/bn.json +1 -1
- package/translation/de.json +46 -19
- package/translation/en.json +54 -21
- package/translation/es.json +17 -19
- package/translation/fi.json +23 -3
- package/translation/fr.json +57 -21
- package/translation/he.json +8 -6
- package/translation/ia.json +19 -19
- package/translation/id.json +13 -13
- package/translation/it.json +13 -8
- package/translation/ko.json +55 -3
- package/translation/lb.json +5 -2
- package/translation/mk.json +5 -5
- package/translation/nl.json +57 -22
- package/translation/or.json +2 -2
- package/translation/pt.json +2 -2
- package/translation/qqq.json +55 -22
- package/translation/ru.json +2 -2
- package/translation/sc.json +2 -2
- package/translation/sk.json +62 -0
- package/translation/sl.json +54 -21
- package/translation/sv.json +20 -19
- package/translation/sw.json +45 -2
- package/translation/zh-hans.json +56 -22
- package/translation/zh-hant.json +26 -19
- package/lib/renderers/abstractDesktop.render.d.ts +0 -12
- package/lib/renderers/abstractDesktop.render.js +0 -67
- package/lib/renderers/abstractDesktop.render.js.map +0 -1
- package/lib/renderers/abstractMobile.render.d.ts +0 -12
- package/lib/renderers/abstractMobile.render.js +0 -54
- package/lib/renderers/abstractMobile.render.js.map +0 -1
- package/lib/renderers/rest-api.renderer.d.ts +0 -7
- package/lib/renderers/rest-api.renderer.js +0 -68
- package/lib/renderers/rest-api.renderer.js.map +0 -1
- package/lib/renderers/visual-editor.renderer.d.ts +0 -7
- package/lib/renderers/visual-editor.renderer.js +0 -76
- package/lib/renderers/visual-editor.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-desktop.renderer.d.ts +0 -4
- package/lib/renderers/wikimedia-desktop.renderer.js +0 -7
- package/lib/renderers/wikimedia-desktop.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-mobile.renderer.d.ts +0 -19
- package/lib/renderers/wikimedia-mobile.renderer.js +0 -243
- package/lib/renderers/wikimedia-mobile.renderer.js.map +0 -1
- package/lib/util/articleListMainPage.d.ts +0 -3
- package/lib/util/articleListMainPage.js +0 -10
- package/lib/util/articleListMainPage.js.map +0 -1
- package/lib/util/articles.d.ts +0 -26
- package/lib/util/articles.js +0 -102
- package/lib/util/articles.js.map +0 -1
- package/lib/util/builders/url/desktop.director.d.ts +0 -8
- package/lib/util/builders/url/desktop.director.js +0 -15
- package/lib/util/builders/url/desktop.director.js.map +0 -1
- package/lib/util/builders/url/mobile.director.d.ts +0 -8
- package/lib/util/builders/url/mobile.director.js +0 -15
- package/lib/util/builders/url/mobile.director.js.map +0 -1
- package/lib/util/builders/url/rest-api.director.d.ts +0 -8
- package/lib/util/builders/url/rest-api.director.js +0 -18
- package/lib/util/builders/url/rest-api.director.js.map +0 -1
- package/lib/util/builders/url/visual-editor.director.d.ts +0 -9
- package/lib/util/builders/url/visual-editor.director.js +0 -18
- package/lib/util/builders/url/visual-editor.director.js.map +0 -1
- package/lib/util/saveArticles.d.ts +0 -8
- package/lib/util/saveArticles.js +0 -411
- package/lib/util/saveArticles.js.map +0 -1
- package/res/article_list_home.js +0 -24
- package/res/content.parsoid.css +0 -196
- package/res/inserted_style.css +0 -45
- package/res/templates/categories.html +0 -11
- package/res/templates/lead_section_wrapper.html +0 -6
- package/res/templates/pageWikimediaDesktop.html +0 -24
- package/res/templates/pageWikimediaMobile.html +0 -24
- package/res/templates/section_wrapper.html +0 -5
- package/res/templates/subcategories.html +0 -23
- package/res/templates/subpages.html +0 -17
- package/res/templates/subsection_wrapper.html +0 -5
- package/res/webpHandler.js +0 -165
- package/res/wm_mobile_override_script.js +0 -15
- package/res/wm_mobile_override_style.css +0 -20
- package/translation/br.json +0 -8
- package/translation/dag.json +0 -9
- package/translation/es-formal.json +0 -29
- package/translation/ha.json +0 -10
- package/translation/hi.json +0 -9
- package/translation/ig.json +0 -10
- package/translation/kaa.json +0 -9
- package/translation/nb.json +0 -9
- package/translation/nqo.json +0 -8
- package/translation/pt-br.json +0 -10
- package/translation/ro.json +0 -9
- package/translation/scn.json +0 -8
- package/translation/sq.json +0 -9
- package/translation/te.json +0 -9
- package/translation/tn.json +0 -9
- package/translation/tr.json +0 -10
package/lib/util/mw-api.js
CHANGED
|
@@ -1,244 +1,165 @@
|
|
|
1
1
|
import pmap from 'p-map';
|
|
2
|
-
import deepmerge from 'deepmerge';
|
|
3
2
|
import * as logger from '../Logger.js';
|
|
4
3
|
import Downloader from '../Downloader.js';
|
|
5
4
|
import Timer from './Timer.js';
|
|
6
5
|
import RedisStore from '../RedisStore.js';
|
|
7
6
|
import MediaWiki from '../MediaWiki.js';
|
|
8
|
-
import { REDIRECT_PAGE_SIGNATURE } from './const.js';
|
|
9
7
|
import { cleanupAxiosError } from './misc.js';
|
|
10
|
-
|
|
8
|
+
const MAX_TITLES_PARAM_SIZE = 7400;
|
|
9
|
+
const MAX_BATCH_SIZE = 50;
|
|
10
|
+
export function trimPageTitlesBatch(pageTitles) {
|
|
11
|
+
const batch = pageTitles.slice(0, MAX_BATCH_SIZE);
|
|
12
|
+
while (batch.length > 1 && new URLSearchParams({ titles: batch.join('|') }).toString().length > MAX_TITLES_PARAM_SIZE) {
|
|
13
|
+
batch.pop();
|
|
14
|
+
}
|
|
15
|
+
return batch;
|
|
16
|
+
}
|
|
17
|
+
export async function getPagesByTitle(pageTitles, purpose, pagesToIgnore, allowedContentModels = ['wikitext'], categoryTitles = new Set()) {
|
|
11
18
|
let from = 0;
|
|
12
19
|
let numThumbnails = 0;
|
|
13
|
-
const MAX_BATCH_SIZE = 50;
|
|
14
|
-
const MAX_URL_SIZE = 7900; // in bytes, approx.
|
|
15
|
-
const { articleDetailXId, redirectsXId } = RedisStore;
|
|
16
20
|
// using async iterator to spawn workers
|
|
17
21
|
await pmap(','
|
|
18
|
-
.repeat(Downloader.
|
|
22
|
+
.repeat(Downloader.workers)
|
|
19
23
|
.split(',')
|
|
20
24
|
.map((_, i) => i), async (workerId) => {
|
|
21
|
-
while (from <
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
urlSize -= encodeURIComponent(articleIdsBatch.pop()).length + 1;
|
|
27
|
-
}
|
|
28
|
-
// Udpate articleIds slicing boundaries
|
|
29
|
-
const to = from + articleIdsBatch.length;
|
|
30
|
-
if (log) {
|
|
31
|
-
const progressPercent = Math.floor((to / articleIds.length) * 100);
|
|
32
|
-
logger.log(`Worker [${workerId}] getting article range [${from}-${to}] of [${articleIds.length}] [${progressPercent}%]`);
|
|
33
|
-
}
|
|
25
|
+
while (from < pageTitles.length) {
|
|
26
|
+
const pageTitlesBatch = trimPageTitlesBatch(pageTitles.slice(from));
|
|
27
|
+
const to = from + pageTitlesBatch.length;
|
|
28
|
+
const progressPercent = Math.floor((to / pageTitles.length) * 100);
|
|
29
|
+
logger.info(`Worker [${workerId}] getting page range [${from}-${to}] of [${pageTitles.length}] for ${purpose} [${progressPercent}%]`);
|
|
34
30
|
from = to;
|
|
35
31
|
// Nothing to do
|
|
36
|
-
if (!
|
|
32
|
+
if (!pageTitlesBatch.length) {
|
|
37
33
|
continue;
|
|
38
34
|
}
|
|
39
35
|
// Retrieve the details and save them in Redis
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
.filter((a) => !a.revisions)
|
|
44
|
-
.map((article) => article.title);
|
|
45
|
-
if (articlesIgnored.length > 0) {
|
|
46
|
-
logger.warn(`Ignoring articles without revisions: ${articlesIgnored.join(', ')}`);
|
|
47
|
-
}
|
|
48
|
-
const mwArticleDetails = Object.fromEntries(Object.entries(allArticleDetails).filter(([, articleDetail]) => !articlesIgnored.includes(articleDetail.title)));
|
|
49
|
-
const articlesWithThumbnail = Object.values(mwArticleDetails).filter((a) => !!a.thumbnail);
|
|
50
|
-
numThumbnails += articlesWithThumbnail.length;
|
|
51
|
-
const articleDetails = mwRetToArticleDetail(mwArticleDetails);
|
|
52
|
-
for (const [articleId, articleDetail] of Object.entries(mwArticleDetails)) {
|
|
53
|
-
if (articleDetail.redirects && articleDetail.redirects.length) {
|
|
54
|
-
await redirectsXId.setMany(articleDetail.redirects.reduce((acc, redirect) => {
|
|
55
|
-
acc[redirect.title] = { title: redirect.title, targetId: articleId, fragment: redirect.fragment || '' };
|
|
56
|
-
return acc;
|
|
57
|
-
}, {}));
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
const keys = Object.keys(articleDetails);
|
|
61
|
-
if (keys.length == 0) {
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
const existingArticleDetails = await articleDetailXId.getMany(keys);
|
|
65
|
-
await articleDetailXId.setMany(deepmerge(existingArticleDetails, articleDetails));
|
|
36
|
+
const allMwPages = await Downloader.getPagesByTitle(pageTitlesBatch, numThumbnails < 100, purpose !== 'categories');
|
|
37
|
+
const { numThumbnails: iterThumbnails } = await processPagesAndSaveToRedis(allMwPages, pagesToIgnore, allowedContentModels, categoryTitles);
|
|
38
|
+
numThumbnails += iterThumbnails;
|
|
66
39
|
}
|
|
67
|
-
}, { concurrency: Downloader.
|
|
40
|
+
}, { concurrency: Downloader.workers });
|
|
68
41
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
return [numArticles];
|
|
42
|
+
export function filterPages(pages, pagesToIgnore, allowedContentModels) {
|
|
43
|
+
function revisionFilter(page) {
|
|
44
|
+
return !page.revisions;
|
|
73
45
|
}
|
|
74
|
-
|
|
75
|
-
return
|
|
46
|
+
function contentModelFilter(page) {
|
|
47
|
+
return !allowedContentModels.includes(page.contentmodel);
|
|
48
|
+
}
|
|
49
|
+
function ignoredPagesFilter(page) {
|
|
50
|
+
return pagesToIgnore.includes(page.title) || pagesToIgnore.includes(page.title.replace(/ /g, '_'));
|
|
51
|
+
}
|
|
52
|
+
// Filter pages without revisions (#2091)
|
|
53
|
+
const revisionIssues = pages.filter(revisionFilter);
|
|
54
|
+
if (revisionIssues.length > 0) {
|
|
55
|
+
logger.debug(`Ignoring pages without revisions: ${revisionIssues.join(', ')}`);
|
|
76
56
|
}
|
|
57
|
+
// Filter pages with unexpected content model (#2445)
|
|
58
|
+
const contentModelIssues = pages.filter(contentModelFilter);
|
|
59
|
+
if (contentModelIssues.length > 0) {
|
|
60
|
+
logger.debug(`Ignoring pages with unexpected content model: ${contentModelIssues.join(', ')}`);
|
|
61
|
+
}
|
|
62
|
+
// Filter pages asked for by user
|
|
63
|
+
const ignoreListIssues = pages.filter(ignoredPagesFilter);
|
|
64
|
+
if (ignoreListIssues.length > 0) {
|
|
65
|
+
logger.debug(`Ignoring pages in list: ${ignoreListIssues.join(', ')}`);
|
|
66
|
+
}
|
|
67
|
+
return pages
|
|
68
|
+
.filter((p) => !revisionFilter(p))
|
|
69
|
+
.filter((p) => !contentModelFilter(p))
|
|
70
|
+
.filter((p) => !ignoredPagesFilter(p));
|
|
77
71
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
logger.error(errorMessage);
|
|
96
|
-
reject(new Error(errorMessage));
|
|
97
|
-
}, timeout);
|
|
98
|
-
try {
|
|
99
|
-
/*
|
|
100
|
-
* fetch article details in chunks and store them in redis
|
|
101
|
-
*/
|
|
102
|
-
do {
|
|
103
|
-
timer.reset();
|
|
104
|
-
curStage = 0;
|
|
105
|
-
chunk = await Downloader.getArticleDetailsNS(ns, chunk && chunk.gapContinue);
|
|
106
|
-
// Filter articles without revisions (#2238)
|
|
107
|
-
const newArticlesToIgnore = Object.values(chunk.articleDetails)
|
|
108
|
-
.filter((a) => !a.revisions)
|
|
109
|
-
.map((article) => article.title);
|
|
110
|
-
if (newArticlesToIgnore.length > 0) {
|
|
111
|
-
logger.warn(`Ignoring articles without revisions: ${newArticlesToIgnore.join(', ')}`);
|
|
112
|
-
if (!articleIdsToIgnore) {
|
|
113
|
-
articleIdsToIgnore = [];
|
|
114
|
-
}
|
|
115
|
-
articleIdsToIgnore.push(...newArticlesToIgnore);
|
|
116
|
-
}
|
|
117
|
-
if (articleIdsToIgnore) {
|
|
118
|
-
Object.keys(chunk.articleDetails).forEach((articleId) => {
|
|
119
|
-
const articleTitle = chunk.articleDetails[articleId].title;
|
|
120
|
-
if (articleIdsToIgnore.includes(articleTitle)) {
|
|
121
|
-
delete chunk.articleDetails[articleId];
|
|
122
|
-
logger.info(`Excluded article ${articleTitle}`);
|
|
123
|
-
}
|
|
72
|
+
async function processPagesAndSaveToRedis(pages, pagesToIgnore, allowedContentModels, categoryTitles = new Set()) {
|
|
73
|
+
const redirects = [];
|
|
74
|
+
const pagesToRemove = [];
|
|
75
|
+
const filteredPages = filterPages(pages, pagesToIgnore, allowedContentModels);
|
|
76
|
+
for (const page of filteredPages) {
|
|
77
|
+
page.categories?.forEach((category) => {
|
|
78
|
+
if (!category.hidden)
|
|
79
|
+
categoryTitles.add(category.title);
|
|
80
|
+
});
|
|
81
|
+
if (page.redirects) {
|
|
82
|
+
for (const target of page.redirects) {
|
|
83
|
+
const targetExistsAsPage = (await RedisStore.pagesStore.exists(target.title)) || filteredPages.filter((a) => a.title == target.title).length > 0;
|
|
84
|
+
if (targetExistsAsPage) {
|
|
85
|
+
logger.warn(`Page '${target.title}' found in redirects of '${page.title}' while it is also listed among titles to fetch ; scraper will automatically recover from this edge case`);
|
|
86
|
+
redirects.push({
|
|
87
|
+
from: page.title,
|
|
88
|
+
to: target.title,
|
|
124
89
|
});
|
|
90
|
+
pagesToRemove.push(page);
|
|
125
91
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
if (targetExistsAsArticle) {
|
|
133
|
-
logger.warn(`Article '${target.title}' found in redirects of '${articleId}' while it is also listed among articles to fetch ; scraper will automatically recover from this edge case`);
|
|
134
|
-
redirects[articleId] = {
|
|
135
|
-
targetId: target.title,
|
|
136
|
-
title: articleId,
|
|
137
|
-
};
|
|
138
|
-
delete chunk.articleDetails[articleId];
|
|
139
|
-
}
|
|
140
|
-
else {
|
|
141
|
-
redirects[target.title] = {
|
|
142
|
-
targetId: articleId,
|
|
143
|
-
title: target.title,
|
|
144
|
-
fragment: target.fragment,
|
|
145
|
-
};
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
curStage += 1;
|
|
151
|
-
/*
|
|
152
|
-
* Don't await redis, push promise into array queue
|
|
153
|
-
* and check if the oldest promise finished in the meantime.
|
|
154
|
-
* Like this we can store in redis simultaniously to downloading the
|
|
155
|
-
* next articles, but also won't run into an ever increasing backlog.
|
|
156
|
-
*/
|
|
157
|
-
const newSavePromise = saveToStore(mwRetToArticleDetail(chunk.articleDetails), redirects, articleDetailXId, redirectsXId);
|
|
158
|
-
if (saveStorePromisQueue.length) {
|
|
159
|
-
/*
|
|
160
|
-
* in normal circumstances, where downloading is slower than storing,
|
|
161
|
-
* this promise will always be resolved here already
|
|
162
|
-
*/
|
|
163
|
-
const [numArticles, err] = await saveStorePromisQueue.shift();
|
|
164
|
-
if (err) {
|
|
165
|
-
timer.clear();
|
|
166
|
-
reject(err);
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
totalArticles += numArticles;
|
|
170
|
-
logger.log(`Got [${numArticles} / ${totalArticles}] articles chunk from namespace [${ns}]`);
|
|
92
|
+
else {
|
|
93
|
+
redirects.push({
|
|
94
|
+
from: target.title,
|
|
95
|
+
to: page.title,
|
|
96
|
+
fragment: target.fragment,
|
|
97
|
+
});
|
|
171
98
|
}
|
|
172
|
-
saveStorePromisQueue.push(newSavePromise);
|
|
173
|
-
// Only for testing purposes
|
|
174
|
-
if (--continueLimit < 0)
|
|
175
|
-
break;
|
|
176
|
-
} while (chunk.gapContinue);
|
|
177
|
-
/*
|
|
178
|
-
* clear up potentially still pending promises
|
|
179
|
-
*/
|
|
180
|
-
curStage = 3;
|
|
181
|
-
const lastPending = await Promise.all(saveStorePromisQueue);
|
|
182
|
-
const errored = lastPending.find(([, err]) => err);
|
|
183
|
-
if (errored) {
|
|
184
|
-
throw errored[1];
|
|
185
99
|
}
|
|
186
|
-
totalArticles += lastPending.reduce((a, [b]) => a + b, 0);
|
|
187
100
|
}
|
|
188
|
-
catch (err) {
|
|
189
|
-
logger.error(`Error fetching article details at ${stages[curStage]}`);
|
|
190
|
-
reject(err);
|
|
191
|
-
return;
|
|
192
|
-
}
|
|
193
|
-
finally {
|
|
194
|
-
timer.clear();
|
|
195
|
-
}
|
|
196
|
-
logger.log(`A total of [${totalArticles}] articles has been found in namespace [${ns}]`);
|
|
197
|
-
resolve();
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
export function normalizeMwResponse(response) {
|
|
201
|
-
if (!response) {
|
|
202
|
-
return {};
|
|
203
101
|
}
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
102
|
+
for (const page of pagesToRemove) {
|
|
103
|
+
filteredPages.splice(filteredPages.indexOf(page), 1);
|
|
104
|
+
}
|
|
105
|
+
const numPages = await RedisStore.pagesStore.setMany(mwRetToPageDetail(filteredPages));
|
|
106
|
+
const numRedirects = await RedisStore.redirectsStore.setMany(redirects.reduce((acc, redirect) => {
|
|
107
|
+
acc[redirect.from] = { from: redirect.from, to: redirect.to, fragment: redirect.fragment || '' };
|
|
207
108
|
return acc;
|
|
208
|
-
}, {});
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
if (typeof id !== 'string' || !id) {
|
|
212
|
-
logger.warn(`Article Id is invalid - expected a string but got [${id}], converting to string and continuing`);
|
|
213
|
-
}
|
|
214
|
-
const articleId = String(id).replace(/ /g, '_');
|
|
215
|
-
if (page.redirects) {
|
|
216
|
-
page.redirects = page.redirects
|
|
217
|
-
// drop redirects from talk (not subject) namespaces and from User namespace, except if namespace has been expressly requested
|
|
218
|
-
.filter((redirect) => (redirect.ns % 2 === 0 && redirect.ns !== 2) || MediaWiki.namespacesToMirror.some((ns) => MediaWiki.namespaces[ns].num === redirect.ns))
|
|
219
|
-
.map((redirect) => {
|
|
220
|
-
// The API returns the redirect title (!?), we fake the
|
|
221
|
-
// redirectId by putting the underscore. That way we
|
|
222
|
-
// secure the URL rewritting works fine.
|
|
223
|
-
redirect.title = String(redirect.title).replace(/ /g, '_');
|
|
224
|
-
return redirect;
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
if (articleId) {
|
|
228
|
-
return {
|
|
229
|
-
...acc,
|
|
230
|
-
[articleId]: page,
|
|
231
|
-
};
|
|
232
|
-
}
|
|
233
|
-
else {
|
|
234
|
-
return acc;
|
|
235
|
-
}
|
|
236
|
-
}, {});
|
|
109
|
+
}, {}));
|
|
110
|
+
const numThumbnails = filteredPages.filter((page) => !!page.thumbnail).length;
|
|
111
|
+
return { numPages, numRedirects, numThumbnails };
|
|
237
112
|
}
|
|
238
|
-
export function
|
|
113
|
+
export async function getPagesByNamespace(namespace, pagesToIgnore, allowedContentModels = ['wikitext'], categoryTitles = new Set(), continueLimit) {
|
|
114
|
+
let totalPages = 0;
|
|
115
|
+
let gapContinue = '';
|
|
116
|
+
const seenGapContinueValues = [];
|
|
117
|
+
// We don't really know how long this is going to take because we have a query continuation parameter which might induce
|
|
118
|
+
// more request to make + we also get categories with more requests which is a recursive call
|
|
119
|
+
const timeout = Math.max(Downloader.requestTimeout * 2, 10 * 60 * 1000);
|
|
120
|
+
const timer = new Timer(() => {
|
|
121
|
+
const errorMessage = `Worker timed out after ${timeout} ms`;
|
|
122
|
+
logger.error(errorMessage);
|
|
123
|
+
throw new Error(errorMessage);
|
|
124
|
+
}, timeout);
|
|
125
|
+
try {
|
|
126
|
+
/*
|
|
127
|
+
* fetch page details in chunks and store them in redis
|
|
128
|
+
*/
|
|
129
|
+
do {
|
|
130
|
+
timer.reset();
|
|
131
|
+
const resp = await Downloader.getPagesByNamespace(namespace, gapContinue);
|
|
132
|
+
gapContinue = resp.gapContinue;
|
|
133
|
+
if (gapContinue) {
|
|
134
|
+
if (seenGapContinueValues.includes(gapContinue)) {
|
|
135
|
+
throw new Error(`Detected continuation cycle while fetching pages in namespace ${namespace}. ` +
|
|
136
|
+
`Repeated gapContinue=${gapContinue} after visiting: [${seenGapContinueValues.join(', ')}]`);
|
|
137
|
+
}
|
|
138
|
+
seenGapContinueValues.push(gapContinue);
|
|
139
|
+
}
|
|
140
|
+
const { numPages } = await processPagesAndSaveToRedis(resp.pages, pagesToIgnore, allowedContentModels, categoryTitles);
|
|
141
|
+
totalPages += numPages;
|
|
142
|
+
logger.info(`Got [${numPages} / ${totalPages}] pages chunk from namespace [${namespace}]`);
|
|
143
|
+
// Only for testing purposes
|
|
144
|
+
if (--continueLimit < 0)
|
|
145
|
+
break;
|
|
146
|
+
} while (gapContinue);
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
timer.clear();
|
|
150
|
+
}
|
|
151
|
+
logger.info(`A total of [${totalPages}] pages has been found in namespace [${namespace}]`);
|
|
152
|
+
}
|
|
153
|
+
export function filterRedirects(mwPage) {
|
|
154
|
+
if (mwPage.redirects) {
|
|
155
|
+
mwPage.redirects = mwPage.redirects
|
|
156
|
+
// keep redirects only to current page namespace + other content namespaces
|
|
157
|
+
.filter((redirect) => redirect.ns == mwPage.ns || MediaWiki.namespacesToMirror.some((ns) => MediaWiki.namespaces[ns].num === redirect.ns));
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
export function mwRetToPageDetail(pages) {
|
|
239
161
|
const ret = {};
|
|
240
|
-
for (const
|
|
241
|
-
const val = obj[key];
|
|
162
|
+
for (const val of pages) {
|
|
242
163
|
const rev = val.revisions && val.revisions[0];
|
|
243
164
|
const geo = val.coordinates && val.coordinates[0];
|
|
244
165
|
let newThumbnail;
|
|
@@ -249,17 +170,29 @@ export function mwRetToArticleDetail(obj) {
|
|
|
249
170
|
source: val.thumbnail.source,
|
|
250
171
|
};
|
|
251
172
|
}
|
|
252
|
-
|
|
173
|
+
let newCategoryinfo;
|
|
174
|
+
if (val.categoryinfo) {
|
|
175
|
+
newCategoryinfo = {
|
|
176
|
+
...val.categoryinfo,
|
|
177
|
+
nogallery: val.pageprops && val.pageprops.nogallery === '',
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
let allCategories;
|
|
181
|
+
if (val.categories) {
|
|
182
|
+
allCategories = val.categories.map(({ title }) => title.split(':').slice(1).join(':'));
|
|
183
|
+
}
|
|
184
|
+
ret[val.title] = {
|
|
253
185
|
title: val.title,
|
|
254
|
-
categories: val.categories,
|
|
255
|
-
subCategories: val.subCategories,
|
|
256
186
|
thumbnail: newThumbnail,
|
|
187
|
+
categoryinfo: newCategoryinfo,
|
|
188
|
+
categories: allCategories,
|
|
257
189
|
missing: val.missing,
|
|
258
190
|
pagelang: val.pagelanguagehtmlcode,
|
|
259
191
|
pagedir: val.pagelanguagedir,
|
|
260
192
|
...(val.ns !== 0 ? { ns: val.ns } : {}),
|
|
261
193
|
...(val.contentmodel !== 'wikitext' ? { contentmodel: val.contentmodel } : {}),
|
|
262
194
|
...(rev ? { revisionId: rev.revid, timestamp: rev.timestamp } : {}),
|
|
195
|
+
...(val.flagged?.stable_revid ? { stableRevisionId: val.flagged.stable_revid } : {}),
|
|
263
196
|
...(geo ? { coordinates: `${geo.lat};${geo.lon}` } : {}),
|
|
264
197
|
};
|
|
265
198
|
}
|
|
@@ -278,7 +211,6 @@ export function mwRetToArticleDetail(obj) {
|
|
|
278
211
|
export async function checkApiAvailability(url, allowedMimeTypes = null) {
|
|
279
212
|
try {
|
|
280
213
|
const resp = await Downloader.request({ url: decodeURI(url), method: 'GET', maxRedirects: 0, ...Downloader.basicRequestOptions });
|
|
281
|
-
const isRedirectPage = typeof resp.data === 'string' && resp.data.startsWith(REDIRECT_PAGE_SIGNATURE);
|
|
282
214
|
// Read https://phabricator.wikimedia.org/T359187 to understand
|
|
283
215
|
// the 'mediawiki-api-error' === 'rest-permission-error' exception
|
|
284
216
|
const isSuccess = resp.status === 200 && (!resp.headers['mediawiki-api-error'] || resp.headers['mediawiki-api-error'] === 'rest-permission-error');
|
|
@@ -295,24 +227,38 @@ export async function checkApiAvailability(url, allowedMimeTypes = null) {
|
|
|
295
227
|
}
|
|
296
228
|
}
|
|
297
229
|
}
|
|
298
|
-
return
|
|
230
|
+
return isSuccess && validMimeType;
|
|
299
231
|
}
|
|
300
232
|
catch (err) {
|
|
301
|
-
logger.
|
|
233
|
+
logger.debug('checkApiAvailability failed: ', cleanupAxiosError(err));
|
|
302
234
|
return false;
|
|
303
235
|
}
|
|
304
236
|
}
|
|
305
|
-
export async function
|
|
237
|
+
export async function getPages(mainPage, pages = [], pagesToIgnore = [], allowedContentModels = ['wikitext']) {
|
|
238
|
+
const categorySet = new Set();
|
|
306
239
|
if (mainPage) {
|
|
307
|
-
await
|
|
240
|
+
await getPagesByTitle([mainPage], 'mainPage', pagesToIgnore, allowedContentModels, categorySet);
|
|
308
241
|
}
|
|
309
|
-
if (
|
|
310
|
-
await
|
|
242
|
+
if (pages.length) {
|
|
243
|
+
await getPagesByTitle(pages, 'pages', pagesToIgnore, allowedContentModels, categorySet);
|
|
311
244
|
}
|
|
312
245
|
else {
|
|
313
246
|
await pmap(MediaWiki.namespacesToMirror, (namespace) => {
|
|
314
|
-
return
|
|
315
|
-
}, { concurrency: Downloader.
|
|
247
|
+
return getPagesByNamespace(MediaWiki.namespaces[namespace].num, pagesToIgnore, allowedContentModels, categorySet);
|
|
248
|
+
}, { concurrency: Downloader.workers });
|
|
249
|
+
}
|
|
250
|
+
if (MediaWiki.getCategories) {
|
|
251
|
+
// Categories can themselves belong to parent categories (subcategories), so we need to walk up the
|
|
252
|
+
// category tree until no new (unprocessed) category is discovered, to also capture root categories
|
|
253
|
+
const processedCategories = new Set();
|
|
254
|
+
let categoriesToFetch = pagesToIgnore ? [...categorySet].filter((title) => !pagesToIgnore.includes(title)) : [...categorySet];
|
|
255
|
+
while (categoriesToFetch.length) {
|
|
256
|
+
categoriesToFetch.forEach((title) => processedCategories.add(title));
|
|
257
|
+
logger.debug(`Fetching category details of ${categoriesToFetch.join(', ')}`);
|
|
258
|
+
const newCategorySet = new Set();
|
|
259
|
+
await getPagesByTitle(categoriesToFetch, 'categories', pagesToIgnore, allowedContentModels, newCategorySet);
|
|
260
|
+
categoriesToFetch = [...newCategorySet].filter((title) => !processedCategories.has(title) && !pagesToIgnore.includes(title));
|
|
261
|
+
}
|
|
316
262
|
}
|
|
317
263
|
}
|
|
318
264
|
//# sourceMappingURL=mw-api.js.map
|
package/lib/util/mw-api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mw-api.js","sourceRoot":"","sources":["../../src/util/mw-api.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,SAAS,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AACtC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,SAAS,MAAM,iBAAiB,CAAA;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAAoB,EAAE,GAAG,GAAG,IAAI;IACrE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,cAAc,GAAG,EAAE,CAAA;IACzB,MAAM,YAAY,GAAG,IAAI,CAAA,CAAC,oBAAoB;IAE9C,MAAM,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,UAAU,CAAA;IAErD,wCAAwC;IACxC,MAAM,IAAI,CACR,GAAG;SACA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;SACxB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACnB,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAChC,2EAA2E;YAC3E,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,GAAG,cAAc,CAAC,CAAA;YACrE,IAAI,OAAO,GAAG,kBAAkB,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;YAClE,OAAO,OAAO,GAAG,YAAY,EAAE,CAAC;gBAC9B,OAAO,IAAI,kBAAkB,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;YACjE,CAAC;YAED,uCAAuC;YACvC,MAAM,EAAE,GAAG,IAAI,GAAG,eAAe,CAAC,MAAM,CAAA;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;gBAClE,MAAM,CAAC,GAAG,CAAC,WAAW,QAAQ,4BAA4B,IAAI,IAAI,EAAE,SAAS,UAAU,CAAC,MAAM,MAAM,eAAe,IAAI,CAAC,CAAA;YAC1H,CAAC;YACD,IAAI,GAAG,EAAE,CAAA;YAET,gBAAgB;YAChB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC5B,SAAQ;YACV,CAAC;YAED,8CAA8C;YAC9C,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,eAAe,EAAE,aAAa,GAAG,GAAG,CAAC,CAAA;YAErG,4CAA4C;YAC5C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC;iBACrD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;iBAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAClC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,wCAAwC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnF,CAAC;YACD,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAE5J,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;YAC1F,aAAa,IAAI,qBAAqB,CAAC,MAAM,CAAA;YAE7C,MAAM,cAAc,GAAG,oBAAoB,CAAC,gBAAgB,CAAC,CAAA;YAE7D,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC1E,IAAI,aAAa,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;oBAC9D,MAAM,YAAY,CAAC,OAAO,CACxB,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC/C,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAA;wBACvG,OAAO,GAAG,CAAA;oBACZ,CAAC,EAAE,EAAE,CAAC,CACP,CAAA;gBACH,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YACxC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,OAAM;YACR,CAAC;YACD,MAAM,sBAAsB,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACnE,MAAM,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,sBAAsB,EAAE,cAAc,CAAC,CAAC,CAAA;QACnF,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,EAAE,CAClC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,cAAkC,EAClC,SAA+B,EAC/B,gBAAqC,EACrC,YAAmC;IAEnC,IAAI,CAAC;QACH,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACpH,OAAO,CAAC,WAAW,CAAC,CAAA;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IACjB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU,EAAE,kBAA6B,EAAE,aAAsB;IAC/F,qDAAqD;IACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,IAAI,aAAa,GAAG,CAAC,CAAA;QACrB,IAAI,KAA0D,CAAA;QAE9D,MAAM,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,UAAU,CAAA;QAErD,MAAM,oBAAoB,GAAgC,EAAE,CAAA;QAE5D;;WAEG;QACH,MAAM,MAAM,GAAG,CAAC,yBAAyB,EAAE,iBAAiB,EAAE,+BAA+B,EAAE,wBAAwB,CAAC,CAAA;QACxH,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,wHAAwH;QACxH,6FAA6F;QAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;QACvE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,YAAY,GAAG,0BAA0B,OAAO,UAAU,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;YAClF,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAA;QACjC,CAAC,EAAE,OAAO,CAAC,CAAA;QAEX,IAAI,CAAC;YACH;;eAEG;YACH,GAAG,CAAC;gBACF,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,QAAQ,GAAG,CAAC,CAAA;gBACZ,KAAK,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,CAAA;gBAE5E,4CAA4C;gBAC5C,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;qBAC5D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;qBAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAClC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC,wCAAwC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;oBACrF,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACxB,kBAAkB,GAAG,EAAE,CAAA;oBACzB,CAAC;oBACD,kBAAkB,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,CAAA;gBACjD,CAAC;gBAED,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;wBACtD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,KAAK,CAAA;wBAC1D,IAAI,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC9C,OAAO,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;4BACtC,MAAM,CAAC,IAAI,CAAC,oBAAoB,YAAY,EAAE,CAAC,CAAA;wBACjD,CAAC;oBACH,CAAC,CAAC,CAAA;gBACJ,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBACb,MAAM,SAAS,GAAyB,EAAE,CAAA;gBAC1C,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC9E,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;wBAC5B,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;4BAC7C,MAAM,qBAAqB,GAAG,CAAC,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;4BAClJ,IAAI,qBAAqB,EAAE,CAAC;gCAC1B,MAAM,CAAC,IAAI,CACT,YAAY,MAAM,CAAC,KAAK,4BAA4B,SAAS,4GAA4G,CAC1K,CAAA;gCACD,SAAS,CAAC,SAAS,CAAC,GAAG;oCACrB,QAAQ,EAAE,MAAM,CAAC,KAAK;oCACtB,KAAK,EAAE,SAAS;iCACjB,CAAA;gCACD,OAAO,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;4BACxC,CAAC;iCAAM,CAAC;gCACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG;oCACxB,QAAQ,EAAE,SAAS;oCACnB,KAAK,EAAE,MAAM,CAAC,KAAK;oCACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iCAC1B,CAAA;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBACb;;;;;mBAKG;gBACH,MAAM,cAAc,GAAG,WAAW,CAAC,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAA;gBAEzH,IAAI,oBAAoB,CAAC,MAAM,EAAE,CAAC;oBAChC;;;uBAGG;oBACH,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAA;oBAC7D,IAAI,GAAG,EAAE,CAAC;wBACR,KAAK,CAAC,KAAK,EAAE,CAAA;wBACb,MAAM,CAAC,GAAG,CAAC,CAAA;wBACX,OAAM;oBACR,CAAC;oBACD,aAAa,IAAI,WAAW,CAAA;oBAC5B,MAAM,CAAC,GAAG,CAAC,QAAQ,WAAW,MAAM,aAAa,oCAAoC,EAAE,GAAG,CAAC,CAAA;gBAC7F,CAAC;gBAED,oBAAoB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;gBAEzC,4BAA4B;gBAC5B,IAAI,EAAG,aAAwB,GAAG,CAAC;oBAAE,MAAK;YAC5C,CAAC,QAAQ,KAAK,CAAC,WAAW,EAAC;YAE3B;;eAEG;YACH,QAAQ,GAAG,CAAC,CAAA;YACZ,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;YAC3D,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAA;YAClD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;YAClB,CAAC;YACD,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,qCAAqC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrE,MAAM,CAAC,GAAG,CAAC,CAAA;YACX,OAAM;QACR,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACf,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,eAAe,aAAa,2CAA2C,EAAE,GAAG,CAAC,CAAA;QACxF,OAAO,EAAE,CAAA;IACX,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA4B;IAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAA;IAEnD,MAAM,UAAU,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE;QACvE,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAA;QACd,OAAO,GAAG,CAAA;IACZ,CAAC,EAAE,EAAE,CAAC,CAAA;IAEN,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC/C,MAAM,EAAE,GAAG,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA,CAAC,4CAA4C;QACzI,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE,wCAAwC,CAAC,CAAA;QAC/G,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAC/C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBAC7B,8HAA8H;iBAC7H,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC;iBAC7J,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAChB,uDAAuD;gBACvD,oDAAoD;gBACpD,wCAAwC;gBACxC,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBAE1D,OAAO,QAAQ,CAAA;YACjB,CAAC,CAAC,CAAA;QACN,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,GAAG,GAAG;gBACN,CAAC,SAAS,CAAC,EAAE,IAAI;aAClB,CAAA;QACH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,CAAA;QACZ,CAAC;IACH,CAAC,EAAE,EAAE,CAAC,CAAA;AACR,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAAe;IAClD,MAAM,GAAG,GAAuB,EAAE,CAAA;IAClC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QACjD,IAAI,YAAY,CAAA;QAChB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,YAAY,GAAG;gBACb,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK;gBAC1B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;gBAC5B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;aAC7B,CAAA;QACH,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,SAAS,EAAE,YAAY;YACvB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,oBAAoB;YAClC,OAAO,EAAE,GAAG,CAAC,eAAe;YAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAA;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,gBAAgB,GAAG,IAAI;IAC7E,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAA;QAEjI,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAA;QAErG,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,uBAAuB,CAAC,CAAA;QAElJ,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,wDAAwD;YACxD,aAAa,GAAG,IAAI,CAAA;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,aAAa,GAAG,IAAI,CAAA;oBACpB,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,cAAc,IAAI,SAAS,IAAI,aAAa,CAAA;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAA;QACpE,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAiB,EAAE,UAAqB,EAAE,kBAA6B;IACzG,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,gBAAgB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAA;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,CACR,SAAS,CAAC,kBAAkB,EAC5B,CAAC,SAAiB,EAAE,EAAE;YACpB,OAAO,eAAe,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAA;QACjF,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,EAAE,CAClC,CAAA;IACH,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"mw-api.js","sourceRoot":"","sources":["../../src/util/mw-api.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AACtC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,SAAS,MAAM,iBAAiB,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAE7C,MAAM,qBAAqB,GAAG,IAAI,CAAA;AAClC,MAAM,cAAc,GAAG,EAAE,CAAA;AAEzB,MAAM,UAAU,mBAAmB,CAAC,UAAuB;IACzD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;IACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;QACtH,KAAK,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAuB,EACvB,OAAe,EACf,aAA2B,EAC3B,uBAAiC,CAAC,UAAU,CAAC,EAC7C,iBAAiC,IAAI,GAAG,EAAE;IAE1C,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,wCAAwC;IACxC,MAAM,IAAI,CACR,GAAG;SACA,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;SAC1B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACnB,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,eAAe,GAAG,mBAAmB,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;YAEnE,MAAM,EAAE,GAAG,IAAI,GAAG,eAAe,CAAC,MAAM,CAAA;YACxC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;YAClE,MAAM,CAAC,IAAI,CAAC,WAAW,QAAQ,yBAAyB,IAAI,IAAI,EAAE,SAAS,UAAU,CAAC,MAAM,SAAS,OAAO,KAAK,eAAe,IAAI,CAAC,CAAA;YACrI,IAAI,GAAG,EAAE,CAAA;YAET,gBAAgB;YAChB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC5B,SAAQ;YACV,CAAC;YAED,8CAA8C;YAC9C,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,eAAe,CAAC,eAAe,EAAE,aAAa,GAAG,GAAG,EAAE,OAAO,KAAK,YAAY,CAAC,CAAA;YACnH,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,MAAM,0BAA0B,CAAC,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YAC3I,aAAa,IAAI,cAAc,CAAA;QACjC,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,OAAO,EAAE,CACpC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAiB,EAAE,aAA0B,EAAE,oBAA8B;IACvG,SAAS,cAAc,CAAC,IAAyB;QAC/C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAA;IACxB,CAAC;IAED,SAAS,kBAAkB,CAAC,IAAyB;QACnD,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC1D,CAAC;IAED,SAAS,kBAAkB,CAAC,IAAyB;QACnD,OAAO,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAE,IAAI,CAAC,KAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAc,CAAC,CAAA;IAC7H,CAAC;IAED,yCAAyC;IACzC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChF,CAAC;IAED,qDAAqD;IACrD,MAAM,kBAAkB,GAAG,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,iDAAiD,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChG,CAAC;IAED,iCAAiC;IACjC,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;IACzD,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,2BAA2B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACxE,CAAC;IAED,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;AAC1C,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,KAAiB,EAAE,aAA0B,EAAE,oBAA8B,EAAE,iBAAiC,IAAI,GAAG,EAAE;IACjK,MAAM,SAAS,GAAmB,EAAE,CAAA;IACpC,MAAM,aAAa,GAA4B,EAAE,CAAA;IACjD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,oBAAoB,CAAC,CAAA;IAC7E,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACpC,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC1D,CAAC,CAAC,CAAA;QACF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpC,MAAM,kBAAkB,GAAG,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAChJ,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CACT,SAAS,MAAM,CAAC,KAAK,4BAA4B,IAAI,CAAC,KAAK,0GAA0G,CACtK,CAAA;oBACD,SAAS,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,IAAI,CAAC,KAAK;wBAChB,EAAE,EAAE,MAAM,CAAC,KAAK;qBACjB,CAAC,CAAA;oBACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC1B,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,MAAM,CAAC,KAAK;wBAClB,EAAE,EAAE,IAAI,CAAC,KAAK;wBACd,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IACtD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAA;IACtF,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,OAAO,CAC1D,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;QACjC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAA;QAChG,OAAO,GAAG,CAAA;IACZ,CAAC,EAAE,EAAE,CAAC,CACP,CAAA;IACD,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAA;IAE7E,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,aAA2B,EAC3B,uBAAiC,CAAC,UAAU,CAAC,EAC7C,iBAAiC,IAAI,GAAG,EAAE,EAC1C,aAAsB;IAEtB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,WAAW,GAAG,EAAE,CAAA;IACpB,MAAM,qBAAqB,GAAa,EAAE,CAAA;IAE1C,wHAAwH;IACxH,6FAA6F;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;IACvE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,YAAY,GAAG,0BAA0B,OAAO,KAAK,CAAA;QAC3D,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAA;IAC/B,CAAC,EAAE,OAAO,CAAC,CAAA;IAEX,IAAI,CAAC;QACH;;WAEG;QACH,GAAG,CAAC;YACF,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;YAEzE,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;YAC9B,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,qBAAqB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CACb,iEAAiE,SAAS,IAAI;wBAC5E,wBAAwB,WAAW,qBAAqB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC9F,CAAA;gBACH,CAAC;gBACD,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACzC,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YACtH,UAAU,IAAI,QAAQ,CAAA;YACtB,MAAM,CAAC,IAAI,CAAC,QAAQ,QAAQ,MAAM,UAAU,iCAAiC,SAAS,GAAG,CAAC,CAAA;YAE1F,4BAA4B;YAC5B,IAAI,EAAG,aAAwB,GAAG,CAAC;gBAAE,MAAK;QAC5C,CAAC,QAAQ,WAAW,EAAC;IACvB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;IACf,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,wCAAwC,SAAS,GAAG,CAAC,CAAA;AAC5F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAA2B;IACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS;YACjC,2EAA2E;aAC1E,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE,IAAI,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9I,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,MAAM,GAAG,GAAoB,EAAE,CAAA;IAC/B,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QACjD,IAAI,YAAY,CAAA;QAChB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,YAAY,GAAG;gBACb,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK;gBAC1B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;gBAC5B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;aAC7B,CAAA;QACH,CAAC;QACD,IAAI,eAAe,CAAA;QACnB,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,eAAe,GAAG;gBAChB,GAAG,GAAG,CAAC,YAAY;gBACnB,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;aAC3D,CAAA;QACH,CAAC;QACD,IAAI,aAAa,CAAA;QACjB,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,aAAa,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACxF,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG;YACf,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,YAAY;YACvB,YAAY,EAAE,eAAe;YAC7B,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,oBAAoB;YAClC,OAAO,EAAE,GAAG,CAAC,eAAe;YAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAA;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,gBAAgB,GAAG,IAAI;IAC7E,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAA;QAEjI,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,uBAAuB,CAAC,CAAA;QAElJ,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,wDAAwD;YACxD,aAAa,GAAG,IAAI,CAAA;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,IAAK,IAAI,CAAC,OAAO,CAAC,cAAc,CAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,aAAa,GAAG,IAAI,CAAA;oBACpB,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,IAAI,aAAa,CAAA;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,iBAAiB,CAAC,GAAU,CAAC,CAAC,CAAA;QAC5E,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAoB,EAAE,QAAqB,EAAE,EAAE,gBAA6B,EAAE,EAAE,uBAAiC,CAAC,UAAU,CAAC;IAC1J,MAAM,WAAW,GAAG,IAAI,GAAG,EAAa,CAAA;IAExC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,eAAe,CAAC,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;IACzF,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,CACR,SAAS,CAAC,kBAAkB,EAC5B,CAAC,SAAiB,EAAE,EAAE;YACpB,OAAO,mBAAmB,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;QACnH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,OAAO,EAAE,CACpC,CAAA;IACH,CAAC;IAED,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;QAC5B,mGAAmG;QACnG,mGAAmG;QACnG,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAa,CAAA;QAChD,IAAI,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,KAAgB,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAA;QAExI,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAChC,iBAAiB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,MAAM,CAAC,KAAK,CAAC,gCAAgC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC5E,MAAM,cAAc,GAAG,IAAI,GAAG,EAAa,CAAA;YAC3C,MAAM,eAAe,CAAC,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YAE3G,iBAAiB,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,KAAgB,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;QACzI,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { encodePageTitleForZimHtmlUrl } from './index.js';
|
|
2
|
+
export function makePageListItem(dump, page) {
|
|
3
|
+
return `<li><a href="${encodePageTitleForZimHtmlUrl(page.title)}">${page.title || ''}<a></li>\n`;
|
|
4
|
+
}
|
|
5
|
+
export function makePageImageTile(dump, page) {
|
|
6
|
+
return `<a class="item" href="${encodePageTitleForZimHtmlUrl(page.title)}">
|
|
7
|
+
<figure><img loading="lazy" src="${page.internalThumbnailUrl}" />
|
|
8
|
+
<figcaption>${page.title || ''}</figcaption></figure></a>\n`;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=pageListMainPage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pageListMainPage.js","sourceRoot":"","sources":["../../src/util/pageListMainPage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAA;AAEzD,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,IAAgB;IAC3D,OAAO,gBAAgB,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,YAAY,CAAA;AAClG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAU,EAAE,IAAgB;IAC5D,OAAO,yBAAyB,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC;+CAC3B,IAAI,CAAC,oBAAoB;0BAC9C,IAAI,CAAC,KAAK,IAAI,EAAE,8BAA8B,CAAA;AACxE,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Check if a given page title is main page of the target ZIM
|
|
3
|
+
*/
|
|
4
|
+
export declare function isMainPage(pageTitle: PageTitle): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Check if a given page title or id is a subpage or not
|
|
7
|
+
*
|
|
8
|
+
* A given is a subpage if subpages are activated on its namespace and it contains a / in its title/id
|
|
9
|
+
*/
|
|
10
|
+
export declare function isSubpage(pageTitleOrId: string): boolean;
|
|
11
|
+
/**
|
|
12
|
+
* For a given namespace number, get the namespace name
|
|
13
|
+
*/
|
|
14
|
+
export declare function getNamespaceName(namespace: number): string;
|
|
15
|
+
/**
|
|
16
|
+
* Extract the JS config variables from its headHtml, typically returned by MW API call when fetching page content
|
|
17
|
+
*/
|
|
18
|
+
export declare function extractJsConfigVars(headHtml: string, extraJsConfigVars?: KVS<any>): KVS<any>;
|
|
19
|
+
/**
|
|
20
|
+
* Extract the CSS class to apply on page <body> tag from its headHtml, typically returned by MW API call when fetching page content
|
|
21
|
+
*/
|
|
22
|
+
export declare function extractBodyCssClass(headHtml: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Extract the CSS class to apply on page <html> tag from its headHtml, typically returned by MW API call when fetching page content
|
|
25
|
+
*/
|
|
26
|
+
export declare function extractHtmlCssClass(headHtml: string): string;
|
|
27
|
+
/**
|
|
28
|
+
* Get the special display title for the main page
|
|
29
|
+
*/
|
|
30
|
+
export declare function getMainpageTitle(): Promise<string>;
|