mwoffliner 1.17.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +355 -62
  2. package/docs/functional_architecture.md +34 -38
  3. package/extensions/wiktionary_fr.js +18 -17
  4. package/jest.config.cjs +2 -2
  5. package/lib/DOMUtils.js +0 -1
  6. package/lib/DOMUtils.js.map +1 -1
  7. package/lib/Downloader.d.ts +39 -31
  8. package/lib/Downloader.js +418 -349
  9. package/lib/Downloader.js.map +1 -1
  10. package/lib/Dump.d.ts +24 -10
  11. package/lib/Dump.js +95 -50
  12. package/lib/Dump.js.map +1 -1
  13. package/lib/Gadgets.d.ts +2 -1
  14. package/lib/Gadgets.js +7 -5
  15. package/lib/Gadgets.js.map +1 -1
  16. package/lib/Logger.d.ts +6 -6
  17. package/lib/Logger.js +33 -33
  18. package/lib/Logger.js.map +1 -1
  19. package/lib/MediaWiki.d.ts +14 -31
  20. package/lib/MediaWiki.js +119 -172
  21. package/lib/MediaWiki.js.map +1 -1
  22. package/lib/RedisStore.d.ts +3 -3
  23. package/lib/RedisStore.js +21 -23
  24. package/lib/RedisStore.js.map +1 -1
  25. package/lib/S3.js +1 -1
  26. package/lib/S3.js.map +1 -1
  27. package/lib/Templates.d.ts +3 -10
  28. package/lib/Templates.js +5 -14
  29. package/lib/Templates.js.map +1 -1
  30. package/lib/cli.js +7 -6
  31. package/lib/cli.js.map +1 -1
  32. package/lib/config.d.ts +10 -16
  33. package/lib/config.js +33 -55
  34. package/lib/config.js.map +1 -1
  35. package/lib/error.manager.d.ts +2 -2
  36. package/lib/error.manager.js +42 -49
  37. package/lib/error.manager.js.map +1 -1
  38. package/lib/i18n.d.ts +2 -0
  39. package/lib/i18n.js +53 -0
  40. package/lib/i18n.js.map +1 -0
  41. package/lib/mutex.d.ts +2 -1
  42. package/lib/mutex.js +2 -1
  43. package/lib/mutex.js.map +1 -1
  44. package/lib/mwoffliner.lib.js +370 -222
  45. package/lib/mwoffliner.lib.js.map +1 -1
  46. package/lib/parameterList.d.ts +20 -8
  47. package/lib/parameterList.js +30 -18
  48. package/lib/parameterList.js.map +1 -1
  49. package/lib/renderers/abstract.renderer.d.ts +48 -55
  50. package/lib/renderers/abstract.renderer.js +377 -98
  51. package/lib/renderers/abstract.renderer.js.map +1 -1
  52. package/lib/renderers/action-parse.renderer.d.ts +22 -3
  53. package/lib/renderers/action-parse.renderer.js +123 -81
  54. package/lib/renderers/action-parse.renderer.js.map +1 -1
  55. package/lib/renderers/renderer.builder.js +6 -69
  56. package/lib/renderers/renderer.builder.js.map +1 -1
  57. package/lib/renderers/rendering.context.d.ts +2 -3
  58. package/lib/renderers/rendering.context.js +6 -14
  59. package/lib/renderers/rendering.context.js.map +1 -1
  60. package/lib/sanitize-argument.d.ts +7 -4
  61. package/lib/sanitize-argument.js +99 -41
  62. package/lib/sanitize-argument.js.map +1 -1
  63. package/lib/util/FileManager.d.ts +36 -0
  64. package/lib/util/FileManager.js +246 -0
  65. package/lib/util/FileManager.js.map +1 -0
  66. package/lib/util/RedisKvs.js +4 -4
  67. package/lib/util/RedisKvs.js.map +1 -1
  68. package/lib/util/RedisQueue.js +1 -1
  69. package/lib/util/RedisQueue.js.map +1 -1
  70. package/lib/util/builders/url/action-parse.director.d.ts +2 -2
  71. package/lib/util/builders/url/action-parse.director.js +7 -7
  72. package/lib/util/builders/url/action-parse.director.js.map +1 -1
  73. package/lib/util/builders/url/api.director.d.ts +2 -4
  74. package/lib/util/builders/url/api.director.js +10 -17
  75. package/lib/util/builders/url/api.director.js.map +1 -1
  76. package/lib/util/builders/url/base.director.d.ts +0 -4
  77. package/lib/util/builders/url/base.director.js +0 -25
  78. package/lib/util/builders/url/base.director.js.map +1 -1
  79. package/lib/util/builders/url/basic.director.d.ts +1 -1
  80. package/lib/util/builders/url/basic.director.js +1 -1
  81. package/lib/util/builders/url/web.director.d.ts +1 -1
  82. package/lib/util/builders/url/web.director.js +2 -2
  83. package/lib/util/builders/url/web.director.js.map +1 -1
  84. package/lib/util/categories.d.ts +5 -5
  85. package/lib/util/categories.js +175 -174
  86. package/lib/util/categories.js.map +1 -1
  87. package/lib/util/const.d.ts +4 -8
  88. package/lib/util/const.js +9 -17
  89. package/lib/util/const.js.map +1 -1
  90. package/lib/util/customCssJs.d.ts +6 -0
  91. package/lib/util/customCssJs.js +70 -0
  92. package/lib/util/customCssJs.js.map +1 -0
  93. package/lib/util/dump.d.ts +15 -4
  94. package/lib/util/dump.js +282 -86
  95. package/lib/util/dump.js.map +1 -1
  96. package/lib/util/index.d.ts +1 -1
  97. package/lib/util/index.js +1 -1
  98. package/lib/util/index.js.map +1 -1
  99. package/lib/util/metaData.js.map +1 -1
  100. package/lib/util/misc.d.ts +23 -13
  101. package/lib/util/misc.js +90 -59
  102. package/lib/util/misc.js.map +1 -1
  103. package/lib/util/mw-api.d.ts +7 -5
  104. package/lib/util/mw-api.js +167 -221
  105. package/lib/util/mw-api.js.map +1 -1
  106. package/lib/util/pageListMainPage.d.ts +3 -0
  107. package/lib/util/pageListMainPage.js +10 -0
  108. package/lib/util/pageListMainPage.js.map +1 -0
  109. package/lib/util/pages.d.ts +30 -0
  110. package/lib/util/pages.js +146 -0
  111. package/lib/util/pages.js.map +1 -0
  112. package/lib/util/rewriteUrls.d.ts +2 -2
  113. package/lib/util/rewriteUrls.js +33 -31
  114. package/lib/util/rewriteUrls.js.map +1 -1
  115. package/lib/util/savePages.d.ts +8 -0
  116. package/lib/util/savePages.js +182 -0
  117. package/lib/util/savePages.js.map +1 -0
  118. package/lib/version.d.ts +1 -1
  119. package/lib/version.js +1 -1
  120. package/lib/version.js.map +1 -1
  121. package/offliner-definition.json +178 -90
  122. package/package.json +50 -49
  123. package/res/page_list_home.js +20 -0
  124. package/res/{article_not_found.svg → page_not_found.svg} +1 -1
  125. package/res/script.js +30 -45
  126. package/res/style.css +52 -68
  127. package/res/templates/download_error_placeholder.html +7 -8
  128. package/res/templates/javaScript.html +19 -0
  129. package/res/templates/pageFallback.html +9 -7
  130. package/res/templates/pageVector2022.html +9 -7
  131. package/res/templates/pageVectorLegacy.html +9 -7
  132. package/res/templates/{article_list_home.html → page_list_home.html} +1 -2
  133. package/translation/ar.json +21 -21
  134. package/translation/bn.json +1 -1
  135. package/translation/de.json +46 -19
  136. package/translation/en.json +54 -21
  137. package/translation/es.json +17 -19
  138. package/translation/fi.json +23 -3
  139. package/translation/fr.json +57 -21
  140. package/translation/he.json +8 -6
  141. package/translation/ia.json +19 -19
  142. package/translation/id.json +13 -13
  143. package/translation/it.json +13 -8
  144. package/translation/ko.json +55 -3
  145. package/translation/lb.json +5 -2
  146. package/translation/mk.json +5 -5
  147. package/translation/nl.json +57 -22
  148. package/translation/or.json +2 -2
  149. package/translation/pt.json +2 -2
  150. package/translation/qqq.json +55 -22
  151. package/translation/ru.json +2 -2
  152. package/translation/sc.json +2 -2
  153. package/translation/sk.json +62 -0
  154. package/translation/sl.json +54 -21
  155. package/translation/sv.json +20 -19
  156. package/translation/sw.json +45 -2
  157. package/translation/zh-hans.json +56 -22
  158. package/translation/zh-hant.json +26 -19
  159. package/lib/renderers/abstractDesktop.render.d.ts +0 -12
  160. package/lib/renderers/abstractDesktop.render.js +0 -67
  161. package/lib/renderers/abstractDesktop.render.js.map +0 -1
  162. package/lib/renderers/abstractMobile.render.d.ts +0 -12
  163. package/lib/renderers/abstractMobile.render.js +0 -54
  164. package/lib/renderers/abstractMobile.render.js.map +0 -1
  165. package/lib/renderers/rest-api.renderer.d.ts +0 -7
  166. package/lib/renderers/rest-api.renderer.js +0 -68
  167. package/lib/renderers/rest-api.renderer.js.map +0 -1
  168. package/lib/renderers/visual-editor.renderer.d.ts +0 -7
  169. package/lib/renderers/visual-editor.renderer.js +0 -76
  170. package/lib/renderers/visual-editor.renderer.js.map +0 -1
  171. package/lib/renderers/wikimedia-desktop.renderer.d.ts +0 -4
  172. package/lib/renderers/wikimedia-desktop.renderer.js +0 -7
  173. package/lib/renderers/wikimedia-desktop.renderer.js.map +0 -1
  174. package/lib/renderers/wikimedia-mobile.renderer.d.ts +0 -19
  175. package/lib/renderers/wikimedia-mobile.renderer.js +0 -243
  176. package/lib/renderers/wikimedia-mobile.renderer.js.map +0 -1
  177. package/lib/util/articleListMainPage.d.ts +0 -3
  178. package/lib/util/articleListMainPage.js +0 -10
  179. package/lib/util/articleListMainPage.js.map +0 -1
  180. package/lib/util/articles.d.ts +0 -26
  181. package/lib/util/articles.js +0 -102
  182. package/lib/util/articles.js.map +0 -1
  183. package/lib/util/builders/url/desktop.director.d.ts +0 -8
  184. package/lib/util/builders/url/desktop.director.js +0 -15
  185. package/lib/util/builders/url/desktop.director.js.map +0 -1
  186. package/lib/util/builders/url/mobile.director.d.ts +0 -8
  187. package/lib/util/builders/url/mobile.director.js +0 -15
  188. package/lib/util/builders/url/mobile.director.js.map +0 -1
  189. package/lib/util/builders/url/rest-api.director.d.ts +0 -8
  190. package/lib/util/builders/url/rest-api.director.js +0 -18
  191. package/lib/util/builders/url/rest-api.director.js.map +0 -1
  192. package/lib/util/builders/url/visual-editor.director.d.ts +0 -9
  193. package/lib/util/builders/url/visual-editor.director.js +0 -18
  194. package/lib/util/builders/url/visual-editor.director.js.map +0 -1
  195. package/lib/util/saveArticles.d.ts +0 -8
  196. package/lib/util/saveArticles.js +0 -411
  197. package/lib/util/saveArticles.js.map +0 -1
  198. package/res/article_list_home.js +0 -24
  199. package/res/content.parsoid.css +0 -196
  200. package/res/inserted_style.css +0 -45
  201. package/res/templates/categories.html +0 -11
  202. package/res/templates/lead_section_wrapper.html +0 -6
  203. package/res/templates/pageWikimediaDesktop.html +0 -24
  204. package/res/templates/pageWikimediaMobile.html +0 -24
  205. package/res/templates/section_wrapper.html +0 -5
  206. package/res/templates/subcategories.html +0 -23
  207. package/res/templates/subpages.html +0 -17
  208. package/res/templates/subsection_wrapper.html +0 -5
  209. package/res/webpHandler.js +0 -165
  210. package/res/wm_mobile_override_script.js +0 -15
  211. package/res/wm_mobile_override_style.css +0 -20
  212. package/translation/br.json +0 -8
  213. package/translation/dag.json +0 -9
  214. package/translation/es-formal.json +0 -29
  215. package/translation/ha.json +0 -10
  216. package/translation/hi.json +0 -9
  217. package/translation/ig.json +0 -10
  218. package/translation/kaa.json +0 -9
  219. package/translation/nb.json +0 -9
  220. package/translation/nqo.json +0 -8
  221. package/translation/pt-br.json +0 -10
  222. package/translation/ro.json +0 -9
  223. package/translation/scn.json +0 -8
  224. package/translation/sq.json +0 -9
  225. package/translation/te.json +0 -9
  226. package/translation/tn.json +0 -9
  227. package/translation/tr.json +0 -10
@@ -1,244 +1,165 @@
1
1
  import pmap from 'p-map';
2
- import deepmerge from 'deepmerge';
3
2
  import * as logger from '../Logger.js';
4
3
  import Downloader from '../Downloader.js';
5
4
  import Timer from './Timer.js';
6
5
  import RedisStore from '../RedisStore.js';
7
6
  import MediaWiki from '../MediaWiki.js';
8
- import { REDIRECT_PAGE_SIGNATURE } from './const.js';
9
7
  import { cleanupAxiosError } from './misc.js';
10
- export async function getArticlesByIds(articleIds, log = true) {
8
+ const MAX_TITLES_PARAM_SIZE = 7400;
9
+ const MAX_BATCH_SIZE = 50;
10
+ export function trimPageTitlesBatch(pageTitles) {
11
+ const batch = pageTitles.slice(0, MAX_BATCH_SIZE);
12
+ while (batch.length > 1 && new URLSearchParams({ titles: batch.join('|') }).toString().length > MAX_TITLES_PARAM_SIZE) {
13
+ batch.pop();
14
+ }
15
+ return batch;
16
+ }
17
+ export async function getPagesByTitle(pageTitles, purpose, pagesToIgnore, allowedContentModels = ['wikitext'], categoryTitles = new Set()) {
11
18
  let from = 0;
12
19
  let numThumbnails = 0;
13
- const MAX_BATCH_SIZE = 50;
14
- const MAX_URL_SIZE = 7900; // in bytes, approx.
15
- const { articleDetailXId, redirectsXId } = RedisStore;
16
20
  // using async iterator to spawn workers
17
21
  await pmap(','
18
- .repeat(Downloader.speed)
22
+ .repeat(Downloader.workers)
19
23
  .split(',')
20
24
  .map((_, i) => i), async (workerId) => {
21
- while (from < articleIds.length) {
22
- // Secure the request has the max articleIds as possible (within boudaries)
23
- const articleIdsBatch = articleIds.slice(from, from + MAX_BATCH_SIZE);
24
- let urlSize = encodeURIComponent(articleIdsBatch.join('|')).length;
25
- while (urlSize > MAX_URL_SIZE) {
26
- urlSize -= encodeURIComponent(articleIdsBatch.pop()).length + 1;
27
- }
28
- // Udpate articleIds slicing boundaries
29
- const to = from + articleIdsBatch.length;
30
- if (log) {
31
- const progressPercent = Math.floor((to / articleIds.length) * 100);
32
- logger.log(`Worker [${workerId}] getting article range [${from}-${to}] of [${articleIds.length}] [${progressPercent}%]`);
33
- }
25
+ while (from < pageTitles.length) {
26
+ const pageTitlesBatch = trimPageTitlesBatch(pageTitles.slice(from));
27
+ const to = from + pageTitlesBatch.length;
28
+ const progressPercent = Math.floor((to / pageTitles.length) * 100);
29
+ logger.info(`Worker [${workerId}] getting page range [${from}-${to}] of [${pageTitles.length}] for ${purpose} [${progressPercent}%]`);
34
30
  from = to;
35
31
  // Nothing to do
36
- if (!articleIdsBatch.length) {
32
+ if (!pageTitlesBatch.length) {
37
33
  continue;
38
34
  }
39
35
  // Retrieve the details and save them in Redis
40
- const allArticleDetails = await Downloader.getArticleDetailsIds(articleIdsBatch, numThumbnails < 100);
41
- // Filter articles without revisions (#2091)
42
- const articlesIgnored = Object.values(allArticleDetails)
43
- .filter((a) => !a.revisions)
44
- .map((article) => article.title);
45
- if (articlesIgnored.length > 0) {
46
- logger.warn(`Ignoring articles without revisions: ${articlesIgnored.join(', ')}`);
47
- }
48
- const mwArticleDetails = Object.fromEntries(Object.entries(allArticleDetails).filter(([, articleDetail]) => !articlesIgnored.includes(articleDetail.title)));
49
- const articlesWithThumbnail = Object.values(mwArticleDetails).filter((a) => !!a.thumbnail);
50
- numThumbnails += articlesWithThumbnail.length;
51
- const articleDetails = mwRetToArticleDetail(mwArticleDetails);
52
- for (const [articleId, articleDetail] of Object.entries(mwArticleDetails)) {
53
- if (articleDetail.redirects && articleDetail.redirects.length) {
54
- await redirectsXId.setMany(articleDetail.redirects.reduce((acc, redirect) => {
55
- acc[redirect.title] = { title: redirect.title, targetId: articleId, fragment: redirect.fragment || '' };
56
- return acc;
57
- }, {}));
58
- }
59
- }
60
- const keys = Object.keys(articleDetails);
61
- if (keys.length == 0) {
62
- return;
63
- }
64
- const existingArticleDetails = await articleDetailXId.getMany(keys);
65
- await articleDetailXId.setMany(deepmerge(existingArticleDetails, articleDetails));
36
+ const allMwPages = await Downloader.getPagesByTitle(pageTitlesBatch, numThumbnails < 100, purpose !== 'categories');
37
+ const { numThumbnails: iterThumbnails } = await processPagesAndSaveToRedis(allMwPages, pagesToIgnore, allowedContentModels, categoryTitles);
38
+ numThumbnails += iterThumbnails;
66
39
  }
67
- }, { concurrency: Downloader.speed });
40
+ }, { concurrency: Downloader.workers });
68
41
  }
69
- async function saveToStore(articleDetails, redirects, articleDetailXId, redirectsXId) {
70
- try {
71
- const [numArticles] = await Promise.all([articleDetailXId.setMany(articleDetails), redirectsXId.setMany(redirects)]);
72
- return [numArticles];
42
+ export function filterPages(pages, pagesToIgnore, allowedContentModels) {
43
+ function revisionFilter(page) {
44
+ return !page.revisions;
73
45
  }
74
- catch (err) {
75
- return [0, err];
46
+ function contentModelFilter(page) {
47
+ return !allowedContentModels.includes(page.contentmodel);
48
+ }
49
+ function ignoredPagesFilter(page) {
50
+ return pagesToIgnore.includes(page.title) || pagesToIgnore.includes(page.title.replace(/ /g, '_'));
51
+ }
52
+ // Filter pages without revisions (#2091)
53
+ const revisionIssues = pages.filter(revisionFilter);
54
+ if (revisionIssues.length > 0) {
55
+ logger.debug(`Ignoring pages without revisions: ${revisionIssues.join(', ')}`);
76
56
  }
57
+ // Filter pages with unexpected content model (#2445)
58
+ const contentModelIssues = pages.filter(contentModelFilter);
59
+ if (contentModelIssues.length > 0) {
60
+ logger.debug(`Ignoring pages with unexpected content model: ${contentModelIssues.join(', ')}`);
61
+ }
62
+ // Filter pages asked for by user
63
+ const ignoreListIssues = pages.filter(ignoredPagesFilter);
64
+ if (ignoreListIssues.length > 0) {
65
+ logger.debug(`Ignoring pages in list: ${ignoreListIssues.join(', ')}`);
66
+ }
67
+ return pages
68
+ .filter((p) => !revisionFilter(p))
69
+ .filter((p) => !contentModelFilter(p))
70
+ .filter((p) => !ignoredPagesFilter(p));
77
71
  }
78
- export function getArticlesByNS(ns, articleIdsToIgnore, continueLimit) {
79
- // eslint-disable-next-line no-async-promise-executor
80
- return new Promise(async (resolve, reject) => {
81
- let totalArticles = 0;
82
- let chunk;
83
- const { articleDetailXId, redirectsXId } = RedisStore;
84
- const saveStorePromisQueue = [];
85
- /*
86
- * timer to detect freezes
87
- */
88
- const stages = ['Download ArticleDetails', 'Check Redirects', 'Store ArticleDetails in Redis', 'Clean Up left Promises'];
89
- let curStage = 0;
90
- // We don't really know how long this is going to take because we have a query continuation parameter which might induce
91
- // more request to make + we also get categories with more requests which is a recursive call
92
- const timeout = Math.max(Downloader.requestTimeout * 2, 10 * 60 * 1000);
93
- const timer = new Timer(() => {
94
- const errorMessage = `Worker timed out after ${timeout} ms at ${stages[curStage]}`;
95
- logger.error(errorMessage);
96
- reject(new Error(errorMessage));
97
- }, timeout);
98
- try {
99
- /*
100
- * fetch article details in chunks and store them in redis
101
- */
102
- do {
103
- timer.reset();
104
- curStage = 0;
105
- chunk = await Downloader.getArticleDetailsNS(ns, chunk && chunk.gapContinue);
106
- // Filter articles without revisions (#2238)
107
- const newArticlesToIgnore = Object.values(chunk.articleDetails)
108
- .filter((a) => !a.revisions)
109
- .map((article) => article.title);
110
- if (newArticlesToIgnore.length > 0) {
111
- logger.warn(`Ignoring articles without revisions: ${newArticlesToIgnore.join(', ')}`);
112
- if (!articleIdsToIgnore) {
113
- articleIdsToIgnore = [];
114
- }
115
- articleIdsToIgnore.push(...newArticlesToIgnore);
116
- }
117
- if (articleIdsToIgnore) {
118
- Object.keys(chunk.articleDetails).forEach((articleId) => {
119
- const articleTitle = chunk.articleDetails[articleId].title;
120
- if (articleIdsToIgnore.includes(articleTitle)) {
121
- delete chunk.articleDetails[articleId];
122
- logger.info(`Excluded article ${articleTitle}`);
123
- }
72
+ async function processPagesAndSaveToRedis(pages, pagesToIgnore, allowedContentModels, categoryTitles = new Set()) {
73
+ const redirects = [];
74
+ const pagesToRemove = [];
75
+ const filteredPages = filterPages(pages, pagesToIgnore, allowedContentModels);
76
+ for (const page of filteredPages) {
77
+ page.categories?.forEach((category) => {
78
+ if (!category.hidden)
79
+ categoryTitles.add(category.title);
80
+ });
81
+ if (page.redirects) {
82
+ for (const target of page.redirects) {
83
+ const targetExistsAsPage = (await RedisStore.pagesStore.exists(target.title)) || filteredPages.filter((a) => a.title == target.title).length > 0;
84
+ if (targetExistsAsPage) {
85
+ logger.warn(`Page '${target.title}' found in redirects of '${page.title}' while it is also listed among titles to fetch ; scraper will automatically recover from this edge case`);
86
+ redirects.push({
87
+ from: page.title,
88
+ to: target.title,
124
89
  });
90
+ pagesToRemove.push(page);
125
91
  }
126
- curStage += 1;
127
- const redirects = {};
128
- for (const [articleId, articleDetail] of Object.entries(chunk.articleDetails)) {
129
- if (articleDetail.redirects) {
130
- for (const target of articleDetail.redirects) {
131
- const targetExistsAsArticle = (await RedisStore.articleDetailXId.exists(target.title)) || Object.keys(chunk.articleDetails).includes(target.title);
132
- if (targetExistsAsArticle) {
133
- logger.warn(`Article '${target.title}' found in redirects of '${articleId}' while it is also listed among articles to fetch ; scraper will automatically recover from this edge case`);
134
- redirects[articleId] = {
135
- targetId: target.title,
136
- title: articleId,
137
- };
138
- delete chunk.articleDetails[articleId];
139
- }
140
- else {
141
- redirects[target.title] = {
142
- targetId: articleId,
143
- title: target.title,
144
- fragment: target.fragment,
145
- };
146
- }
147
- }
148
- }
149
- }
150
- curStage += 1;
151
- /*
152
- * Don't await redis, push promise into array queue
153
- * and check if the oldest promise finished in the meantime.
154
- * Like this we can store in redis simultaniously to downloading the
155
- * next articles, but also won't run into an ever increasing backlog.
156
- */
157
- const newSavePromise = saveToStore(mwRetToArticleDetail(chunk.articleDetails), redirects, articleDetailXId, redirectsXId);
158
- if (saveStorePromisQueue.length) {
159
- /*
160
- * in normal circumstances, where downloading is slower than storing,
161
- * this promise will always be resolved here already
162
- */
163
- const [numArticles, err] = await saveStorePromisQueue.shift();
164
- if (err) {
165
- timer.clear();
166
- reject(err);
167
- return;
168
- }
169
- totalArticles += numArticles;
170
- logger.log(`Got [${numArticles} / ${totalArticles}] articles chunk from namespace [${ns}]`);
92
+ else {
93
+ redirects.push({
94
+ from: target.title,
95
+ to: page.title,
96
+ fragment: target.fragment,
97
+ });
171
98
  }
172
- saveStorePromisQueue.push(newSavePromise);
173
- // Only for testing purposes
174
- if (--continueLimit < 0)
175
- break;
176
- } while (chunk.gapContinue);
177
- /*
178
- * clear up potentially still pending promises
179
- */
180
- curStage = 3;
181
- const lastPending = await Promise.all(saveStorePromisQueue);
182
- const errored = lastPending.find(([, err]) => err);
183
- if (errored) {
184
- throw errored[1];
185
99
  }
186
- totalArticles += lastPending.reduce((a, [b]) => a + b, 0);
187
100
  }
188
- catch (err) {
189
- logger.error(`Error fetching article details at ${stages[curStage]}`);
190
- reject(err);
191
- return;
192
- }
193
- finally {
194
- timer.clear();
195
- }
196
- logger.log(`A total of [${totalArticles}] articles has been found in namespace [${ns}]`);
197
- resolve();
198
- });
199
- }
200
- export function normalizeMwResponse(response) {
201
- if (!response) {
202
- return {};
203
101
  }
204
- const { normalized: _normalized, pages } = response;
205
- const normalized = (_normalized || []).reduce((acc, { from, to }) => {
206
- acc[to] = from;
102
+ for (const page of pagesToRemove) {
103
+ filteredPages.splice(filteredPages.indexOf(page), 1);
104
+ }
105
+ const numPages = await RedisStore.pagesStore.setMany(mwRetToPageDetail(filteredPages));
106
+ const numRedirects = await RedisStore.redirectsStore.setMany(redirects.reduce((acc, redirect) => {
107
+ acc[redirect.from] = { from: redirect.from, to: redirect.to, fragment: redirect.fragment || '' };
207
108
  return acc;
208
- }, {});
209
- return Object.values(pages).reduce((acc, page) => {
210
- const id = normalized.hasOwnProperty(page.title) ? normalized[page.title] : page.title || ''; // eslint-disable-line no-prototype-builtins
211
- if (typeof id !== 'string' || !id) {
212
- logger.warn(`Article Id is invalid - expected a string but got [${id}], converting to string and continuing`);
213
- }
214
- const articleId = String(id).replace(/ /g, '_');
215
- if (page.redirects) {
216
- page.redirects = page.redirects
217
- // drop redirects from talk (not subject) namespaces and from User namespace, except if namespace has been expressly requested
218
- .filter((redirect) => (redirect.ns % 2 === 0 && redirect.ns !== 2) || MediaWiki.namespacesToMirror.some((ns) => MediaWiki.namespaces[ns].num === redirect.ns))
219
- .map((redirect) => {
220
- // The API returns the redirect title (!?), we fake the
221
- // redirectId by putting the underscore. That way we
222
- // secure the URL rewritting works fine.
223
- redirect.title = String(redirect.title).replace(/ /g, '_');
224
- return redirect;
225
- });
226
- }
227
- if (articleId) {
228
- return {
229
- ...acc,
230
- [articleId]: page,
231
- };
232
- }
233
- else {
234
- return acc;
235
- }
236
- }, {});
109
+ }, {}));
110
+ const numThumbnails = filteredPages.filter((page) => !!page.thumbnail).length;
111
+ return { numPages, numRedirects, numThumbnails };
237
112
  }
238
- export function mwRetToArticleDetail(obj) {
113
+ export async function getPagesByNamespace(namespace, pagesToIgnore, allowedContentModels = ['wikitext'], categoryTitles = new Set(), continueLimit) {
114
+ let totalPages = 0;
115
+ let gapContinue = '';
116
+ const seenGapContinueValues = [];
117
+ // We don't really know how long this is going to take because we have a query continuation parameter which might induce
118
+ // more request to make + we also get categories with more requests which is a recursive call
119
+ const timeout = Math.max(Downloader.requestTimeout * 2, 10 * 60 * 1000);
120
+ const timer = new Timer(() => {
121
+ const errorMessage = `Worker timed out after ${timeout} ms`;
122
+ logger.error(errorMessage);
123
+ throw new Error(errorMessage);
124
+ }, timeout);
125
+ try {
126
+ /*
127
+ * fetch page details in chunks and store them in redis
128
+ */
129
+ do {
130
+ timer.reset();
131
+ const resp = await Downloader.getPagesByNamespace(namespace, gapContinue);
132
+ gapContinue = resp.gapContinue;
133
+ if (gapContinue) {
134
+ if (seenGapContinueValues.includes(gapContinue)) {
135
+ throw new Error(`Detected continuation cycle while fetching pages in namespace ${namespace}. ` +
136
+ `Repeated gapContinue=${gapContinue} after visiting: [${seenGapContinueValues.join(', ')}]`);
137
+ }
138
+ seenGapContinueValues.push(gapContinue);
139
+ }
140
+ const { numPages } = await processPagesAndSaveToRedis(resp.pages, pagesToIgnore, allowedContentModels, categoryTitles);
141
+ totalPages += numPages;
142
+ logger.info(`Got [${numPages} / ${totalPages}] pages chunk from namespace [${namespace}]`);
143
+ // Only for testing purposes
144
+ if (--continueLimit < 0)
145
+ break;
146
+ } while (gapContinue);
147
+ }
148
+ finally {
149
+ timer.clear();
150
+ }
151
+ logger.info(`A total of [${totalPages}] pages has been found in namespace [${namespace}]`);
152
+ }
153
+ export function filterRedirects(mwPage) {
154
+ if (mwPage.redirects) {
155
+ mwPage.redirects = mwPage.redirects
156
+ // keep redirects only to current page namespace + other content namespaces
157
+ .filter((redirect) => redirect.ns == mwPage.ns || MediaWiki.namespacesToMirror.some((ns) => MediaWiki.namespaces[ns].num === redirect.ns));
158
+ }
159
+ }
160
+ export function mwRetToPageDetail(pages) {
239
161
  const ret = {};
240
- for (const key of Object.keys(obj)) {
241
- const val = obj[key];
162
+ for (const val of pages) {
242
163
  const rev = val.revisions && val.revisions[0];
243
164
  const geo = val.coordinates && val.coordinates[0];
244
165
  let newThumbnail;
@@ -249,17 +170,29 @@ export function mwRetToArticleDetail(obj) {
249
170
  source: val.thumbnail.source,
250
171
  };
251
172
  }
252
- ret[key] = {
173
+ let newCategoryinfo;
174
+ if (val.categoryinfo) {
175
+ newCategoryinfo = {
176
+ ...val.categoryinfo,
177
+ nogallery: val.pageprops && val.pageprops.nogallery === '',
178
+ };
179
+ }
180
+ let allCategories;
181
+ if (val.categories) {
182
+ allCategories = val.categories.map(({ title }) => title.split(':').slice(1).join(':'));
183
+ }
184
+ ret[val.title] = {
253
185
  title: val.title,
254
- categories: val.categories,
255
- subCategories: val.subCategories,
256
186
  thumbnail: newThumbnail,
187
+ categoryinfo: newCategoryinfo,
188
+ categories: allCategories,
257
189
  missing: val.missing,
258
190
  pagelang: val.pagelanguagehtmlcode,
259
191
  pagedir: val.pagelanguagedir,
260
192
  ...(val.ns !== 0 ? { ns: val.ns } : {}),
261
193
  ...(val.contentmodel !== 'wikitext' ? { contentmodel: val.contentmodel } : {}),
262
194
  ...(rev ? { revisionId: rev.revid, timestamp: rev.timestamp } : {}),
195
+ ...(val.flagged?.stable_revid ? { stableRevisionId: val.flagged.stable_revid } : {}),
263
196
  ...(geo ? { coordinates: `${geo.lat};${geo.lon}` } : {}),
264
197
  };
265
198
  }
@@ -278,7 +211,6 @@ export function mwRetToArticleDetail(obj) {
278
211
  export async function checkApiAvailability(url, allowedMimeTypes = null) {
279
212
  try {
280
213
  const resp = await Downloader.request({ url: decodeURI(url), method: 'GET', maxRedirects: 0, ...Downloader.basicRequestOptions });
281
- const isRedirectPage = typeof resp.data === 'string' && resp.data.startsWith(REDIRECT_PAGE_SIGNATURE);
282
214
  // Read https://phabricator.wikimedia.org/T359187 to understand
283
215
  // the 'mediawiki-api-error' === 'rest-permission-error' exception
284
216
  const isSuccess = resp.status === 200 && (!resp.headers['mediawiki-api-error'] || resp.headers['mediawiki-api-error'] === 'rest-permission-error');
@@ -295,24 +227,38 @@ export async function checkApiAvailability(url, allowedMimeTypes = null) {
295
227
  }
296
228
  }
297
229
  }
298
- return !isRedirectPage && isSuccess && validMimeType;
230
+ return isSuccess && validMimeType;
299
231
  }
300
232
  catch (err) {
301
- logger.info('checkApiAvailability failed: ', cleanupAxiosError(err));
233
+ logger.debug('checkApiAvailability failed: ', cleanupAxiosError(err));
302
234
  return false;
303
235
  }
304
236
  }
305
- export async function getArticleIds(mainPage, articleIds, articleIdsToIgnore) {
237
+ export async function getPages(mainPage, pages = [], pagesToIgnore = [], allowedContentModels = ['wikitext']) {
238
+ const categorySet = new Set();
306
239
  if (mainPage) {
307
- await getArticlesByIds([mainPage]);
240
+ await getPagesByTitle([mainPage], 'mainPage', pagesToIgnore, allowedContentModels, categorySet);
308
241
  }
309
- if (articleIds) {
310
- await getArticlesByIds(articleIds);
242
+ if (pages.length) {
243
+ await getPagesByTitle(pages, 'pages', pagesToIgnore, allowedContentModels, categorySet);
311
244
  }
312
245
  else {
313
246
  await pmap(MediaWiki.namespacesToMirror, (namespace) => {
314
- return getArticlesByNS(MediaWiki.namespaces[namespace].num, articleIdsToIgnore);
315
- }, { concurrency: Downloader.speed });
247
+ return getPagesByNamespace(MediaWiki.namespaces[namespace].num, pagesToIgnore, allowedContentModels, categorySet);
248
+ }, { concurrency: Downloader.workers });
249
+ }
250
+ if (MediaWiki.getCategories) {
251
+ // Categories can themselves belong to parent categories (subcategories), so we need to walk up the
252
+ // category tree until no new (unprocessed) category is discovered, to also capture root categories
253
+ const processedCategories = new Set();
254
+ let categoriesToFetch = pagesToIgnore ? [...categorySet].filter((title) => !pagesToIgnore.includes(title)) : [...categorySet];
255
+ while (categoriesToFetch.length) {
256
+ categoriesToFetch.forEach((title) => processedCategories.add(title));
257
+ logger.debug(`Fetching category details of ${categoriesToFetch.join(', ')}`);
258
+ const newCategorySet = new Set();
259
+ await getPagesByTitle(categoriesToFetch, 'categories', pagesToIgnore, allowedContentModels, newCategorySet);
260
+ categoriesToFetch = [...newCategorySet].filter((title) => !processedCategories.has(title) && !pagesToIgnore.includes(title));
261
+ }
316
262
  }
317
263
  }
318
264
  //# sourceMappingURL=mw-api.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"mw-api.js","sourceRoot":"","sources":["../../src/util/mw-api.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,SAAS,MAAM,WAAW,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AACtC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,SAAS,MAAM,iBAAiB,CAAA;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,YAAY,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,UAAoB,EAAE,GAAG,GAAG,IAAI;IACrE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,cAAc,GAAG,EAAE,CAAA;IACzB,MAAM,YAAY,GAAG,IAAI,CAAA,CAAC,oBAAoB;IAE9C,MAAM,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,UAAU,CAAA;IAErD,wCAAwC;IACxC,MAAM,IAAI,CACR,GAAG;SACA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;SACxB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACnB,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAChC,2EAA2E;YAC3E,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,GAAG,cAAc,CAAC,CAAA;YACrE,IAAI,OAAO,GAAG,kBAAkB,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;YAClE,OAAO,OAAO,GAAG,YAAY,EAAE,CAAC;gBAC9B,OAAO,IAAI,kBAAkB,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;YACjE,CAAC;YAED,uCAAuC;YACvC,MAAM,EAAE,GAAG,IAAI,GAAG,eAAe,CAAC,MAAM,CAAA;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;gBAClE,MAAM,CAAC,GAAG,CAAC,WAAW,QAAQ,4BAA4B,IAAI,IAAI,EAAE,SAAS,UAAU,CAAC,MAAM,MAAM,eAAe,IAAI,CAAC,CAAA;YAC1H,CAAC;YACD,IAAI,GAAG,EAAE,CAAA;YAET,gBAAgB;YAChB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC5B,SAAQ;YACV,CAAC;YAED,8CAA8C;YAC9C,MAAM,iBAAiB,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,eAAe,EAAE,aAAa,GAAG,GAAG,CAAC,CAAA;YAErG,4CAA4C;YAC5C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC;iBACrD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;iBAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAClC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,wCAAwC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnF,CAAC;YACD,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAE5J,MAAM,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;YAC1F,aAAa,IAAI,qBAAqB,CAAC,MAAM,CAAA;YAE7C,MAAM,cAAc,GAAG,oBAAoB,CAAC,gBAAgB,CAAC,CAAA;YAE7D,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC1E,IAAI,aAAa,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;oBAC9D,MAAM,YAAY,CAAC,OAAO,CACxB,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC/C,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAA;wBACvG,OAAO,GAAG,CAAA;oBACZ,CAAC,EAAE,EAAE,CAAC,CACP,CAAA;gBACH,CAAC;YACH,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YACxC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,OAAM;YACR,CAAC;YACD,MAAM,sBAAsB,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACnE,MAAM,gBAAgB,CAAC,OAAO,CAAC,SAAS,CAAC,sBAAsB,EAAE,cAAc,CAAC,CAAC,CAAA;QACnF,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,EAAE,CAClC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,cAAkC,EAClC,SAA+B,EAC/B,gBAAqC,EACrC,YAAmC;IAEnC,IAAI,CAAC;QACH,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACpH,OAAO,CAAC,WAAW,CAAC,CAAA;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;IACjB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU,EAAE,kBAA6B,EAAE,aAAsB;IAC/F,qDAAqD;IACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,IAAI,aAAa,GAAG,CAAC,CAAA;QACrB,IAAI,KAA0D,CAAA;QAE9D,MAAM,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,UAAU,CAAA;QAErD,MAAM,oBAAoB,GAAgC,EAAE,CAAA;QAE5D;;WAEG;QACH,MAAM,MAAM,GAAG,CAAC,yBAAyB,EAAE,iBAAiB,EAAE,+BAA+B,EAAE,wBAAwB,CAAC,CAAA;QACxH,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,wHAAwH;QACxH,6FAA6F;QAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;QACvE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,YAAY,GAAG,0BAA0B,OAAO,UAAU,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;YAClF,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAA;QACjC,CAAC,EAAE,OAAO,CAAC,CAAA;QAEX,IAAI,CAAC;YACH;;eAEG;YACH,GAAG,CAAC;gBACF,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,QAAQ,GAAG,CAAC,CAAA;gBACZ,KAAK,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,CAAA;gBAE5E,4CAA4C;gBAC5C,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC;qBAC5D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;qBAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAClC,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC,wCAAwC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;oBACrF,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBACxB,kBAAkB,GAAG,EAAE,CAAA;oBACzB,CAAC;oBACD,kBAAkB,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,CAAA;gBACjD,CAAC;gBAED,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;wBACtD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,KAAK,CAAA;wBAC1D,IAAI,kBAAkB,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC9C,OAAO,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;4BACtC,MAAM,CAAC,IAAI,CAAC,oBAAoB,YAAY,EAAE,CAAC,CAAA;wBACjD,CAAC;oBACH,CAAC,CAAC,CAAA;gBACJ,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBACb,MAAM,SAAS,GAAyB,EAAE,CAAA;gBAC1C,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC9E,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;wBAC5B,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;4BAC7C,MAAM,qBAAqB,GAAG,CAAC,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;4BAClJ,IAAI,qBAAqB,EAAE,CAAC;gCAC1B,MAAM,CAAC,IAAI,CACT,YAAY,MAAM,CAAC,KAAK,4BAA4B,SAAS,4GAA4G,CAC1K,CAAA;gCACD,SAAS,CAAC,SAAS,CAAC,GAAG;oCACrB,QAAQ,EAAE,MAAM,CAAC,KAAK;oCACtB,KAAK,EAAE,SAAS;iCACjB,CAAA;gCACD,OAAO,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;4BACxC,CAAC;iCAAM,CAAC;gCACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG;oCACxB,QAAQ,EAAE,SAAS;oCACnB,KAAK,EAAE,MAAM,CAAC,KAAK;oCACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iCAC1B,CAAA;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBACb;;;;;mBAKG;gBACH,MAAM,cAAc,GAAG,WAAW,CAAC,oBAAoB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAA;gBAEzH,IAAI,oBAAoB,CAAC,MAAM,EAAE,CAAC;oBAChC;;;uBAGG;oBACH,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAA;oBAC7D,IAAI,GAAG,EAAE,CAAC;wBACR,KAAK,CAAC,KAAK,EAAE,CAAA;wBACb,MAAM,CAAC,GAAG,CAAC,CAAA;wBACX,OAAM;oBACR,CAAC;oBACD,aAAa,IAAI,WAAW,CAAA;oBAC5B,MAAM,CAAC,GAAG,CAAC,QAAQ,WAAW,MAAM,aAAa,oCAAoC,EAAE,GAAG,CAAC,CAAA;gBAC7F,CAAC;gBAED,oBAAoB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;gBAEzC,4BAA4B;gBAC5B,IAAI,EAAG,aAAwB,GAAG,CAAC;oBAAE,MAAK;YAC5C,CAAC,QAAQ,KAAK,CAAC,WAAW,EAAC;YAE3B;;eAEG;YACH,QAAQ,GAAG,CAAC,CAAA;YACZ,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;YAC3D,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAA;YAClD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;YAClB,CAAC;YACD,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,qCAAqC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrE,MAAM,CAAC,GAAG,CAAC,CAAA;YACX,OAAM;QACR,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACf,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,eAAe,aAAa,2CAA2C,EAAE,GAAG,CAAC,CAAA;QACxF,OAAO,EAAE,CAAA;IACX,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA4B;IAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,CAAA;IACX,CAAC;IACD,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAA;IAEnD,MAAM,UAAU,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE;QACvE,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAA;QACd,OAAO,GAAG,CAAA;IACZ,CAAC,EAAE,EAAE,CAAC,CAAA;IAEN,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC/C,MAAM,EAAE,GAAG,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA,CAAC,4CAA4C;QACzI,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE,wCAAwC,CAAC,CAAA;QAC/G,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAC/C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;gBAC7B,8HAA8H;iBAC7H,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC;iBAC7J,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAChB,uDAAuD;gBACvD,oDAAoD;gBACpD,wCAAwC;gBACxC,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBAE1D,OAAO,QAAQ,CAAA;YACjB,CAAC,CAAC,CAAA;QACN,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,GAAG,GAAG;gBACN,CAAC,SAAS,CAAC,EAAE,IAAI;aAClB,CAAA;QACH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,CAAA;QACZ,CAAC;IACH,CAAC,EAAE,EAAE,CAAC,CAAA;AACR,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAAe;IAClD,MAAM,GAAG,GAAuB,EAAE,CAAA;IAClC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QACjD,IAAI,YAAY,CAAA;QAChB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,YAAY,GAAG;gBACb,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK;gBAC1B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;gBAC5B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;aAC7B,CAAA;QACH,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG;YACT,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,SAAS,EAAE,YAAY;YACvB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,oBAAoB;YAClC,OAAO,EAAE,GAAG,CAAC,eAAe;YAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAA;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,gBAAgB,GAAG,IAAI;IAC7E,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAA;QAEjI,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAA;QAErG,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,uBAAuB,CAAC,CAAA;QAElJ,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,wDAAwD;YACxD,aAAa,GAAG,IAAI,CAAA;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,aAAa,GAAG,IAAI,CAAA;oBACpB,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,CAAC,cAAc,IAAI,SAAS,IAAI,aAAa,CAAA;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAA;QACpE,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAiB,EAAE,UAAqB,EAAE,kBAA6B;IACzG,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,gBAAgB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IACpC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAA;IACpC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,CACR,SAAS,CAAC,kBAAkB,EAC5B,CAAC,SAAiB,EAAE,EAAE;YACpB,OAAO,eAAe,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAA;QACjF,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,EAAE,CAClC,CAAA;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"mw-api.js","sourceRoot":"","sources":["../../src/util/mw-api.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AACtC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,SAAS,MAAM,iBAAiB,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAE7C,MAAM,qBAAqB,GAAG,IAAI,CAAA;AAClC,MAAM,cAAc,GAAG,EAAE,CAAA;AAEzB,MAAM,UAAU,mBAAmB,CAAC,UAAuB;IACzD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;IACjD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;QACtH,KAAK,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAuB,EACvB,OAAe,EACf,aAA2B,EAC3B,uBAAiC,CAAC,UAAU,CAAC,EAC7C,iBAAiC,IAAI,GAAG,EAAE;IAE1C,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,wCAAwC;IACxC,MAAM,IAAI,CACR,GAAG;SACA,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;SAC1B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACnB,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,eAAe,GAAG,mBAAmB,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;YAEnE,MAAM,EAAE,GAAG,IAAI,GAAG,eAAe,CAAC,MAAM,CAAA;YACxC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAA;YAClE,MAAM,CAAC,IAAI,CAAC,WAAW,QAAQ,yBAAyB,IAAI,IAAI,EAAE,SAAS,UAAU,CAAC,MAAM,SAAS,OAAO,KAAK,eAAe,IAAI,CAAC,CAAA;YACrI,IAAI,GAAG,EAAE,CAAA;YAET,gBAAgB;YAChB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;gBAC5B,SAAQ;YACV,CAAC;YAED,8CAA8C;YAC9C,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,eAAe,CAAC,eAAe,EAAE,aAAa,GAAG,GAAG,EAAE,OAAO,KAAK,YAAY,CAAC,CAAA;YACnH,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,MAAM,0BAA0B,CAAC,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YAC3I,aAAa,IAAI,cAAc,CAAA;QACjC,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,OAAO,EAAE,CACpC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAiB,EAAE,aAA0B,EAAE,oBAA8B;IACvG,SAAS,cAAc,CAAC,IAAyB;QAC/C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAA;IACxB,CAAC;IAED,SAAS,kBAAkB,CAAC,IAAyB;QACnD,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC1D,CAAC;IAED,SAAS,kBAAkB,CAAC,IAAyB;QACnD,OAAO,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAE,IAAI,CAAC,KAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAc,CAAC,CAAA;IAC7H,CAAC;IAED,yCAAyC;IACzC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACnD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChF,CAAC;IAED,qDAAqD;IACrD,MAAM,kBAAkB,GAAG,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,iDAAiD,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChG,CAAC;IAED,iCAAiC;IACjC,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;IACzD,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,2BAA2B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACxE,CAAC;IAED,OAAO,KAAK;SACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;SACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAA;AAC1C,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,KAAiB,EAAE,aAA0B,EAAE,oBAA8B,EAAE,iBAAiC,IAAI,GAAG,EAAE;IACjK,MAAM,SAAS,GAAmB,EAAE,CAAA;IACpC,MAAM,aAAa,GAA4B,EAAE,CAAA;IACjD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,EAAE,aAAa,EAAE,oBAAoB,CAAC,CAAA;IAC7E,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACpC,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC1D,CAAC,CAAC,CAAA;QACF,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpC,MAAM,kBAAkB,GAAG,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;gBAChJ,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CACT,SAAS,MAAM,CAAC,KAAK,4BAA4B,IAAI,CAAC,KAAK,0GAA0G,CACtK,CAAA;oBACD,SAAS,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,IAAI,CAAC,KAAK;wBAChB,EAAE,EAAE,MAAM,CAAC,KAAK;qBACjB,CAAC,CAAA;oBACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC1B,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,MAAM,CAAC,KAAK;wBAClB,EAAE,EAAE,IAAI,CAAC,KAAK;wBACd,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IACtD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC,CAAA;IACtF,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,OAAO,CAC1D,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE;QACjC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAA;QAChG,OAAO,GAAG,CAAA;IACZ,CAAC,EAAE,EAAE,CAAC,CACP,CAAA;IACD,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAA;IAE7E,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,SAAiB,EACjB,aAA2B,EAC3B,uBAAiC,CAAC,UAAU,CAAC,EAC7C,iBAAiC,IAAI,GAAG,EAAE,EAC1C,aAAsB;IAEtB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,WAAW,GAAG,EAAE,CAAA;IACpB,MAAM,qBAAqB,GAAa,EAAE,CAAA;IAE1C,wHAAwH;IACxH,6FAA6F;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;IACvE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE;QAC3B,MAAM,YAAY,GAAG,0BAA0B,OAAO,KAAK,CAAA;QAC3D,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAA;IAC/B,CAAC,EAAE,OAAO,CAAC,CAAA;IAEX,IAAI,CAAC;QACH;;WAEG;QACH,GAAG,CAAC;YACF,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;YAEzE,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;YAC9B,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,qBAAqB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CACb,iEAAiE,SAAS,IAAI;wBAC5E,wBAAwB,WAAW,qBAAqB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC9F,CAAA;gBACH,CAAC;gBACD,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACzC,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,0BAA0B,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YACtH,UAAU,IAAI,QAAQ,CAAA;YACtB,MAAM,CAAC,IAAI,CAAC,QAAQ,QAAQ,MAAM,UAAU,iCAAiC,SAAS,GAAG,CAAC,CAAA;YAE1F,4BAA4B;YAC5B,IAAI,EAAG,aAAwB,GAAG,CAAC;gBAAE,MAAK;QAC5C,CAAC,QAAQ,WAAW,EAAC;IACvB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,CAAA;IACf,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,wCAAwC,SAAS,GAAG,CAAC,CAAA;AAC5F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAA2B;IACzD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS;YACjC,2EAA2E;aAC1E,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,MAAM,CAAC,EAAE,IAAI,SAAS,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9I,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,MAAM,GAAG,GAAoB,EAAE,CAAA;IAC/B,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;QACjD,IAAI,YAAY,CAAA;QAChB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,YAAY,GAAG;gBACb,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK;gBAC1B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;gBAC5B,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,MAAM;aAC7B,CAAA;QACH,CAAC;QACD,IAAI,eAAe,CAAA;QACnB,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,eAAe,GAAG;gBAChB,GAAG,GAAG,CAAC,YAAY;gBACnB,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,SAAS,KAAK,EAAE;aAC3D,CAAA;QACH,CAAC;QACD,IAAI,aAAa,CAAA;QACjB,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,aAAa,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACxF,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG;YACf,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,YAAY;YACvB,YAAY,EAAE,eAAe;YAC7B,UAAU,EAAE,aAAa;YACzB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,oBAAoB;YAClC,OAAO,EAAE,GAAG,CAAC,eAAe;YAC5B,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,YAAY,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAA;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW,EAAE,gBAAgB,GAAG,IAAI;IAC7E,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,GAAG,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAA;QAEjI,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,uBAAuB,CAAC,CAAA;QAElJ,IAAI,aAAa,GAAG,KAAK,CAAA;QACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,wDAAwD;YACxD,aAAa,GAAG,IAAI,CAAA;QACtB,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;gBACxC,IAAK,IAAI,CAAC,OAAO,CAAC,cAAc,CAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,aAAa,GAAG,IAAI,CAAA;oBACpB,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,IAAI,aAAa,CAAA;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,iBAAiB,CAAC,GAAU,CAAC,CAAC,CAAA;QAC5E,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAoB,EAAE,QAAqB,EAAE,EAAE,gBAA6B,EAAE,EAAE,uBAAiC,CAAC,UAAU,CAAC;IAC1J,MAAM,WAAW,GAAG,IAAI,GAAG,EAAa,CAAA;IAExC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,eAAe,CAAC,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;IACzF,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,CACR,SAAS,CAAC,kBAAkB,EAC5B,CAAC,SAAiB,EAAE,EAAE;YACpB,OAAO,mBAAmB,CAAC,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,aAAa,EAAE,oBAAoB,EAAE,WAAW,CAAC,CAAA;QACnH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,OAAO,EAAE,CACpC,CAAA;IACH,CAAC;IAED,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;QAC5B,mGAAmG;QACnG,mGAAmG;QACnG,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAa,CAAA;QAChD,IAAI,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,KAAgB,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAA;QAExI,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAChC,iBAAiB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,MAAM,CAAC,KAAK,CAAC,gCAAgC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC5E,MAAM,cAAc,GAAG,IAAI,GAAG,EAAa,CAAA;YAC3C,MAAM,eAAe,CAAC,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,cAAc,CAAC,CAAA;YAE3G,iBAAiB,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,KAAgB,EAAE,EAAE,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;QACzI,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Dump } from '../Dump.js';
2
+ export declare function makePageListItem(dump: Dump, page: PageDetail): string;
3
+ export declare function makePageImageTile(dump: Dump, page: PageDetail): string;
@@ -0,0 +1,10 @@
1
+ import { encodePageTitleForZimHtmlUrl } from './index.js';
2
+ export function makePageListItem(dump, page) {
3
+ return `<li><a href="${encodePageTitleForZimHtmlUrl(page.title)}">${page.title || ''}<a></li>\n`;
4
+ }
5
+ export function makePageImageTile(dump, page) {
6
+ return `<a class="item" href="${encodePageTitleForZimHtmlUrl(page.title)}">
7
+ <figure><img loading="lazy" src="${page.internalThumbnailUrl}" />
8
+ <figcaption>${page.title || ''}</figcaption></figure></a>\n`;
9
+ }
10
+ //# sourceMappingURL=pageListMainPage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pageListMainPage.js","sourceRoot":"","sources":["../../src/util/pageListMainPage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAA;AAEzD,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,IAAgB;IAC3D,OAAO,gBAAgB,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,YAAY,CAAA;AAClG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAU,EAAE,IAAgB;IAC5D,OAAO,yBAAyB,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC;+CAC3B,IAAI,CAAC,oBAAoB;0BAC9C,IAAI,CAAC,KAAK,IAAI,EAAE,8BAA8B,CAAA;AACxE,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Check if a given page title is main page of the target ZIM
3
+ */
4
+ export declare function isMainPage(pageTitle: PageTitle): boolean;
5
+ /**
6
+ * Check if a given page title or id is a subpage or not
7
+ *
8
+ * A given is a subpage if subpages are activated on its namespace and it contains a / in its title/id
9
+ */
10
+ export declare function isSubpage(pageTitleOrId: string): boolean;
11
+ /**
12
+ * For a given namespace number, get the namespace name
13
+ */
14
+ export declare function getNamespaceName(namespace: number): string;
15
+ /**
16
+ * Extract the JS config variables from its headHtml, typically returned by MW API call when fetching page content
17
+ */
18
+ export declare function extractJsConfigVars(headHtml: string, extraJsConfigVars?: KVS<any>): KVS<any>;
19
+ /**
20
+ * Extract the CSS class to apply on page <body> tag from its headHtml, typically returned by MW API call when fetching page content
21
+ */
22
+ export declare function extractBodyCssClass(headHtml: string): string;
23
+ /**
24
+ * Extract the CSS class to apply on page <html> tag from its headHtml, typically returned by MW API call when fetching page content
25
+ */
26
+ export declare function extractHtmlCssClass(headHtml: string): string;
27
+ /**
28
+ * Get the special display title for the main page
29
+ */
30
+ export declare function getMainpageTitle(): Promise<string>;