mwoffliner 1.17.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +355 -62
- package/docs/functional_architecture.md +34 -38
- package/extensions/wiktionary_fr.js +18 -17
- package/jest.config.cjs +2 -2
- package/lib/DOMUtils.js +0 -1
- package/lib/DOMUtils.js.map +1 -1
- package/lib/Downloader.d.ts +39 -31
- package/lib/Downloader.js +418 -349
- package/lib/Downloader.js.map +1 -1
- package/lib/Dump.d.ts +24 -10
- package/lib/Dump.js +95 -50
- package/lib/Dump.js.map +1 -1
- package/lib/Gadgets.d.ts +2 -1
- package/lib/Gadgets.js +7 -5
- package/lib/Gadgets.js.map +1 -1
- package/lib/Logger.d.ts +6 -6
- package/lib/Logger.js +33 -33
- package/lib/Logger.js.map +1 -1
- package/lib/MediaWiki.d.ts +14 -31
- package/lib/MediaWiki.js +119 -172
- package/lib/MediaWiki.js.map +1 -1
- package/lib/RedisStore.d.ts +3 -3
- package/lib/RedisStore.js +21 -23
- package/lib/RedisStore.js.map +1 -1
- package/lib/S3.js +1 -1
- package/lib/S3.js.map +1 -1
- package/lib/Templates.d.ts +3 -10
- package/lib/Templates.js +5 -14
- package/lib/Templates.js.map +1 -1
- package/lib/cli.js +7 -6
- package/lib/cli.js.map +1 -1
- package/lib/config.d.ts +10 -16
- package/lib/config.js +33 -55
- package/lib/config.js.map +1 -1
- package/lib/error.manager.d.ts +2 -2
- package/lib/error.manager.js +42 -49
- package/lib/error.manager.js.map +1 -1
- package/lib/i18n.d.ts +2 -0
- package/lib/i18n.js +53 -0
- package/lib/i18n.js.map +1 -0
- package/lib/mutex.d.ts +2 -1
- package/lib/mutex.js +2 -1
- package/lib/mutex.js.map +1 -1
- package/lib/mwoffliner.lib.js +370 -222
- package/lib/mwoffliner.lib.js.map +1 -1
- package/lib/parameterList.d.ts +20 -8
- package/lib/parameterList.js +30 -18
- package/lib/parameterList.js.map +1 -1
- package/lib/renderers/abstract.renderer.d.ts +48 -55
- package/lib/renderers/abstract.renderer.js +377 -98
- package/lib/renderers/abstract.renderer.js.map +1 -1
- package/lib/renderers/action-parse.renderer.d.ts +22 -3
- package/lib/renderers/action-parse.renderer.js +123 -81
- package/lib/renderers/action-parse.renderer.js.map +1 -1
- package/lib/renderers/renderer.builder.js +6 -69
- package/lib/renderers/renderer.builder.js.map +1 -1
- package/lib/renderers/rendering.context.d.ts +2 -3
- package/lib/renderers/rendering.context.js +6 -14
- package/lib/renderers/rendering.context.js.map +1 -1
- package/lib/sanitize-argument.d.ts +7 -4
- package/lib/sanitize-argument.js +99 -41
- package/lib/sanitize-argument.js.map +1 -1
- package/lib/util/FileManager.d.ts +36 -0
- package/lib/util/FileManager.js +246 -0
- package/lib/util/FileManager.js.map +1 -0
- package/lib/util/RedisKvs.js +4 -4
- package/lib/util/RedisKvs.js.map +1 -1
- package/lib/util/RedisQueue.js +1 -1
- package/lib/util/RedisQueue.js.map +1 -1
- package/lib/util/builders/url/action-parse.director.d.ts +2 -2
- package/lib/util/builders/url/action-parse.director.js +7 -7
- package/lib/util/builders/url/action-parse.director.js.map +1 -1
- package/lib/util/builders/url/api.director.d.ts +2 -4
- package/lib/util/builders/url/api.director.js +10 -17
- package/lib/util/builders/url/api.director.js.map +1 -1
- package/lib/util/builders/url/base.director.d.ts +0 -4
- package/lib/util/builders/url/base.director.js +0 -25
- package/lib/util/builders/url/base.director.js.map +1 -1
- package/lib/util/builders/url/basic.director.d.ts +1 -1
- package/lib/util/builders/url/basic.director.js +1 -1
- package/lib/util/builders/url/web.director.d.ts +1 -1
- package/lib/util/builders/url/web.director.js +2 -2
- package/lib/util/builders/url/web.director.js.map +1 -1
- package/lib/util/categories.d.ts +5 -5
- package/lib/util/categories.js +175 -174
- package/lib/util/categories.js.map +1 -1
- package/lib/util/const.d.ts +4 -8
- package/lib/util/const.js +9 -17
- package/lib/util/const.js.map +1 -1
- package/lib/util/customCssJs.d.ts +6 -0
- package/lib/util/customCssJs.js +70 -0
- package/lib/util/customCssJs.js.map +1 -0
- package/lib/util/dump.d.ts +15 -4
- package/lib/util/dump.js +282 -86
- package/lib/util/dump.js.map +1 -1
- package/lib/util/index.d.ts +1 -1
- package/lib/util/index.js +1 -1
- package/lib/util/index.js.map +1 -1
- package/lib/util/metaData.js.map +1 -1
- package/lib/util/misc.d.ts +23 -13
- package/lib/util/misc.js +90 -59
- package/lib/util/misc.js.map +1 -1
- package/lib/util/mw-api.d.ts +7 -5
- package/lib/util/mw-api.js +167 -221
- package/lib/util/mw-api.js.map +1 -1
- package/lib/util/pageListMainPage.d.ts +3 -0
- package/lib/util/pageListMainPage.js +10 -0
- package/lib/util/pageListMainPage.js.map +1 -0
- package/lib/util/pages.d.ts +30 -0
- package/lib/util/pages.js +146 -0
- package/lib/util/pages.js.map +1 -0
- package/lib/util/rewriteUrls.d.ts +2 -2
- package/lib/util/rewriteUrls.js +33 -31
- package/lib/util/rewriteUrls.js.map +1 -1
- package/lib/util/savePages.d.ts +8 -0
- package/lib/util/savePages.js +182 -0
- package/lib/util/savePages.js.map +1 -0
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/offliner-definition.json +178 -90
- package/package.json +50 -49
- package/res/page_list_home.js +20 -0
- package/res/{article_not_found.svg → page_not_found.svg} +1 -1
- package/res/script.js +30 -45
- package/res/style.css +52 -68
- package/res/templates/download_error_placeholder.html +7 -8
- package/res/templates/javaScript.html +19 -0
- package/res/templates/pageFallback.html +9 -7
- package/res/templates/pageVector2022.html +9 -7
- package/res/templates/pageVectorLegacy.html +9 -7
- package/res/templates/{article_list_home.html → page_list_home.html} +1 -2
- package/translation/ar.json +21 -21
- package/translation/bn.json +1 -1
- package/translation/de.json +46 -19
- package/translation/en.json +54 -21
- package/translation/es.json +17 -19
- package/translation/fi.json +23 -3
- package/translation/fr.json +57 -21
- package/translation/he.json +8 -6
- package/translation/ia.json +19 -19
- package/translation/id.json +13 -13
- package/translation/it.json +13 -8
- package/translation/ko.json +55 -3
- package/translation/lb.json +5 -2
- package/translation/mk.json +5 -5
- package/translation/nl.json +57 -22
- package/translation/or.json +2 -2
- package/translation/pt.json +2 -2
- package/translation/qqq.json +55 -22
- package/translation/ru.json +2 -2
- package/translation/sc.json +2 -2
- package/translation/sk.json +62 -0
- package/translation/sl.json +54 -21
- package/translation/sv.json +20 -19
- package/translation/sw.json +45 -2
- package/translation/zh-hans.json +56 -22
- package/translation/zh-hant.json +26 -19
- package/lib/renderers/abstractDesktop.render.d.ts +0 -12
- package/lib/renderers/abstractDesktop.render.js +0 -67
- package/lib/renderers/abstractDesktop.render.js.map +0 -1
- package/lib/renderers/abstractMobile.render.d.ts +0 -12
- package/lib/renderers/abstractMobile.render.js +0 -54
- package/lib/renderers/abstractMobile.render.js.map +0 -1
- package/lib/renderers/rest-api.renderer.d.ts +0 -7
- package/lib/renderers/rest-api.renderer.js +0 -68
- package/lib/renderers/rest-api.renderer.js.map +0 -1
- package/lib/renderers/visual-editor.renderer.d.ts +0 -7
- package/lib/renderers/visual-editor.renderer.js +0 -76
- package/lib/renderers/visual-editor.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-desktop.renderer.d.ts +0 -4
- package/lib/renderers/wikimedia-desktop.renderer.js +0 -7
- package/lib/renderers/wikimedia-desktop.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-mobile.renderer.d.ts +0 -19
- package/lib/renderers/wikimedia-mobile.renderer.js +0 -243
- package/lib/renderers/wikimedia-mobile.renderer.js.map +0 -1
- package/lib/util/articleListMainPage.d.ts +0 -3
- package/lib/util/articleListMainPage.js +0 -10
- package/lib/util/articleListMainPage.js.map +0 -1
- package/lib/util/articles.d.ts +0 -26
- package/lib/util/articles.js +0 -102
- package/lib/util/articles.js.map +0 -1
- package/lib/util/builders/url/desktop.director.d.ts +0 -8
- package/lib/util/builders/url/desktop.director.js +0 -15
- package/lib/util/builders/url/desktop.director.js.map +0 -1
- package/lib/util/builders/url/mobile.director.d.ts +0 -8
- package/lib/util/builders/url/mobile.director.js +0 -15
- package/lib/util/builders/url/mobile.director.js.map +0 -1
- package/lib/util/builders/url/rest-api.director.d.ts +0 -8
- package/lib/util/builders/url/rest-api.director.js +0 -18
- package/lib/util/builders/url/rest-api.director.js.map +0 -1
- package/lib/util/builders/url/visual-editor.director.d.ts +0 -9
- package/lib/util/builders/url/visual-editor.director.js +0 -18
- package/lib/util/builders/url/visual-editor.director.js.map +0 -1
- package/lib/util/saveArticles.d.ts +0 -8
- package/lib/util/saveArticles.js +0 -411
- package/lib/util/saveArticles.js.map +0 -1
- package/res/article_list_home.js +0 -24
- package/res/content.parsoid.css +0 -196
- package/res/inserted_style.css +0 -45
- package/res/templates/categories.html +0 -11
- package/res/templates/lead_section_wrapper.html +0 -6
- package/res/templates/pageWikimediaDesktop.html +0 -24
- package/res/templates/pageWikimediaMobile.html +0 -24
- package/res/templates/section_wrapper.html +0 -5
- package/res/templates/subcategories.html +0 -23
- package/res/templates/subpages.html +0 -17
- package/res/templates/subsection_wrapper.html +0 -5
- package/res/webpHandler.js +0 -165
- package/res/wm_mobile_override_script.js +0 -15
- package/res/wm_mobile_override_style.css +0 -20
- package/translation/br.json +0 -8
- package/translation/dag.json +0 -9
- package/translation/es-formal.json +0 -29
- package/translation/ha.json +0 -10
- package/translation/hi.json +0 -9
- package/translation/ig.json +0 -10
- package/translation/kaa.json +0 -9
- package/translation/nb.json +0 -9
- package/translation/nqo.json +0 -8
- package/translation/pt-br.json +0 -10
- package/translation/ro.json +0 -9
- package/translation/scn.json +0 -8
- package/translation/sq.json +0 -9
- package/translation/te.json +0 -9
- package/translation/tn.json +0 -9
- package/translation/tr.json +0 -10
package/lib/util/saveArticles.js
DELETED
|
@@ -1,411 +0,0 @@
|
|
|
1
|
-
import * as logger from '../Logger.js';
|
|
2
|
-
import Downloader from '../Downloader.js';
|
|
3
|
-
import RedisStore from '../RedisStore.js';
|
|
4
|
-
import { StringItem } from '@openzim/libzim';
|
|
5
|
-
import pmap from 'p-map';
|
|
6
|
-
import * as domino from 'domino';
|
|
7
|
-
import Timer from './Timer.js';
|
|
8
|
-
import { jsPath } from './index.js';
|
|
9
|
-
import { config } from '../config.js';
|
|
10
|
-
import { getSizeFromUrl, parseRetryAfterHeader } from './misc.js';
|
|
11
|
-
import { FILES_DOWNLOAD_FAILURE_MINIMUM_FOR_CHECK, FILES_DOWNLOAD_FAILURE_TRESHOLD_PER_TEN_THOUSAND, MAX_FILE_DOWNLOAD_RETRIES } from './const.js';
|
|
12
|
-
import urlHelper from './url.helper.js';
|
|
13
|
-
import RenderingContext from '../renderers/rendering.context.js';
|
|
14
|
-
import { fileDownloadMutex, zimCreatorMutex } from '../mutex.js';
|
|
15
|
-
import { truncateUtf8Bytes } from './misc.js';
|
|
16
|
-
import { isMainPage } from './articles.js';
|
|
17
|
-
import RedisQueue from './RedisQueue.js';
|
|
18
|
-
export async function downloadFiles(fileStore, zimCreator, dump) {
|
|
19
|
-
let prevPercentProgress;
|
|
20
|
-
// create structure + Redis queue to requests hosts in a responsible manner
|
|
21
|
-
const hosts = new Map();
|
|
22
|
-
const filesTotal = await RedisStore.filesToDownloadXPath.len();
|
|
23
|
-
await RedisStore.filesToDownloadXPath.iterateItems(1, async (filesToDownload) => {
|
|
24
|
-
for (const [path, { url, mult, width, kind }] of Object.entries(filesToDownload)) {
|
|
25
|
-
const hostname = new URL(urlHelper.deserializeUrl(url)).hostname;
|
|
26
|
-
if (!hosts.has(hostname)) {
|
|
27
|
-
const filesToDownload = new RedisQueue(RedisStore.client, `${hostname}-files`);
|
|
28
|
-
RedisStore.filesQueues.push(filesToDownload);
|
|
29
|
-
filesToDownload.flush();
|
|
30
|
-
hosts.set(hostname, {
|
|
31
|
-
filesToDownload,
|
|
32
|
-
requestInterval: 10, // initial request interval is 10 ms
|
|
33
|
-
downloadSuccess: 0,
|
|
34
|
-
downloadFailure: 0,
|
|
35
|
-
downloadsComplete: false,
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
hosts.get(hostname).filesToDownload.push({
|
|
39
|
-
path: path,
|
|
40
|
-
url: url,
|
|
41
|
-
mult: mult,
|
|
42
|
-
width: width,
|
|
43
|
-
kind: kind,
|
|
44
|
-
downloadAttempts: 0,
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
});
|
|
48
|
-
await RedisStore.filesToDownloadXPath.flush();
|
|
49
|
-
/**
|
|
50
|
-
* Return next file ready to download or wait if all hosts need to make a pause.
|
|
51
|
-
* Return null when there is no more file to download.
|
|
52
|
-
*/
|
|
53
|
-
async function getNextFileToDownload() {
|
|
54
|
-
// loop until we've found a file to download or list is empty
|
|
55
|
-
while (true) {
|
|
56
|
-
// check if all donwloads have completed and exit
|
|
57
|
-
const hostValues = Array.from(hosts.values());
|
|
58
|
-
const completedHosts = hostValues.reduce((buf, host) => {
|
|
59
|
-
return host.downloadsComplete ? buf + 1 : 0;
|
|
60
|
-
}, 0);
|
|
61
|
-
if (completedHosts == hostValues.length) {
|
|
62
|
-
return null;
|
|
63
|
-
}
|
|
64
|
-
for (const [hostname, hostData] of hosts.entries()) {
|
|
65
|
-
// check conditions which leads to ignore current host
|
|
66
|
-
if (hostData.downloadsComplete ||
|
|
67
|
-
(hostData.notBeforeDate && hostData.notBeforeDate > Date.now()) ||
|
|
68
|
-
(hostData.lastRequestDate && hostData.lastRequestDate + hostData.requestInterval > Date.now())) {
|
|
69
|
-
continue;
|
|
70
|
-
}
|
|
71
|
-
// grab next item from Redis queue
|
|
72
|
-
const fileToDownload = await hostData.filesToDownload.pop();
|
|
73
|
-
if (!fileToDownload) {
|
|
74
|
-
hostData.downloadsComplete = true;
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
// modify lastRequestDate immediately so that all workers are aware
|
|
78
|
-
hostData.lastRequestDate = Date.now();
|
|
79
|
-
return { fileToDownload, hostData, hostname };
|
|
80
|
-
}
|
|
81
|
-
// pause few milliseconds, no host has something to process (just to not burn CPU)
|
|
82
|
-
await new Promise((resolve) => {
|
|
83
|
-
setTimeout(resolve, 10);
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* Really try to download one file, handling potential errors received to slow down host, stop scraper, ignore failed file
|
|
89
|
-
* @param fileToDownload information about the file to download
|
|
90
|
-
* @param hostname hostname to process
|
|
91
|
-
* @param hostData data about the host to process
|
|
92
|
-
* @param workerId ID of worker currently processing this download
|
|
93
|
-
*/
|
|
94
|
-
async function workerDownloadFile(fileToDownload, hostname, hostData, workerId) {
|
|
95
|
-
if ((dump.status.files.success + dump.status.files.fail) % (10 * Downloader.speed) === 0) {
|
|
96
|
-
const percentProgress = (((dump.status.files.success + dump.status.files.fail) / filesTotal) * 100).toFixed(1);
|
|
97
|
-
if (percentProgress !== prevPercentProgress) {
|
|
98
|
-
prevPercentProgress = percentProgress;
|
|
99
|
-
logger.log(`Progress downloading files [${dump.status.files.success + dump.status.files.fail}/${filesTotal}] [${percentProgress}%]`);
|
|
100
|
-
}
|
|
101
|
-
if (dump.status.files.fail > FILES_DOWNLOAD_FAILURE_MINIMUM_FOR_CHECK && (dump.status.files.fail * 10000) / filesTotal > FILES_DOWNLOAD_FAILURE_TRESHOLD_PER_TEN_THOUSAND) {
|
|
102
|
-
throw new Error(`Too many files failed to download: [${dump.status.files.fail}/${filesTotal}]`);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
fileToDownload.downloadAttempts += 1;
|
|
106
|
-
logger.info(`Worker ${workerId} downloading ${urlHelper.deserializeUrl(fileToDownload.url)} (${fileToDownload.kind})`);
|
|
107
|
-
await Downloader.downloadContent(fileToDownload.url, fileToDownload.kind, false)
|
|
108
|
-
.then(async (resp) => {
|
|
109
|
-
if (resp && resp.content && resp.contentType) {
|
|
110
|
-
// { FRONT_ARTICLE: 0 } is here very important, should we retrieve HTML we want to be sure the libzim will
|
|
111
|
-
// not consider it for title index
|
|
112
|
-
const item = new StringItem(fileToDownload.path, resp.contentType, null, { FRONT_ARTICLE: 0 }, resp.content);
|
|
113
|
-
await zimCreatorMutex.runExclusive(() => zimCreator.addItem(item));
|
|
114
|
-
dump.status.files.success += 1;
|
|
115
|
-
hostData.downloadSuccess += 1;
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
throw new Error(`Bad response received: ${resp}`);
|
|
119
|
-
}
|
|
120
|
-
})
|
|
121
|
-
.catch(async (err) => {
|
|
122
|
-
if (fileToDownload.downloadAttempts > MAX_FILE_DOWNLOAD_RETRIES || (err.response && err.response.status == 404)) {
|
|
123
|
-
logger.warn(`Error downloading file [${urlHelper.deserializeUrl(fileToDownload.url)}] [status=${err.response?.status}], skipping`);
|
|
124
|
-
dump.status.files.fail += 1;
|
|
125
|
-
hostData.downloadFailure += 1;
|
|
126
|
-
}
|
|
127
|
-
else {
|
|
128
|
-
if (err.response) {
|
|
129
|
-
const retryAfterHeader = err.response.headers['retry-after']?.toString();
|
|
130
|
-
if (retryAfterHeader) {
|
|
131
|
-
const retryDate = parseRetryAfterHeader(retryAfterHeader);
|
|
132
|
-
if (retryDate) {
|
|
133
|
-
hostData.notBeforeDate = retryDate;
|
|
134
|
-
logger.log(`Received a [Retry-After=${retryAfterHeader}], pausing down ${hostname} until ${hostData.notBeforeDate}`);
|
|
135
|
-
}
|
|
136
|
-
else {
|
|
137
|
-
logger.warn(`Received a [Retry-After=${retryAfterHeader}] from ${hostname} but failed to interpret it`);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
// slow down except for wikimedia thumbnails whose server is known to be lying (see https://github.com/openzim/mwoffliner/issues/2572)
|
|
142
|
-
if (err.response &&
|
|
143
|
-
[429, 503, 524].includes(err.response.status) &&
|
|
144
|
-
!urlHelper.deserializeUrl(fileToDownload.url).match(/^https?:\/\/upload\.wikimedia\.org\/.*\/thumb\//)) {
|
|
145
|
-
hostData.requestInterval = hostData.requestInterval * 1.2; // 1.2 is arbitrary value to progressively slow requests to host down
|
|
146
|
-
logger.log(`Received a [status=${err.response.status}], slowing down ${hostname} to ${hostData.requestInterval}ms interval`);
|
|
147
|
-
}
|
|
148
|
-
await hostData.filesToDownload.push(fileToDownload);
|
|
149
|
-
}
|
|
150
|
-
});
|
|
151
|
-
}
|
|
152
|
-
await pmap(Array.from({ length: Downloader.speed }, (_, i) => i), async (workerId) => {
|
|
153
|
-
while (true) {
|
|
154
|
-
// get next file to download in a Mutex (we do not want two workers trying to get next file at same time
|
|
155
|
-
// since we need to take into account limits per hostname, so this getNextFileToDownload will update
|
|
156
|
-
// data about load per host)
|
|
157
|
-
const nextFileData = await fileDownloadMutex.runExclusive(getNextFileToDownload);
|
|
158
|
-
if (!nextFileData)
|
|
159
|
-
break;
|
|
160
|
-
const { fileToDownload, hostname, hostData } = nextFileData;
|
|
161
|
-
await workerDownloadFile(fileToDownload, hostname, hostData, workerId);
|
|
162
|
-
}
|
|
163
|
-
}, { concurrency: Downloader.speed });
|
|
164
|
-
logger.log(`Done with downloading ${filesTotal} files: ${dump.status.files.success} success, ${dump.status.files.fail} fail: `, JSON.stringify(Object.fromEntries([...hosts].map(([hostname, hostData]) => [hostname, { success: hostData.downloadSuccess, fail: hostData.downloadFailure }])), null, '\t'));
|
|
165
|
-
}
|
|
166
|
-
async function getAllArticlesToKeep(articleDetailXId, dump, mainPageRenderer, articlesRenderer) {
|
|
167
|
-
await articleDetailXId.iterateItems(Downloader.speed, async (articleKeyValuePairs) => {
|
|
168
|
-
for (const [articleId, articleDetail] of Object.entries(articleKeyValuePairs)) {
|
|
169
|
-
let rets;
|
|
170
|
-
try {
|
|
171
|
-
const mainPage = isMainPage(articleId);
|
|
172
|
-
const renderer = mainPage ? mainPageRenderer : articlesRenderer;
|
|
173
|
-
const leadSectionId = dump.nodet && !articleDetail.contentmodel && !articleDetail.missing ? config.filters.leadSectionId : '';
|
|
174
|
-
const articleUrl = mainPage ? Downloader.getMainPageUrl(articleId) : Downloader.getArticleUrl(articleId, { sectionId: leadSectionId });
|
|
175
|
-
rets = await Downloader.getArticle(articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail);
|
|
176
|
-
for (const { articleId, html } of rets) {
|
|
177
|
-
if (!html) {
|
|
178
|
-
continue;
|
|
179
|
-
}
|
|
180
|
-
const doc = domino.createDocument(html);
|
|
181
|
-
if (!mainPage && !(await dump.customProcessor.shouldKeepArticle(articleId, doc))) {
|
|
182
|
-
articleDetailXId.delete(articleId);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
catch (err) {
|
|
187
|
-
logger.warn(`Error downloading article [${articleId}], skipping`, err);
|
|
188
|
-
articleDetailXId.delete(articleId);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
});
|
|
192
|
-
}
|
|
193
|
-
function flattenPromises(promisArr) {
|
|
194
|
-
return [
|
|
195
|
-
// first articleId
|
|
196
|
-
promisArr[0][0],
|
|
197
|
-
// promise resolving to first error or void
|
|
198
|
-
(async () => {
|
|
199
|
-
const resolved = await Promise.all(promisArr.map((p) => p[1]));
|
|
200
|
-
return resolved.find((err) => err);
|
|
201
|
-
})(),
|
|
202
|
-
];
|
|
203
|
-
}
|
|
204
|
-
/*
|
|
205
|
-
* Parse fetched article HTML, store files
|
|
206
|
-
* and dependencies and save in Zim
|
|
207
|
-
*/
|
|
208
|
-
async function saveArticle(zimCreator, finalHTML, mediaDependencies, imageDependencies, videoDependencies, subtitles, articleId, articleTitle) {
|
|
209
|
-
try {
|
|
210
|
-
const filesToDownload = {};
|
|
211
|
-
if (subtitles?.length > 0) {
|
|
212
|
-
subtitles.forEach((s) => {
|
|
213
|
-
filesToDownload[s.path] = { url: s.url, kind: 'subtitle' };
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
|
-
if (mediaDependencies && mediaDependencies.length) {
|
|
217
|
-
const existingVals = await RedisStore.filesToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path));
|
|
218
|
-
for (const dep of mediaDependencies) {
|
|
219
|
-
const { mult, width } = getSizeFromUrl(dep.url);
|
|
220
|
-
const existingVal = existingVals[dep.path];
|
|
221
|
-
const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1);
|
|
222
|
-
if (currentDepIsHigherRes) {
|
|
223
|
-
filesToDownload[dep.path] = {
|
|
224
|
-
url: urlHelper.serializeUrl(dep.url),
|
|
225
|
-
kind: 'media',
|
|
226
|
-
mult,
|
|
227
|
-
width,
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
if (imageDependencies && imageDependencies.length) {
|
|
233
|
-
const existingVals = await RedisStore.filesToDownloadXPath.getMany(imageDependencies.map((dep) => dep.path));
|
|
234
|
-
for (const dep of imageDependencies) {
|
|
235
|
-
const { mult, width } = getSizeFromUrl(dep.url);
|
|
236
|
-
const existingVal = existingVals[dep.path];
|
|
237
|
-
const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1);
|
|
238
|
-
if (currentDepIsHigherRes) {
|
|
239
|
-
filesToDownload[dep.path] = {
|
|
240
|
-
url: urlHelper.serializeUrl(dep.url),
|
|
241
|
-
kind: 'image',
|
|
242
|
-
mult,
|
|
243
|
-
width,
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
if (videoDependencies && videoDependencies.length) {
|
|
249
|
-
const existingVals = await RedisStore.filesToDownloadXPath.getMany(videoDependencies.map((dep) => dep.path));
|
|
250
|
-
for (const dep of videoDependencies) {
|
|
251
|
-
const { mult, width } = getSizeFromUrl(dep.url);
|
|
252
|
-
const existingVal = existingVals[dep.path];
|
|
253
|
-
const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1);
|
|
254
|
-
if (currentDepIsHigherRes) {
|
|
255
|
-
filesToDownload[dep.path] = {
|
|
256
|
-
url: urlHelper.serializeUrl(dep.url),
|
|
257
|
-
kind: 'video',
|
|
258
|
-
mult,
|
|
259
|
-
width,
|
|
260
|
-
};
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
await RedisStore.filesToDownloadXPath.setMany(filesToDownload);
|
|
265
|
-
const zimArticle = new StringItem(articleId, 'text/html', truncateUtf8Bytes(articleTitle, 245), { FRONT_ARTICLE: 1 }, finalHTML);
|
|
266
|
-
await zimCreatorMutex.runExclusive(() => zimCreator.addItem(zimArticle));
|
|
267
|
-
return null;
|
|
268
|
-
}
|
|
269
|
-
catch (err) {
|
|
270
|
-
return err;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
/*
|
|
274
|
-
* Fetch Articles
|
|
275
|
-
*/
|
|
276
|
-
export async function saveArticles(zimCreator, dump) {
|
|
277
|
-
const jsModuleDependencies = new Set();
|
|
278
|
-
const cssModuleDependencies = new Set();
|
|
279
|
-
const staticFilesList = new Set();
|
|
280
|
-
let jsConfigVars = '';
|
|
281
|
-
let prevPercentProgress;
|
|
282
|
-
const { articleDetailXId } = RedisStore;
|
|
283
|
-
const articlesTotal = await articleDetailXId.len();
|
|
284
|
-
// number of articles allowed to fail is the greater of 5 or 0.001% (1 per 100k) articles
|
|
285
|
-
dump.maxHardFailedArticles = Math.max(5, Math.floor(articlesTotal / 100000));
|
|
286
|
-
if (dump.customProcessor?.shouldKeepArticle) {
|
|
287
|
-
await getAllArticlesToKeep(articleDetailXId, dump, RenderingContext.mainPageRenderer, RenderingContext.articlesRenderer);
|
|
288
|
-
}
|
|
289
|
-
const stages = ['Download Article and dependencies', 'Parse and Save to ZIM', 'Await left-over promises'];
|
|
290
|
-
// depending on which renderer we use, we can have up to 2 requests to make to download article details
|
|
291
|
-
// each request we need to make can be retried 10 times. Each retry attempttakes at most requestTimeout + retry interval
|
|
292
|
-
// retry interval is an exponentional value from 1 to 60s
|
|
293
|
-
// we assume rest of processing is "fast" and takes at most 1 minute
|
|
294
|
-
const timeout = 2 * (Downloader.requestTimeout * 10 + (1 + 2 + 4 + 8 + 16 + 32 + 60 * 4) * 1000) + 60000;
|
|
295
|
-
await articleDetailXId.iterateItems(Downloader.speed, (articleKeyValuePairs, runningWorkers) => {
|
|
296
|
-
// eslint-disable-next-line no-async-promise-executor
|
|
297
|
-
return new Promise(async (resolve, reject) => {
|
|
298
|
-
/*
|
|
299
|
-
* timer to detect freezes
|
|
300
|
-
*/
|
|
301
|
-
let curStage = 0;
|
|
302
|
-
let curArticle = '';
|
|
303
|
-
const timer = new Timer(() => {
|
|
304
|
-
const errorMessage = `Worker timed out after ${timeout} ms at ${stages[curStage]} ${curArticle}`;
|
|
305
|
-
logger.error(errorMessage);
|
|
306
|
-
reject(new Error(errorMessage));
|
|
307
|
-
}, timeout);
|
|
308
|
-
logger.info(`Worker processing batch of article ids [${logger.logifyArray(Object.keys(articleKeyValuePairs))}] - ${runningWorkers} worker(s) running`);
|
|
309
|
-
const parsePromiseQueue = [];
|
|
310
|
-
for (const [articleId, articleDetail] of Object.entries(articleKeyValuePairs)) {
|
|
311
|
-
timer.reset();
|
|
312
|
-
curStage = 0;
|
|
313
|
-
curArticle = articleId;
|
|
314
|
-
const promises = [];
|
|
315
|
-
let rets;
|
|
316
|
-
try {
|
|
317
|
-
const mainPage = isMainPage(articleId);
|
|
318
|
-
const renderer = mainPage ? RenderingContext.mainPageRenderer : RenderingContext.articlesRenderer;
|
|
319
|
-
const leadSectionId = dump.nodet && !articleDetail.contentmodel && !articleDetail.missing ? config.filters.leadSectionId : '';
|
|
320
|
-
const articleUrl = mainPage ? Downloader.getMainPageUrl(articleId) : Downloader.getArticleUrl(articleId, { sectionId: leadSectionId });
|
|
321
|
-
rets = await Downloader.getArticle(articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail);
|
|
322
|
-
curStage += 1;
|
|
323
|
-
for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, imageDependencies, videoDependencies, moduleDependencies, staticFiles, subtitles, } of rets) {
|
|
324
|
-
if (!finalHTML) {
|
|
325
|
-
logger.warn(`No HTML returned for article [${articleId}], skipping`);
|
|
326
|
-
continue;
|
|
327
|
-
}
|
|
328
|
-
for (const dep of moduleDependencies.jsDependenciesList || []) {
|
|
329
|
-
jsModuleDependencies.add(dep);
|
|
330
|
-
}
|
|
331
|
-
for (const dep of moduleDependencies.styleDependenciesList || []) {
|
|
332
|
-
cssModuleDependencies.add(dep);
|
|
333
|
-
}
|
|
334
|
-
for (const file of staticFiles) {
|
|
335
|
-
staticFilesList.add(file);
|
|
336
|
-
}
|
|
337
|
-
jsConfigVars = moduleDependencies.jsConfigVars || '';
|
|
338
|
-
/*
|
|
339
|
-
* getModuleDependencies and downloader.getArticle are
|
|
340
|
-
* network heavy while parsing and saving is I/O.
|
|
341
|
-
* To parse and download simultaniously, we don't await on save,
|
|
342
|
-
* but instead cache the promise in a queue and check it later
|
|
343
|
-
*/
|
|
344
|
-
promises.push([articleId, saveArticle(zimCreator, finalHTML, mediaDependencies, imageDependencies, videoDependencies, subtitles, articleId, articleTitle)]);
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
catch (err) {
|
|
348
|
-
logger.error(`Error downloading/rendering article ${articleId}`);
|
|
349
|
-
reject(err);
|
|
350
|
-
return;
|
|
351
|
-
}
|
|
352
|
-
curStage += 1;
|
|
353
|
-
if (parsePromiseQueue.length) {
|
|
354
|
-
const [articleId, parsePromise] = parsePromiseQueue.shift();
|
|
355
|
-
curArticle = articleId;
|
|
356
|
-
/*
|
|
357
|
-
* in normal circumstances, where downloading is slower than
|
|
358
|
-
* saving, this promise will always be resolved here already
|
|
359
|
-
*/
|
|
360
|
-
const err = await parsePromise;
|
|
361
|
-
if (err) {
|
|
362
|
-
console.log(err);
|
|
363
|
-
logger.error(`Error parsing article ${articleId}`);
|
|
364
|
-
timer.clear();
|
|
365
|
-
reject(err);
|
|
366
|
-
return;
|
|
367
|
-
}
|
|
368
|
-
dump.status.articles.success += 1;
|
|
369
|
-
}
|
|
370
|
-
if (promises.length) {
|
|
371
|
-
parsePromiseQueue.push(flattenPromises(promises));
|
|
372
|
-
}
|
|
373
|
-
if ((dump.status.articles.success + dump.status.articles.hardFail + dump.status.articles.softFail) % 10 === 0) {
|
|
374
|
-
const percentProgress = (((dump.status.articles.success + dump.status.articles.hardFail + dump.status.articles.softFail) / articlesTotal) * 100).toFixed(1);
|
|
375
|
-
if (percentProgress !== prevPercentProgress) {
|
|
376
|
-
prevPercentProgress = percentProgress;
|
|
377
|
-
logger.log(`Progress downloading articles [${dump.status.articles.success + dump.status.articles.hardFail + dump.status.articles.softFail}/${articlesTotal}] [${percentProgress}%]`);
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
/*
|
|
382
|
-
* clear up potentially still pending promises
|
|
383
|
-
*/
|
|
384
|
-
curStage += 1;
|
|
385
|
-
if (parsePromiseQueue.length) {
|
|
386
|
-
const [articleId, parsePromise] = flattenPromises(parsePromiseQueue);
|
|
387
|
-
curArticle = articleId;
|
|
388
|
-
const err = await parsePromise;
|
|
389
|
-
if (err) {
|
|
390
|
-
timer.clear();
|
|
391
|
-
reject(err);
|
|
392
|
-
return;
|
|
393
|
-
}
|
|
394
|
-
dump.status.articles.success += parsePromiseQueue.length;
|
|
395
|
-
}
|
|
396
|
-
timer.clear();
|
|
397
|
-
resolve();
|
|
398
|
-
});
|
|
399
|
-
});
|
|
400
|
-
logger.log(`Done with downloading a total of [${articlesTotal}] articles`);
|
|
401
|
-
if (jsConfigVars) {
|
|
402
|
-
const jsConfigVarArticle = new StringItem(jsPath('jsConfigVars', config.output.dirs.mediawiki), 'application/javascript', null, { FRONT_ARTICLE: 0 }, jsConfigVars);
|
|
403
|
-
await zimCreatorMutex.runExclusive(() => zimCreator.addItem(jsConfigVarArticle));
|
|
404
|
-
}
|
|
405
|
-
return {
|
|
406
|
-
staticFilesList,
|
|
407
|
-
jsModuleDependencies,
|
|
408
|
-
cssModuleDependencies,
|
|
409
|
-
};
|
|
410
|
-
}
|
|
411
|
-
//# sourceMappingURL=saveArticles.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"saveArticles.js","sourceRoot":"","sources":["../../src/util/saveArticles.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AACtC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,UAAU,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAW,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAErD,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAA;AAEhC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACnC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAA;AACjE,OAAO,EAAE,wCAAwC,EAAE,gDAAgD,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAA;AAClJ,OAAO,SAAS,MAAM,iBAAiB,CAAA;AAEvC,OAAO,gBAAgB,MAAM,mCAAmC,CAAA;AAChE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,UAAU,MAAM,iBAAiB,CAAA;AAExC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,SAA2B,EAAE,UAAmB,EAAE,IAAU;IAW9F,IAAI,mBAA2B,CAAA;IAE/B,2EAA2E;IAC3E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;IACzC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAAA;IAE9D,MAAM,UAAU,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,eAAgC,EAAE,EAAE;QAC/F,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;YACjF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAA;YAChE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACzB,MAAM,eAAe,GAAG,IAAI,UAAU,CAAiB,UAAU,CAAC,MAAM,EAAE,GAAG,QAAQ,QAAQ,CAAC,CAAA;gBAC9F,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;gBAC5C,eAAe,CAAC,KAAK,EAAE,CAAA;gBACvB,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;oBAClB,eAAe;oBACf,eAAe,EAAE,EAAE,EAAE,oCAAoC;oBACzD,eAAe,EAAE,CAAC;oBAClB,eAAe,EAAE,CAAC;oBAClB,iBAAiB,EAAE,KAAK;iBACzB,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC;gBACvC,IAAI,EAAE,IAAI;gBACV,GAAG,EAAE,GAAG;gBACR,IAAI,EAAE,IAAI;gBACV,KAAK,EAAE,KAAK;gBACZ,IAAI,EAAE,IAAI;gBACV,gBAAgB,EAAE,CAAC;aACpB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,UAAU,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAA;IAE7C;;;OAGG;IACH,KAAK,UAAU,qBAAqB;QAClC,6DAA6D;QAC7D,OAAO,IAAI,EAAE,CAAC;YACZ,iDAAiD;YACjD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBACrD,OAAO,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAC7C,CAAC,EAAE,CAAC,CAAC,CAAA;YACL,IAAI,cAAc,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACxC,OAAO,IAAI,CAAA;YACb,CAAC;YAED,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACnD,sDAAsD;gBACtD,IACE,QAAQ,CAAC,iBAAiB;oBAC1B,CAAC,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC/D,CAAC,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,eAAe,GAAG,QAAQ,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAC9F,CAAC;oBACD,SAAQ;gBACV,CAAC;gBAED,kCAAkC;gBAClC,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,GAAG,EAAE,CAAA;gBAC3D,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,QAAQ,CAAC,iBAAiB,GAAG,IAAI,CAAA;oBACjC,SAAQ;gBACV,CAAC;gBAED,mEAAmE;gBACnE,QAAQ,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACrC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAA;YAC/C,CAAC;YAED,kFAAkF;YAClF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;YACzB,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,UAAU,kBAAkB,CAAC,cAA8B,EAAE,QAAgB,EAAE,QAAkB,EAAE,QAAgB;QACtH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACzF,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAC9G,IAAI,eAAe,KAAK,mBAAmB,EAAE,CAAC;gBAC5C,mBAAmB,GAAG,eAAe,CAAA;gBACrC,MAAM,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,UAAU,MAAM,eAAe,IAAI,CAAC,CAAA;YACtI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,wCAAwC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,UAAU,GAAG,gDAAgD,EAAE,CAAC;gBAC1K,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,UAAU,GAAG,CAAC,CAAA;YACjG,CAAC;QACH,CAAC;QAED,cAAc,CAAC,gBAAgB,IAAI,CAAC,CAAA;QACpC,MAAM,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,SAAS,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,IAAI,GAAG,CAAC,CAAA;QACtH,MAAM,UAAU,CAAC,eAAe,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC;aAC7E,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACnB,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC7C,0GAA0G;gBAC1G,kCAAkC;gBAClC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;gBAC5G,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;gBAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAA;gBAC9B,QAAQ,CAAC,eAAe,IAAI,CAAC,CAAA;YAC/B,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAA;YACnD,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACnB,IAAI,cAAc,CAAC,gBAAgB,GAAG,yBAAyB,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;gBAChH,MAAM,CAAC,IAAI,CAAC,2BAA2B,SAAS,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,QAAQ,EAAE,MAAM,aAAa,CAAC,CAAA;gBAClI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAA;gBAC3B,QAAQ,CAAC,eAAe,IAAI,CAAC,CAAA;YAC/B,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;oBACjB,MAAM,gBAAgB,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,CAAA;oBACxE,IAAI,gBAAgB,EAAE,CAAC;wBACrB,MAAM,SAAS,GAAG,qBAAqB,CAAC,gBAAgB,CAAC,CAAA;wBACzD,IAAI,SAAS,EAAE,CAAC;4BACd,QAAQ,CAAC,aAAa,GAAG,SAAS,CAAA;4BAClC,MAAM,CAAC,GAAG,CAAC,2BAA2B,gBAAgB,mBAAmB,QAAQ,UAAU,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAA;wBACtH,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,IAAI,CAAC,2BAA2B,gBAAgB,UAAU,QAAQ,6BAA6B,CAAC,CAAA;wBACzG,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,sIAAsI;gBACtI,IACE,GAAG,CAAC,QAAQ;oBACZ,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC7C,CAAC,SAAS,CAAC,cAAc,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,iDAAiD,CAAC,EACtG,CAAC;oBACD,QAAQ,CAAC,eAAe,GAAG,QAAQ,CAAC,eAAe,GAAG,GAAG,CAAA,CAAC,qEAAqE;oBAC/H,MAAM,CAAC,GAAG,CAAC,sBAAsB,GAAG,CAAC,QAAQ,CAAC,MAAM,mBAAmB,QAAQ,OAAO,QAAQ,CAAC,eAAe,aAAa,CAAC,CAAA;gBAC9H,CAAC;gBACD,MAAM,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YACrD,CAAC;QACH,CAAC,CAAC,CAAA;IACN,CAAC;IAED,MAAM,IAAI,CACR,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACrD,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzB,OAAO,IAAI,EAAE,CAAC;YACZ,wGAAwG;YACxG,oGAAoG;YACpG,4BAA4B;YAC5B,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAA;YAChF,IAAI,CAAC,YAAY;gBAAE,MAAK;YACxB,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAA;YAC3D,MAAM,kBAAkB,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;IACH,CAAC,EACD,EAAE,WAAW,EAAE,UAAU,CAAC,KAAK,EAAE,CAClC,CAAA;IAED,MAAM,CAAC,GAAG,CACR,yBAAyB,UAAU,WAAW,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,aAAa,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,SAAS,EACnH,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,IAAI,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAC5K,CAAA;AACH,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,gBAAqC,EAAE,IAAU,EAAE,gBAA0B,EAAE,gBAA0B;IAC3I,MAAM,gBAAgB,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE;QACnF,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC;YAC9E,IAAI,IAAS,CAAA;YACb,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;gBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAA;gBAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAA;gBAC7H,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;gBAEtI,IAAI,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,aAAa,CAAC,CAAA;gBAC1G,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;oBACvC,IAAI,CAAC,IAAI,EAAE,CAAC;wBACV,SAAQ;oBACV,CAAC;oBAED,MAAM,GAAG,GAAG,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;oBACvC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;wBACjF,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,8BAA8B,SAAS,aAAa,EAAE,GAAG,CAAC,CAAA;gBACtE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,SAAqC;IAC5D,OAAO;QACL,kBAAkB;QAClB,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACf,2CAA2C;QAC3C,CAAC,KAAK,IAAI,EAAE;YACV,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAA;QACpC,CAAC,CAAC,EAAE;KACL,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACxB,UAAmB,EACnB,SAAiB,EACjB,iBAAsB,EACtB,iBAAsB,EACtB,iBAAsB,EACtB,SAAc,EACd,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC;QACH,MAAM,eAAe,GAAoB,EAAE,CAAA;QAE3C,IAAI,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACtB,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,CAAA;YAC5D,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YAE5G,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;gBACpC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBAC/C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC1C,MAAM,qBAAqB,GAAG,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;gBACnH,IAAI,qBAAqB,EAAE,CAAC;oBAC1B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;wBAC1B,GAAG,EAAE,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;wBACpC,IAAI,EAAE,OAAO;wBACb,IAAI;wBACJ,KAAK;qBACN,CAAA;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YAE5G,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;gBACpC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBAC/C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC1C,MAAM,qBAAqB,GAAG,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;gBACnH,IAAI,qBAAqB,EAAE,CAAC;oBAC1B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;wBAC1B,GAAG,EAAE,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;wBACpC,IAAI,EAAE,OAAO;wBACb,IAAI;wBACJ,KAAK;qBACN,CAAA;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YAE5G,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;gBACpC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBAC/C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC1C,MAAM,qBAAqB,GAAG,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;gBACnH,IAAI,qBAAqB,EAAE,CAAC;oBAC1B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG;wBAC1B,GAAG,EAAE,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;wBACpC,IAAI,EAAE,OAAO;wBACb,IAAI;wBACJ,KAAK;qBACN,CAAA;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;QAE9D,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,WAAW,EAAE,iBAAiB,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAChI,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;QAExE,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,GAAG,CAAA;IACZ,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,UAAmB,EAAE,IAAU;IAChE,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9C,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAA;IAC/C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAA;IACzC,IAAI,YAAY,GAAG,EAAE,CAAA;IACrB,IAAI,mBAA2B,CAAA;IAC/B,MAAM,EAAE,gBAAgB,EAAE,GAAG,UAAU,CAAA;IACvC,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,CAAA;IAElD,yFAAyF;IACzF,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,MAAM,CAAC,CAAC,CAAA;IAE5E,IAAI,IAAI,CAAC,eAAe,EAAE,iBAAiB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,CAAC,gBAAgB,EAAE,IAAI,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,gBAAgB,CAAC,CAAA;IAC1H,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,mCAAmC,EAAE,uBAAuB,EAAE,0BAA0B,CAAC,CAAA;IACzG,uGAAuG;IACvG,wHAAwH;IACxH,yDAAyD;IACzD,oEAAoE;IACpE,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAA;IACxG,MAAM,gBAAgB,CAAC,YAAY,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,oBAAoB,EAAE,cAAc,EAAE,EAAE;QAC7F,qDAAqD;QACrD,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C;;eAEG;YACH,IAAI,QAAQ,GAAG,CAAC,CAAA;YAChB,IAAI,UAAU,GAAG,EAAE,CAAA;YACnB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,EAAE;gBAC3B,MAAM,YAAY,GAAG,0BAA0B,OAAO,UAAU,MAAM,CAAC,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAA;gBAChG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;gBAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAA;YACjC,CAAC,EAAE,OAAO,CAAC,CAAA;YAEX,MAAM,CAAC,IAAI,CAAC,2CAA2C,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,OAAO,cAAc,oBAAoB,CAAC,CAAA;YAEtJ,MAAM,iBAAiB,GAA+B,EAAE,CAAA;YAExD,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBAC9E,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,QAAQ,GAAG,CAAC,CAAA;gBACZ,UAAU,GAAG,SAAS,CAAA;gBACtB,MAAM,QAAQ,GAA+B,EAAE,CAAA;gBAE/C,IAAI,IAAS,CAAA;gBACb,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;oBACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC,gBAAgB,CAAA;oBACjG,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAA;oBAC7H,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAA;oBAEtI,IAAI,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,aAAa,CAAC,CAAA;oBAE1G,QAAQ,IAAI,CAAC,CAAA;oBACb,KAAK,MAAM,EACT,SAAS,EACT,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,SAAS,EACf,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,GACV,IAAI,IAAI,EAAE,CAAC;wBACV,IAAI,CAAC,SAAS,EAAE,CAAC;4BACf,MAAM,CAAC,IAAI,CAAC,iCAAiC,SAAS,aAAa,CAAC,CAAA;4BACpE,SAAQ;wBACV,CAAC;wBAED,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;4BAC9D,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;wBAC/B,CAAC;wBACD,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,qBAAqB,IAAI,EAAE,EAAE,CAAC;4BACjE,qBAAqB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;wBAChC,CAAC;wBAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;4BAC/B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;wBAC3B,CAAC;wBAED,YAAY,GAAG,kBAAkB,CAAC,YAAY,IAAI,EAAE,CAAA;wBAEpD;;;;;2BAKG;wBACH,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,CAAA;oBAC7J,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAA;oBAChE,MAAM,CAAC,GAAG,CAAC,CAAA;oBACX,OAAM;gBACR,CAAC;gBAED,QAAQ,IAAI,CAAC,CAAA;gBACb,IAAI,iBAAiB,CAAC,MAAM,EAAE,CAAC;oBAC7B,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAA;oBAC3D,UAAU,GAAG,SAAS,CAAA;oBACtB;;;uBAGG;oBACH,MAAM,GAAG,GAAG,MAAM,YAAY,CAAA;oBAC9B,IAAI,GAAG,EAAE,CAAC;wBACR,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;wBAEhB,MAAM,CAAC,KAAK,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAA;wBAClD,KAAK,CAAC,KAAK,EAAE,CAAA;wBACb,MAAM,CAAC,GAAG,CAAC,CAAA;wBACX,OAAM;oBACR,CAAC;oBACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAA;gBACnC,CAAC;gBAED,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpB,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAA;gBACnD,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC9G,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;oBAC3J,IAAI,eAAe,KAAK,mBAAmB,EAAE,CAAC;wBAC5C,mBAAmB,GAAG,eAAe,CAAA;wBACrC,MAAM,CAAC,GAAG,CACR,kCACE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QACtF,IAAI,aAAa,MAAM,eAAe,IAAI,CAC3C,CAAA;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED;;eAEG;YACH,QAAQ,IAAI,CAAC,CAAA;YACb,IAAI,iBAAiB,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,eAAe,CAAC,iBAAiB,CAAC,CAAA;gBACpE,UAAU,GAAG,SAAS,CAAA;gBACtB,MAAM,GAAG,GAAG,MAAM,YAAY,CAAA;gBAC9B,IAAI,GAAG,EAAE,CAAC;oBACR,KAAK,CAAC,KAAK,EAAE,CAAA;oBACb,MAAM,CAAC,GAAG,CAAC,CAAA;oBACX,OAAM;gBACR,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,IAAI,iBAAiB,CAAC,MAAM,CAAA;YAC1D,CAAC;YAED,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,OAAO,EAAE,CAAA;QACX,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,MAAM,CAAC,GAAG,CAAC,qCAAqC,aAAa,YAAY,CAAC,CAAA;IAE1E,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,kBAAkB,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,wBAAwB,EAAE,IAAI,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,YAAY,CAAC,CAAA;QACnK,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAA;IAClF,CAAC;IAED,OAAO;QACL,eAAe;QACf,oBAAoB;QACpB,qBAAqB;KACtB,CAAA;AACH,CAAC"}
|
package/res/article_list_home.js
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
(function () {
|
|
3
|
-
|
|
4
|
-
document.addEventListener("DOMContentLoaded", function () {
|
|
5
|
-
|
|
6
|
-
var body = document.body;
|
|
7
|
-
if (body.classList.contains('article-list-home')) {
|
|
8
|
-
// 'import' to avoid not defined js error
|
|
9
|
-
var imagesLoaded = window['imagesLoaded'];
|
|
10
|
-
var Masonry = window['Masonry'];
|
|
11
|
-
var grid = document.getElementById('content');
|
|
12
|
-
|
|
13
|
-
// js available, remove nojs styles
|
|
14
|
-
document.getElementsByClassName('mw-body-content')[0].classList.remove('nojs');
|
|
15
|
-
|
|
16
|
-
imagesLoaded(grid, function () {
|
|
17
|
-
new Masonry(grid, {
|
|
18
|
-
itemSelector: '.item'
|
|
19
|
-
});
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
});
|
|
24
|
-
})();
|
package/res/content.parsoid.css
DELETED
|
@@ -1,196 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Style Parsoid HTML+RDFa output consistent with wikitext from PHP parser.
|
|
3
|
-
* Source here :; https://github.com/voidlinux/void-wiki/blob/master/resources/src/mediawiki.skinning/content.parsoid.less
|
|
4
|
-
* converted from less to css
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
/*csslint regex-selectors:false */
|
|
8
|
-
|
|
9
|
-
/*
|
|
10
|
-
* Auto-numbered external links
|
|
11
|
-
* Parsoid renders those as link without content, and lets CSS do the
|
|
12
|
-
* counting. This way the counting style can be customized, and counts update
|
|
13
|
-
* automatically when content is modified.
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
.mw-body-content {
|
|
17
|
-
counter-reset: mw-numbered-ext-link;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
.mw-body-content a[rel~='mw:ExtLink']:empty:after {
|
|
21
|
-
content: '[' counter(mw-numbered-ext-link) ']';
|
|
22
|
-
counter-increment: mw-numbered-ext-link;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* References
|
|
27
|
-
*
|
|
28
|
-
* Parser and Extension:Cite output reference numbers for <sup>[1]</sup> for <ref> tags.
|
|
29
|
-
*
|
|
30
|
-
* Markup:
|
|
31
|
-
* Cake is good<sup>[2]</sup>
|
|
32
|
-
* The cake is a lie<span class="reference">[1]</span>
|
|
33
|
-
*
|
|
34
|
-
* Styleguide 1.1.
|
|
35
|
-
*/
|
|
36
|
-
|
|
37
|
-
span.reference {
|
|
38
|
-
font-size: 80%;
|
|
39
|
-
line-height: 1;
|
|
40
|
-
vertical-align: super;
|
|
41
|
-
unicode-bidi: -moz-isolate;
|
|
42
|
-
unicode-bidi: isolate;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
sup,
|
|
46
|
-
sub {
|
|
47
|
-
line-height: 1;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Block media items
|
|
52
|
-
*/
|
|
53
|
-
|
|
54
|
-
figure[typeof*='mw:File'],
|
|
55
|
-
figure[typeof*='mw:Image'],
|
|
56
|
-
figure[typeof*='mw:Video'],
|
|
57
|
-
figure[typeof*='mw:Audio'] {
|
|
58
|
-
margin: 0;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
figure[typeof*='mw:File'] a,
|
|
62
|
-
figure[typeof*='mw:Image'] a,
|
|
63
|
-
figure[typeof*='mw:Video'] a,
|
|
64
|
-
figure[typeof*='mw:Audio'] a {
|
|
65
|
-
border: 0;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
figure[typeof*='mw:File'].mw-halign-right,
|
|
69
|
-
figure[typeof*='mw:Image'].mw-halign-right,
|
|
70
|
-
figure[typeof*='mw:Video'].mw-halign-right,
|
|
71
|
-
figure[typeof*='mw:Audio'].mw-halign-right {
|
|
72
|
-
/* @noflip */
|
|
73
|
-
margin: 0.5em 0 1.3em 1.4em;
|
|
74
|
-
/* @noflip */
|
|
75
|
-
clear: right;
|
|
76
|
-
/* @noflip */
|
|
77
|
-
float: right;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
figure[typeof*='mw:File'].mw-halign-left,
|
|
81
|
-
figure[typeof*='mw:Image'].mw-halign-left,
|
|
82
|
-
figure[typeof*='mw:Video'].mw-halign-left,
|
|
83
|
-
figure[typeof*='mw:Audio'].mw-halign-left {
|
|
84
|
-
/* @noflip */
|
|
85
|
-
margin: 0.5em 1.4em 1.3em 0;
|
|
86
|
-
/* @noflip */
|
|
87
|
-
clear: left;
|
|
88
|
-
/* @noflip */
|
|
89
|
-
float: left;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
figure[typeof*='mw:File'].mw-halign-none,
|
|
93
|
-
figure[typeof*='mw:Image'].mw-halign-none,
|
|
94
|
-
figure[typeof*='mw:Video'].mw-halign-none,
|
|
95
|
-
figure[typeof*='mw:Audio'].mw-halign-none {
|
|
96
|
-
margin: 0;
|
|
97
|
-
clear: none;
|
|
98
|
-
float: none;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
figure[typeof*='mw:File'].mw-halign-center,
|
|
102
|
-
figure[typeof*='mw:Image'].mw-halign-center,
|
|
103
|
-
figure[typeof*='mw:Video'].mw-halign-center,
|
|
104
|
-
figure[typeof*='mw:Audio'].mw-halign-center {
|
|
105
|
-
margin: 0 auto 0.5em auto;
|
|
106
|
-
display: table;
|
|
107
|
-
clear: none;
|
|
108
|
-
float: none;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
figure[typeof*='mw:File']>figcaption,
|
|
112
|
-
figure[typeof*='mw:Image']>figcaption,
|
|
113
|
-
figure[typeof*='mw:Video']>figcaption,
|
|
114
|
-
figure[typeof*='mw:Audio']>figcaption {
|
|
115
|
-
display: table-caption;
|
|
116
|
-
caption-side: bottom;
|
|
117
|
-
/* In mw-core the font-size is duplicated, 94% in thumbiner
|
|
118
|
-
* and again 94% in thumbcaption. 88.4% for font size of the
|
|
119
|
-
* caption results in the same behavior. */
|
|
120
|
-
font-size: 88.4%;
|
|
121
|
-
line-height: 1.4em;
|
|
122
|
-
text-align: left;
|
|
123
|
-
border: 1px solid #c8ccd1;
|
|
124
|
-
border-top: 0;
|
|
125
|
-
/* taken from .thumbcaption, plus .thumbinner */
|
|
126
|
-
padding: 0 6px 6px 6px;
|
|
127
|
-
background-color: #f8f9fa;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
figure[typeof*='mw:File']>figcaption table,
|
|
131
|
-
figure[typeof*='mw:Image']>figcaption table,
|
|
132
|
-
figure[typeof*='mw:Video']>figcaption table,
|
|
133
|
-
figure[typeof*='mw:Audio']>figcaption table {
|
|
134
|
-
/* reset caption side for tables inside figcaptions */
|
|
135
|
-
caption-side: top;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
figure[typeof~='mw:File/Thumb'],
|
|
139
|
-
figure[typeof~='mw:Image/Thumb'],
|
|
140
|
-
figure[typeof~='mw:Video/Thumb'],
|
|
141
|
-
figure[typeof~='mw:Audio/Thumb'],
|
|
142
|
-
figure[typeof~='mw:File/Frame'],
|
|
143
|
-
figure[typeof~='mw:Image/Frame'],
|
|
144
|
-
figure[typeof~='mw:Video/Frame'],
|
|
145
|
-
figure[typeof~='mw:Audio/Frame'] {
|
|
146
|
-
display: table;
|
|
147
|
-
overflow: auto;
|
|
148
|
-
text-align: center;
|
|
149
|
-
border: 1px solid #c8ccd1;
|
|
150
|
-
border-bottom: 0;
|
|
151
|
-
border-collapse: collapse;
|
|
152
|
-
background-color: #f8f9fa;
|
|
153
|
-
margin: 0.5em 0 1.3em 1.4em;
|
|
154
|
-
clear: right;
|
|
155
|
-
float: right;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
figure[typeof~='mw:File/Thumb']>*:first-child>img,
|
|
159
|
-
figure[typeof~='mw:File/Thumb']>*:first-child>video,
|
|
160
|
-
figure[typeof~='mw:File/Thumb']>*:first-child>audio,
|
|
161
|
-
figure[typeof~='mw:Image/Thumb']>*:first-child>img,
|
|
162
|
-
figure[typeof~='mw:Video/Thumb']>*:first-child>video,
|
|
163
|
-
figure[typeof~='mw:Audio/Thumb']>*:first-child>video,
|
|
164
|
-
figure[typeof~='mw:Audio/Thumb']>*:first-child>audio,
|
|
165
|
-
figure[typeof~='mw:File/Frame']>*:first-child>img,
|
|
166
|
-
figure[typeof~='mw:File/Frame']>*:first-child>video,
|
|
167
|
-
figure[typeof~='mw:File/Frame']>*:first-child>audio,
|
|
168
|
-
figure[typeof~='mw:Image/Frame']>*:first-child>img,
|
|
169
|
-
figure[typeof~='mw:Video/Frame']>*:first-child>video,
|
|
170
|
-
figure[typeof~='mw:Audio/Frame']>*:first-child>video,
|
|
171
|
-
figure[typeof~='mw:Audio/Frame']>*:first-child>audio {
|
|
172
|
-
border: 1px solid #c8ccd1;
|
|
173
|
-
margin: 3px;
|
|
174
|
-
background: #fff;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/* Same as img.thumbborder in content.css */
|
|
178
|
-
|
|
179
|
-
.mw-image-border>*:first-child>img,
|
|
180
|
-
.mw-image-border>*:first-child>video,
|
|
181
|
-
.mw-image-border>*:first-child>audio {
|
|
182
|
-
border: 1px solid #eaecf0;
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
/* Hide the caption for frameless and plain floated images */
|
|
186
|
-
|
|
187
|
-
figure[typeof~='mw:File/Frameless']>figcaption,
|
|
188
|
-
figure[typeof~='mw:Image/Frameless']>figcaption,
|
|
189
|
-
figure[typeof~='mw:Video/Frameless']>figcaption,
|
|
190
|
-
figure[typeof~='mw:Audio/Frameless']>figcaption,
|
|
191
|
-
figure[typeof~='mw:File']>figcaption,
|
|
192
|
-
figure[typeof~='mw:Image']>figcaption,
|
|
193
|
-
figure[typeof~='mw:Video']>figcaption,
|
|
194
|
-
figure[typeof~='mw:Audio']>figcaption {
|
|
195
|
-
display: none;
|
|
196
|
-
}
|