mwoffliner 1.17.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +355 -62
- package/docs/functional_architecture.md +34 -38
- package/extensions/wiktionary_fr.js +18 -17
- package/jest.config.cjs +2 -2
- package/lib/DOMUtils.js +0 -1
- package/lib/DOMUtils.js.map +1 -1
- package/lib/Downloader.d.ts +39 -31
- package/lib/Downloader.js +418 -349
- package/lib/Downloader.js.map +1 -1
- package/lib/Dump.d.ts +24 -10
- package/lib/Dump.js +95 -50
- package/lib/Dump.js.map +1 -1
- package/lib/Gadgets.d.ts +2 -1
- package/lib/Gadgets.js +7 -5
- package/lib/Gadgets.js.map +1 -1
- package/lib/Logger.d.ts +6 -6
- package/lib/Logger.js +33 -33
- package/lib/Logger.js.map +1 -1
- package/lib/MediaWiki.d.ts +14 -31
- package/lib/MediaWiki.js +119 -172
- package/lib/MediaWiki.js.map +1 -1
- package/lib/RedisStore.d.ts +3 -3
- package/lib/RedisStore.js +21 -23
- package/lib/RedisStore.js.map +1 -1
- package/lib/S3.js +1 -1
- package/lib/S3.js.map +1 -1
- package/lib/Templates.d.ts +3 -10
- package/lib/Templates.js +5 -14
- package/lib/Templates.js.map +1 -1
- package/lib/cli.js +7 -6
- package/lib/cli.js.map +1 -1
- package/lib/config.d.ts +10 -16
- package/lib/config.js +33 -55
- package/lib/config.js.map +1 -1
- package/lib/error.manager.d.ts +2 -2
- package/lib/error.manager.js +42 -49
- package/lib/error.manager.js.map +1 -1
- package/lib/i18n.d.ts +2 -0
- package/lib/i18n.js +53 -0
- package/lib/i18n.js.map +1 -0
- package/lib/mutex.d.ts +2 -1
- package/lib/mutex.js +2 -1
- package/lib/mutex.js.map +1 -1
- package/lib/mwoffliner.lib.js +370 -222
- package/lib/mwoffliner.lib.js.map +1 -1
- package/lib/parameterList.d.ts +20 -8
- package/lib/parameterList.js +30 -18
- package/lib/parameterList.js.map +1 -1
- package/lib/renderers/abstract.renderer.d.ts +48 -55
- package/lib/renderers/abstract.renderer.js +377 -98
- package/lib/renderers/abstract.renderer.js.map +1 -1
- package/lib/renderers/action-parse.renderer.d.ts +22 -3
- package/lib/renderers/action-parse.renderer.js +123 -81
- package/lib/renderers/action-parse.renderer.js.map +1 -1
- package/lib/renderers/renderer.builder.js +6 -69
- package/lib/renderers/renderer.builder.js.map +1 -1
- package/lib/renderers/rendering.context.d.ts +2 -3
- package/lib/renderers/rendering.context.js +6 -14
- package/lib/renderers/rendering.context.js.map +1 -1
- package/lib/sanitize-argument.d.ts +7 -4
- package/lib/sanitize-argument.js +99 -41
- package/lib/sanitize-argument.js.map +1 -1
- package/lib/util/FileManager.d.ts +36 -0
- package/lib/util/FileManager.js +246 -0
- package/lib/util/FileManager.js.map +1 -0
- package/lib/util/RedisKvs.js +4 -4
- package/lib/util/RedisKvs.js.map +1 -1
- package/lib/util/RedisQueue.js +1 -1
- package/lib/util/RedisQueue.js.map +1 -1
- package/lib/util/builders/url/action-parse.director.d.ts +2 -2
- package/lib/util/builders/url/action-parse.director.js +7 -7
- package/lib/util/builders/url/action-parse.director.js.map +1 -1
- package/lib/util/builders/url/api.director.d.ts +2 -4
- package/lib/util/builders/url/api.director.js +10 -17
- package/lib/util/builders/url/api.director.js.map +1 -1
- package/lib/util/builders/url/base.director.d.ts +0 -4
- package/lib/util/builders/url/base.director.js +0 -25
- package/lib/util/builders/url/base.director.js.map +1 -1
- package/lib/util/builders/url/basic.director.d.ts +1 -1
- package/lib/util/builders/url/basic.director.js +1 -1
- package/lib/util/builders/url/web.director.d.ts +1 -1
- package/lib/util/builders/url/web.director.js +2 -2
- package/lib/util/builders/url/web.director.js.map +1 -1
- package/lib/util/categories.d.ts +5 -5
- package/lib/util/categories.js +175 -174
- package/lib/util/categories.js.map +1 -1
- package/lib/util/const.d.ts +4 -8
- package/lib/util/const.js +9 -17
- package/lib/util/const.js.map +1 -1
- package/lib/util/customCssJs.d.ts +6 -0
- package/lib/util/customCssJs.js +70 -0
- package/lib/util/customCssJs.js.map +1 -0
- package/lib/util/dump.d.ts +15 -4
- package/lib/util/dump.js +282 -86
- package/lib/util/dump.js.map +1 -1
- package/lib/util/index.d.ts +1 -1
- package/lib/util/index.js +1 -1
- package/lib/util/index.js.map +1 -1
- package/lib/util/metaData.js.map +1 -1
- package/lib/util/misc.d.ts +23 -13
- package/lib/util/misc.js +90 -59
- package/lib/util/misc.js.map +1 -1
- package/lib/util/mw-api.d.ts +7 -5
- package/lib/util/mw-api.js +167 -221
- package/lib/util/mw-api.js.map +1 -1
- package/lib/util/pageListMainPage.d.ts +3 -0
- package/lib/util/pageListMainPage.js +10 -0
- package/lib/util/pageListMainPage.js.map +1 -0
- package/lib/util/pages.d.ts +30 -0
- package/lib/util/pages.js +146 -0
- package/lib/util/pages.js.map +1 -0
- package/lib/util/rewriteUrls.d.ts +2 -2
- package/lib/util/rewriteUrls.js +33 -31
- package/lib/util/rewriteUrls.js.map +1 -1
- package/lib/util/savePages.d.ts +8 -0
- package/lib/util/savePages.js +182 -0
- package/lib/util/savePages.js.map +1 -0
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/offliner-definition.json +178 -90
- package/package.json +50 -49
- package/res/page_list_home.js +20 -0
- package/res/{article_not_found.svg → page_not_found.svg} +1 -1
- package/res/script.js +30 -45
- package/res/style.css +52 -68
- package/res/templates/download_error_placeholder.html +7 -8
- package/res/templates/javaScript.html +19 -0
- package/res/templates/pageFallback.html +9 -7
- package/res/templates/pageVector2022.html +9 -7
- package/res/templates/pageVectorLegacy.html +9 -7
- package/res/templates/{article_list_home.html → page_list_home.html} +1 -2
- package/translation/ar.json +21 -21
- package/translation/bn.json +1 -1
- package/translation/de.json +46 -19
- package/translation/en.json +54 -21
- package/translation/es.json +17 -19
- package/translation/fi.json +23 -3
- package/translation/fr.json +57 -21
- package/translation/he.json +8 -6
- package/translation/ia.json +19 -19
- package/translation/id.json +13 -13
- package/translation/it.json +13 -8
- package/translation/ko.json +55 -3
- package/translation/lb.json +5 -2
- package/translation/mk.json +5 -5
- package/translation/nl.json +57 -22
- package/translation/or.json +2 -2
- package/translation/pt.json +2 -2
- package/translation/qqq.json +55 -22
- package/translation/ru.json +2 -2
- package/translation/sc.json +2 -2
- package/translation/sk.json +62 -0
- package/translation/sl.json +54 -21
- package/translation/sv.json +20 -19
- package/translation/sw.json +45 -2
- package/translation/zh-hans.json +56 -22
- package/translation/zh-hant.json +26 -19
- package/lib/renderers/abstractDesktop.render.d.ts +0 -12
- package/lib/renderers/abstractDesktop.render.js +0 -67
- package/lib/renderers/abstractDesktop.render.js.map +0 -1
- package/lib/renderers/abstractMobile.render.d.ts +0 -12
- package/lib/renderers/abstractMobile.render.js +0 -54
- package/lib/renderers/abstractMobile.render.js.map +0 -1
- package/lib/renderers/rest-api.renderer.d.ts +0 -7
- package/lib/renderers/rest-api.renderer.js +0 -68
- package/lib/renderers/rest-api.renderer.js.map +0 -1
- package/lib/renderers/visual-editor.renderer.d.ts +0 -7
- package/lib/renderers/visual-editor.renderer.js +0 -76
- package/lib/renderers/visual-editor.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-desktop.renderer.d.ts +0 -4
- package/lib/renderers/wikimedia-desktop.renderer.js +0 -7
- package/lib/renderers/wikimedia-desktop.renderer.js.map +0 -1
- package/lib/renderers/wikimedia-mobile.renderer.d.ts +0 -19
- package/lib/renderers/wikimedia-mobile.renderer.js +0 -243
- package/lib/renderers/wikimedia-mobile.renderer.js.map +0 -1
- package/lib/util/articleListMainPage.d.ts +0 -3
- package/lib/util/articleListMainPage.js +0 -10
- package/lib/util/articleListMainPage.js.map +0 -1
- package/lib/util/articles.d.ts +0 -26
- package/lib/util/articles.js +0 -102
- package/lib/util/articles.js.map +0 -1
- package/lib/util/builders/url/desktop.director.d.ts +0 -8
- package/lib/util/builders/url/desktop.director.js +0 -15
- package/lib/util/builders/url/desktop.director.js.map +0 -1
- package/lib/util/builders/url/mobile.director.d.ts +0 -8
- package/lib/util/builders/url/mobile.director.js +0 -15
- package/lib/util/builders/url/mobile.director.js.map +0 -1
- package/lib/util/builders/url/rest-api.director.d.ts +0 -8
- package/lib/util/builders/url/rest-api.director.js +0 -18
- package/lib/util/builders/url/rest-api.director.js.map +0 -1
- package/lib/util/builders/url/visual-editor.director.d.ts +0 -9
- package/lib/util/builders/url/visual-editor.director.js +0 -18
- package/lib/util/builders/url/visual-editor.director.js.map +0 -1
- package/lib/util/saveArticles.d.ts +0 -8
- package/lib/util/saveArticles.js +0 -411
- package/lib/util/saveArticles.js.map +0 -1
- package/res/article_list_home.js +0 -24
- package/res/content.parsoid.css +0 -196
- package/res/inserted_style.css +0 -45
- package/res/templates/categories.html +0 -11
- package/res/templates/lead_section_wrapper.html +0 -6
- package/res/templates/pageWikimediaDesktop.html +0 -24
- package/res/templates/pageWikimediaMobile.html +0 -24
- package/res/templates/section_wrapper.html +0 -5
- package/res/templates/subcategories.html +0 -23
- package/res/templates/subpages.html +0 -17
- package/res/templates/subsection_wrapper.html +0 -5
- package/res/webpHandler.js +0 -165
- package/res/wm_mobile_override_script.js +0 -15
- package/res/wm_mobile_override_style.css +0 -20
- package/translation/br.json +0 -8
- package/translation/dag.json +0 -9
- package/translation/es-formal.json +0 -29
- package/translation/ha.json +0 -10
- package/translation/hi.json +0 -9
- package/translation/ig.json +0 -10
- package/translation/kaa.json +0 -9
- package/translation/nb.json +0 -9
- package/translation/nqo.json +0 -8
- package/translation/pt-br.json +0 -10
- package/translation/ro.json +0 -9
- package/translation/scn.json +0 -8
- package/translation/sq.json +0 -9
- package/translation/te.json +0 -9
- package/translation/tn.json +0 -9
- package/translation/tr.json +0 -10
|
@@ -1,21 +1,10 @@
|
|
|
1
1
|
import MediaWiki from './../MediaWiki.js';
|
|
2
|
-
import { VisualEditorRenderer } from './visual-editor.renderer.js';
|
|
3
|
-
import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js';
|
|
4
|
-
import { WikimediaMobileRenderer } from './wikimedia-mobile.renderer.js';
|
|
5
|
-
import { RestApiRenderer } from './rest-api.renderer.js';
|
|
6
2
|
import * as logger from './../Logger.js';
|
|
7
3
|
import { ActionParseRenderer } from './action-parse.renderer.js';
|
|
8
4
|
export class RendererBuilder {
|
|
9
5
|
async createRenderer(options) {
|
|
10
6
|
const { renderType, renderName } = options;
|
|
11
|
-
const [
|
|
12
|
-
MediaWiki.hasVisualEditorApi(),
|
|
13
|
-
MediaWiki.hasWikimediaDesktopApi(),
|
|
14
|
-
MediaWiki.hasWikimediaMobileApi(),
|
|
15
|
-
MediaWiki.hasRestApi(),
|
|
16
|
-
MediaWiki.hasActionParseApi(),
|
|
17
|
-
MediaWiki.hasModuleApi(),
|
|
18
|
-
]);
|
|
7
|
+
const [hasActionParseApi, hasModuleApi] = await Promise.all([MediaWiki.hasActionParseApi(), MediaWiki.hasModuleApi()]);
|
|
19
8
|
if (!hasModuleApi) {
|
|
20
9
|
logger.error('Module API not available while mandatory');
|
|
21
10
|
process.exit(1);
|
|
@@ -25,16 +14,6 @@ export class RendererBuilder {
|
|
|
25
14
|
if (hasActionParseApi) {
|
|
26
15
|
return new ActionParseRenderer();
|
|
27
16
|
}
|
|
28
|
-
else if (hasWikimediaDesktopApi) {
|
|
29
|
-
// Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
|
|
30
|
-
return new WikimediaDesktopRenderer();
|
|
31
|
-
}
|
|
32
|
-
else if (hasVisualEditorApi) {
|
|
33
|
-
return new VisualEditorRenderer();
|
|
34
|
-
}
|
|
35
|
-
else if (hasRestApi) {
|
|
36
|
-
return new RestApiRenderer();
|
|
37
|
-
}
|
|
38
17
|
else {
|
|
39
18
|
logger.error('No available desktop renderer.');
|
|
40
19
|
process.exit(1);
|
|
@@ -44,29 +23,15 @@ export class RendererBuilder {
|
|
|
44
23
|
if (hasActionParseApi) {
|
|
45
24
|
return new ActionParseRenderer();
|
|
46
25
|
}
|
|
47
|
-
else
|
|
48
|
-
|
|
26
|
+
else {
|
|
27
|
+
logger.error('No available mobile renderer.');
|
|
28
|
+
process.exit(1);
|
|
49
29
|
}
|
|
50
|
-
logger.error('No available mobile renderer.');
|
|
51
|
-
process.exit(1);
|
|
52
30
|
break; // prettier-ignore
|
|
53
31
|
case 'auto':
|
|
54
32
|
if (hasActionParseApi) {
|
|
55
33
|
return new ActionParseRenderer();
|
|
56
34
|
}
|
|
57
|
-
else if (hasWikimediaDesktopApi) {
|
|
58
|
-
// Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
|
|
59
|
-
return new WikimediaDesktopRenderer();
|
|
60
|
-
}
|
|
61
|
-
else if (hasVisualEditorApi) {
|
|
62
|
-
return new VisualEditorRenderer();
|
|
63
|
-
}
|
|
64
|
-
else if (hasRestApi) {
|
|
65
|
-
return new RestApiRenderer();
|
|
66
|
-
}
|
|
67
|
-
else if (hasWikimediaMobileApi) {
|
|
68
|
-
return new WikimediaMobileRenderer();
|
|
69
|
-
}
|
|
70
35
|
else {
|
|
71
36
|
logger.error('No render available at all.');
|
|
72
37
|
process.exit(1);
|
|
@@ -74,12 +39,12 @@ export class RendererBuilder {
|
|
|
74
39
|
break; // prettier-ignore
|
|
75
40
|
case 'specific':
|
|
76
41
|
// renderName argument is required for 'specific' mode
|
|
77
|
-
return this.handleSpecificRender(renderName,
|
|
42
|
+
return this.handleSpecificRender(renderName, hasActionParseApi);
|
|
78
43
|
default:
|
|
79
44
|
throw new Error(`Unknown render: ${renderType}`);
|
|
80
45
|
}
|
|
81
46
|
}
|
|
82
|
-
handleSpecificRender(renderName,
|
|
47
|
+
handleSpecificRender(renderName, hasActionParseApi) {
|
|
83
48
|
// renderName argument is required for 'specific' mode
|
|
84
49
|
switch (renderName) {
|
|
85
50
|
case 'ActionParse':
|
|
@@ -89,34 +54,6 @@ export class RendererBuilder {
|
|
|
89
54
|
logger.error('Cannot create an instance of ActionParse renderer.');
|
|
90
55
|
process.exit(1);
|
|
91
56
|
break; // prettier-ignore
|
|
92
|
-
case 'WikimediaDesktop':
|
|
93
|
-
if (hasWikimediaDesktopApi) {
|
|
94
|
-
return new WikimediaDesktopRenderer();
|
|
95
|
-
}
|
|
96
|
-
logger.error('Cannot create an instance of WikimediaDesktop renderer.');
|
|
97
|
-
process.exit(1);
|
|
98
|
-
break; // prettier-ignore
|
|
99
|
-
case 'VisualEditor':
|
|
100
|
-
if (hasVisualEditorApi) {
|
|
101
|
-
return new VisualEditorRenderer();
|
|
102
|
-
}
|
|
103
|
-
logger.error('Cannot create an instance of VisualEditor renderer.');
|
|
104
|
-
process.exit(1);
|
|
105
|
-
break; // prettier-ignore
|
|
106
|
-
case 'WikimediaMobile':
|
|
107
|
-
if (hasWikimediaMobileApi) {
|
|
108
|
-
return new WikimediaMobileRenderer();
|
|
109
|
-
}
|
|
110
|
-
logger.error('No available mobile renderer.');
|
|
111
|
-
process.exit(1);
|
|
112
|
-
break; // prettier-ignore
|
|
113
|
-
case 'RestApi':
|
|
114
|
-
if (hasRestApi) {
|
|
115
|
-
return new RestApiRenderer();
|
|
116
|
-
}
|
|
117
|
-
logger.error('Cannot create an instance of RestApi renderer.');
|
|
118
|
-
process.exit(1);
|
|
119
|
-
break; // prettier-ignore
|
|
120
57
|
default:
|
|
121
58
|
throw new Error(`Unknown renderName for specific mode: ${renderName}`);
|
|
122
59
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"renderer.builder.js","sourceRoot":"","sources":["../../src/renderers/renderer.builder.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAA;AAEzC,OAAO,
|
|
1
|
+
{"version":3,"file":"renderer.builder.js","sourceRoot":"","sources":["../../src/renderers/renderer.builder.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAA;AAEzC,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,MAAM,OAAO,eAAe;IACnB,KAAK,CAAC,cAAc,CAAC,OAA+B;QACzD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAA;QAE1C,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;QAEtH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAA;YACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,SAAS;gBACZ,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;oBAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,QAAQ;gBACX,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;oBAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,MAAM;gBACT,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;oBAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,UAAU;gBACb,sDAAsD;gBACtD,OAAO,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;YACjE;gBACE,MAAM,IAAI,KAAK,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,UAAkB,EAAE,iBAA0B;QACzE,sDAAsD;QACtD,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,aAAa;gBAChB,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B;gBACE,MAAM,IAAI,KAAK,CAAC,yCAAyC,UAAU,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { Renderer, renderName } from './abstract.renderer.js';
|
|
2
2
|
declare class RenderingContext {
|
|
3
3
|
private static instance;
|
|
4
|
-
|
|
5
|
-
articlesRenderer: Renderer;
|
|
4
|
+
pagesRenderer: Renderer;
|
|
6
5
|
static getInstance(): RenderingContext;
|
|
7
|
-
createRenderers(forceRender: renderName | null
|
|
6
|
+
createRenderers(forceRender: renderName | null): Promise<void>;
|
|
8
7
|
}
|
|
9
8
|
declare const _default: RenderingContext;
|
|
10
9
|
export default _default;
|
|
@@ -3,34 +3,26 @@ import { RendererBuilder } from './renderer.builder.js';
|
|
|
3
3
|
import * as logger from '../Logger.js';
|
|
4
4
|
class RenderingContext {
|
|
5
5
|
static instance;
|
|
6
|
-
|
|
7
|
-
articlesRenderer;
|
|
6
|
+
pagesRenderer;
|
|
8
7
|
static getInstance() {
|
|
9
8
|
if (!RenderingContext.instance) {
|
|
10
9
|
RenderingContext.instance = new RenderingContext();
|
|
11
10
|
}
|
|
12
11
|
return RenderingContext.instance;
|
|
13
12
|
}
|
|
14
|
-
async createRenderers(forceRender
|
|
13
|
+
async createRenderers(forceRender) {
|
|
15
14
|
const rendererBuilder = new RendererBuilder();
|
|
16
15
|
if (forceRender) {
|
|
17
|
-
|
|
18
|
-
const renderer = await rendererBuilder.createRenderer({
|
|
16
|
+
this.pagesRenderer = await rendererBuilder.createRenderer({
|
|
19
17
|
renderType: 'specific',
|
|
20
18
|
renderName: forceRender,
|
|
21
19
|
});
|
|
22
|
-
this.mainPageRenderer = renderer;
|
|
23
|
-
this.articlesRenderer = renderer;
|
|
24
20
|
}
|
|
25
21
|
else {
|
|
26
|
-
this.
|
|
27
|
-
this.articlesRenderer = await rendererBuilder.createRenderer({
|
|
28
|
-
renderType: hasWikimediaMobileApi ? 'mobile' : 'auto',
|
|
29
|
-
});
|
|
22
|
+
this.pagesRenderer = await rendererBuilder.createRenderer({ renderType: 'auto' });
|
|
30
23
|
}
|
|
31
|
-
logger.
|
|
32
|
-
|
|
33
|
-
Downloader.setUrlsDirectors(this.mainPageRenderer, this.articlesRenderer);
|
|
24
|
+
logger.info(`Using ${this.pagesRenderer.constructor.name} for pages renderer`);
|
|
25
|
+
Downloader.setUrlsDirectors(this.pagesRenderer);
|
|
34
26
|
}
|
|
35
27
|
}
|
|
36
28
|
const instance = RenderingContext.getInstance();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rendering.context.js","sourceRoot":"","sources":["../../src/renderers/rendering.context.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,kBAAkB,CAAA;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AAEtC,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAElC,
|
|
1
|
+
{"version":3,"file":"rendering.context.js","sourceRoot":"","sources":["../../src/renderers/rendering.context.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,kBAAkB,CAAA;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AAEtC,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAElC,aAAa,CAAU;IAEvB,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YAC/B,gBAAgB,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;QACpD,CAAC;QACD,OAAO,gBAAgB,CAAC,QAAQ,CAAA;IAClC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,WAA8B;QACzD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAE7C,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC,aAAa,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBACxD,UAAU,EAAE,UAAU;gBACtB,UAAU,EAAE,WAAW;aACxB,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,aAAa,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAA;QACnF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,qBAAqB,CAAC,CAAA;QAC9E,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACjD,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAA;AAC/C,eAAe,QAA4B,CAAA"}
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
import * as logger from './Logger.js';
|
|
2
1
|
export declare function sanitize_all(argv: any): Promise<void>;
|
|
3
2
|
export declare function check_all(argv: any): Promise<void>;
|
|
4
|
-
export declare function sanitizeWikiPath(mwWikiPath?: string): string;
|
|
5
3
|
export declare function sanitizeApiPathParam(apiPathParam: string): string;
|
|
6
4
|
export declare function sanitizeStringMaxLength(text: string, key: string, length: number): void;
|
|
7
|
-
export declare function
|
|
8
|
-
export declare function
|
|
5
|
+
export declare function sanitize_logLevel(logLevel: string | undefined): void;
|
|
6
|
+
export declare function sanitize_customName_langVariant(customZimName: string, customZimFilename: string, langVariant: string | string[]): void;
|
|
7
|
+
export declare function sanitizePageSelection(pageList: string, addNamespaces: string, onlyNamespaces: string): void;
|
|
9
8
|
export declare function sanitizeDoubleUsedParameters(options: object): void;
|
|
10
9
|
export declare function sanitize_speed(_speed: any): void;
|
|
10
|
+
export declare function sanitizeCategoriesPageSize(categoriesPageSize: any): void;
|
|
11
|
+
export declare function sanitize_javaScript_addModules(javaScript: any, addModules: any): void;
|
|
11
12
|
export declare function check_mwApiReachability(mwUrl: string, mwActionApiPath: string): Promise<void>;
|
|
13
|
+
export declare function sanitize_mwUrl(mwUrl: string): void;
|
|
12
14
|
export declare function sanitize_adminEmail(adminEmail: any): void;
|
|
13
15
|
export declare function check_redis(): Promise<void>;
|
|
14
16
|
export declare function check_customZimFavicon(customZimFavicon: any): Promise<void>;
|
|
@@ -25,4 +27,5 @@ export declare function check_s3(optimisationCacheUrl: string, insecure: boolean
|
|
|
25
27
|
* path resolution.
|
|
26
28
|
*/
|
|
27
29
|
export declare function sanitize_customFlavour(customFlavour: string): string;
|
|
30
|
+
export declare function sanitize_mathJax(mathJaxSource: any, mathJaxConfig: any, mathJaxEntryPoint: any, mathJaxAllPages: any): void;
|
|
28
31
|
export declare function sanitize_forceRender(renderName: string): string;
|
package/lib/sanitize-argument.js
CHANGED
|
@@ -13,18 +13,20 @@ import MediaWiki from './MediaWiki.js';
|
|
|
13
13
|
import { config } from './config.js';
|
|
14
14
|
const __filename = fileURLToPath(import.meta.url);
|
|
15
15
|
const __dirname = path.dirname(__filename);
|
|
16
|
-
const parametersWithArrayType = ['format'];
|
|
16
|
+
const parametersWithArrayType = ['format', 'langVariant'];
|
|
17
17
|
// Perform "static check" of arguments not depending on external connectivity
|
|
18
18
|
// Stack is not yet sufficiently ready to perform live external connectivity checks
|
|
19
19
|
export async function sanitize_all(argv) {
|
|
20
20
|
// extracting all arguments
|
|
21
|
-
const {
|
|
21
|
+
const { pageList, addNamespaces, onlyNamespaces, speed, adminEmail, mwUrl, logLevel, customZimLongDescription, customZimDescription, customZimName, customZimFilename, forceRender, javaScript, addModules, langVariant, mwActionApiPath, mwModulePath, categoriesPageSize, mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages, } = argv;
|
|
22
22
|
sanitizeDoubleUsedParameters(argv);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
sanitize_customName_langVariant(customZimName, customZimFilename, langVariant);
|
|
24
|
+
sanitizePageSelection(pageList, addNamespaces, onlyNamespaces);
|
|
25
|
+
// sanitizing logLevel
|
|
26
|
+
sanitize_logLevel(logLevel);
|
|
26
27
|
// sanitizing speed
|
|
27
28
|
sanitize_speed(speed);
|
|
29
|
+
sanitizeCategoriesPageSize(categoriesPageSize);
|
|
28
30
|
// sanitizing longDescription
|
|
29
31
|
sanitizeStringMaxLength(customZimLongDescription, 'customZimLongDescription', 4000);
|
|
30
32
|
// sanitizing description
|
|
@@ -36,18 +38,8 @@ export async function sanitize_all(argv) {
|
|
|
36
38
|
throw new Error('Custom Flavour not found');
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
|
-
// sanitizing
|
|
40
|
-
|
|
41
|
-
argv.mwWikiPath = sanitizeWikiPath(mwWikiPath);
|
|
42
|
-
}
|
|
43
|
-
// sanitizing mwIndexPhpPath
|
|
44
|
-
if (mwIndexPhpPath) {
|
|
45
|
-
argv.mwIndexPhpPath = sanitizeApiPathParam(mwIndexPhpPath);
|
|
46
|
-
}
|
|
47
|
-
// sanitizing mwRestApiPath
|
|
48
|
-
if (mwRestApiPath) {
|
|
49
|
-
argv.mwRestApiPath = sanitizeApiPathParam(mwRestApiPath);
|
|
50
|
-
}
|
|
41
|
+
// sanitizing javaScript
|
|
42
|
+
sanitize_javaScript_addModules(javaScript, addModules);
|
|
51
43
|
// sanitizing mwActionApiPath
|
|
52
44
|
if (mwActionApiPath) {
|
|
53
45
|
argv.mwActionApiPath = sanitizeApiPathParam(mwActionApiPath);
|
|
@@ -58,14 +50,24 @@ export async function sanitize_all(argv) {
|
|
|
58
50
|
}
|
|
59
51
|
// sanitize Custom Main Page
|
|
60
52
|
if (argv.customMainPage) {
|
|
61
|
-
argv.customMainPage = argv.customMainPage.replace(/
|
|
53
|
+
argv.customMainPage = argv.customMainPage.replace(/_/g, ' ');
|
|
54
|
+
}
|
|
55
|
+
if (!mwUrl || mwUrl.trim() === '') {
|
|
56
|
+
throw new Error('Option --mwUrl is required and cannot be empty');
|
|
62
57
|
}
|
|
58
|
+
if (!adminEmail || adminEmail.trim() === '') {
|
|
59
|
+
throw new Error('Option --adminEmail is required and cannot be empty');
|
|
60
|
+
}
|
|
61
|
+
//sanitizing mwUrl
|
|
62
|
+
sanitize_mwUrl(mwUrl);
|
|
63
63
|
// sanitizing adminEmail
|
|
64
64
|
sanitize_adminEmail(adminEmail);
|
|
65
65
|
// sanitizing renderer
|
|
66
66
|
if (forceRender) {
|
|
67
67
|
sanitize_forceRender(forceRender);
|
|
68
68
|
}
|
|
69
|
+
// sanitizing MathJax options
|
|
70
|
+
sanitize_mathJax(mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages);
|
|
69
71
|
}
|
|
70
72
|
// perform live checks of arguments needing a connection "somewhere"
|
|
71
73
|
export async function check_all(argv) {
|
|
@@ -83,18 +85,6 @@ export async function check_all(argv) {
|
|
|
83
85
|
await check_customZimFavicon(customZimFavicon);
|
|
84
86
|
}
|
|
85
87
|
}
|
|
86
|
-
export function sanitizeWikiPath(mwWikiPath = '') {
|
|
87
|
-
mwWikiPath = sanitizeApiPathParam(mwWikiPath);
|
|
88
|
-
// Remove trailing $1 since we don't need it
|
|
89
|
-
if (mwWikiPath?.endsWith('$1')) {
|
|
90
|
-
mwWikiPath = mwWikiPath.substring(0, mwWikiPath.length - 3);
|
|
91
|
-
}
|
|
92
|
-
// Make sure wikiPath always has forward slash at the end for the correct URL building
|
|
93
|
-
if (!mwWikiPath?.endsWith('/')) {
|
|
94
|
-
mwWikiPath += '/';
|
|
95
|
-
}
|
|
96
|
-
return mwWikiPath;
|
|
97
|
-
}
|
|
98
88
|
export function sanitizeApiPathParam(apiPathParam) {
|
|
99
89
|
if (!apiPathParam) {
|
|
100
90
|
return;
|
|
@@ -110,14 +100,28 @@ export function sanitizeStringMaxLength(text, key, length) {
|
|
|
110
100
|
throw new Error(`${key} should be less than ${length} characters.`);
|
|
111
101
|
}
|
|
112
102
|
}
|
|
113
|
-
export function
|
|
114
|
-
if (
|
|
115
|
-
throw new Error(`"${
|
|
103
|
+
export function sanitize_logLevel(logLevel) {
|
|
104
|
+
if (logLevel !== undefined && !logger.logLevels.includes(logLevel)) {
|
|
105
|
+
throw new Error(`"${logLevel}" is not a valid value for option --log-level. It should be one of [${logger.logLevels.join(', ')}].`);
|
|
116
106
|
}
|
|
117
107
|
}
|
|
118
|
-
export function
|
|
119
|
-
if (
|
|
120
|
-
throw new Error('
|
|
108
|
+
export function sanitize_customName_langVariant(customZimName, customZimFilename, langVariant) {
|
|
109
|
+
if (customZimName && Array.isArray(langVariant) && !customZimName.includes('{lang_or_variant}')) {
|
|
110
|
+
throw new Error('option --customZimName needs to include the {lang_or_variant} placeholder when using multiple --langVariant');
|
|
111
|
+
}
|
|
112
|
+
if (customZimFilename && Array.isArray(langVariant) && !customZimFilename.includes('{lang_or_variant}')) {
|
|
113
|
+
throw new Error('option --customZimFilename needs to include the {lang_or_variant} placeholder when using multiple --langVariant');
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
export function sanitizePageSelection(pageList, addNamespaces, onlyNamespaces) {
|
|
117
|
+
if (pageList && addNamespaces) {
|
|
118
|
+
throw new Error('options --pageList and --addNamespaces cannot be used together');
|
|
119
|
+
}
|
|
120
|
+
if (pageList && onlyNamespaces) {
|
|
121
|
+
throw new Error('options --pageList and --onlyNamespaces cannot be used together');
|
|
122
|
+
}
|
|
123
|
+
if (addNamespaces && onlyNamespaces) {
|
|
124
|
+
throw new Error('options --addNamespaces and --onlyNamespaces cannot be used together');
|
|
121
125
|
}
|
|
122
126
|
}
|
|
123
127
|
export function sanitizeDoubleUsedParameters(options) {
|
|
@@ -129,21 +133,64 @@ export function sanitizeDoubleUsedParameters(options) {
|
|
|
129
133
|
}
|
|
130
134
|
}
|
|
131
135
|
export function sanitize_speed(_speed) {
|
|
132
|
-
if (_speed &&
|
|
133
|
-
|
|
136
|
+
if (_speed !== undefined && _speed !== null) {
|
|
137
|
+
const numSpeed = Number(_speed);
|
|
138
|
+
if (isNaN(numSpeed) || numSpeed <= 0) {
|
|
139
|
+
throw new Error('speed must be a positive number. Values >= 1 set the number of parallel workers (integer). Values < 1 slow down the scraper.');
|
|
140
|
+
}
|
|
141
|
+
if (numSpeed >= 1 && !Number.isInteger(numSpeed)) {
|
|
142
|
+
throw new Error('speed values >= 1 must be integers (number of parallel workers).');
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
export function sanitizeCategoriesPageSize(categoriesPageSize) {
|
|
147
|
+
if (categoriesPageSize !== undefined && categoriesPageSize !== null) {
|
|
148
|
+
const numCategoriesPageSize = Number(categoriesPageSize);
|
|
149
|
+
if (isNaN(numCategoriesPageSize)) {
|
|
150
|
+
throw new Error('categoriesPageSize must be a number.');
|
|
151
|
+
}
|
|
152
|
+
if (numCategoriesPageSize <= 0) {
|
|
153
|
+
throw new Error('categoriesPageSize must be a positive.');
|
|
154
|
+
}
|
|
155
|
+
if (!Number.isInteger(numCategoriesPageSize)) {
|
|
156
|
+
throw new Error('categoriesPageSize must be integer.');
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
export function sanitize_javaScript_addModules(javaScript, addModules) {
|
|
161
|
+
if (javaScript && !['none', 'trusted', 'all'].includes(javaScript)) {
|
|
162
|
+
throw new Error(`"${javaScript}" is not a valid value for option javaScript. It should be empty or one of [none, trusted, all].`);
|
|
163
|
+
}
|
|
164
|
+
if (javaScript === 'none' && addModules) {
|
|
165
|
+
throw new Error('options --javaScript=none and --addModules cannot be used together');
|
|
134
166
|
}
|
|
135
167
|
}
|
|
136
168
|
export async function check_mwApiReachability(mwUrl, mwActionApiPath) {
|
|
137
169
|
// We are building an API query "by-hand" because we've not yet initialized the whole URL director / URL builder stack
|
|
138
170
|
const apiQueryUrl = `${new URL(mwActionApiPath || MediaWiki.actionApiPath, mwUrl).toString()}?action=query&format=json&formatversion=2&maxlag=${config.defaults.maxlag}`;
|
|
139
171
|
const value = await Downloader.get(apiQueryUrl).catch((err) => {
|
|
140
|
-
throw new Error(`Mediawiki API is not reachable with ${apiQueryUrl}\n${err}
|
|
172
|
+
throw new Error(`Mediawiki API is not reachable with ${apiQueryUrl}\n${err}\nYou may need to pass a (different) value for --mwActionApiPath, see the FAQ.`);
|
|
141
173
|
});
|
|
142
174
|
const contentType = value.headers['content-type'];
|
|
143
175
|
if (!contentType.includes('application/json')) {
|
|
144
176
|
throw new Error(`Mediawiki API is returning '${contentType}' Content-Type instead of 'application/json' with ${apiQueryUrl}`);
|
|
145
177
|
}
|
|
146
178
|
}
|
|
179
|
+
export function sanitize_mwUrl(mwUrl) {
|
|
180
|
+
let url;
|
|
181
|
+
try {
|
|
182
|
+
url = new URL(mwUrl);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
throw new Error(`mwUrl is not a valid URL.`);
|
|
186
|
+
}
|
|
187
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
188
|
+
throw new Error(`mwUrl must use http or https protocol, got '${url.protocol.replace(':', '')}'.`);
|
|
189
|
+
}
|
|
190
|
+
if (url.pathname !== '/') {
|
|
191
|
+
throw new Error(`mwUrl must not contain a path (only the hostname is expected), got '${url.pathname}'. See the FAQ.`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
147
194
|
export function sanitize_adminEmail(adminEmail) {
|
|
148
195
|
if (!isValidEmail(adminEmail)) {
|
|
149
196
|
throw new Error(`Admin email [${adminEmail}] is not valid.`);
|
|
@@ -151,7 +198,7 @@ export function sanitize_adminEmail(adminEmail) {
|
|
|
151
198
|
}
|
|
152
199
|
export async function check_redis() {
|
|
153
200
|
await RedisStore.connect(false);
|
|
154
|
-
logger.
|
|
201
|
+
logger.info('closing sanitize redis DB');
|
|
155
202
|
await RedisStore.close();
|
|
156
203
|
}
|
|
157
204
|
export async function check_customZimFavicon(customZimFavicon) {
|
|
@@ -174,7 +221,7 @@ export async function check_s3(optimisationCacheUrl, insecure) {
|
|
|
174
221
|
const s3Url = (s3UrlObj.protocol || 'https:') + '//' + (s3UrlObj.host || '') + (s3UrlObj.pathname || '');
|
|
175
222
|
const s3Obj = new S3(s3Url, s3UrlObj.searchParams, 1000 * 60, insecure);
|
|
176
223
|
await s3Obj.initialise().then(() => {
|
|
177
|
-
logger.
|
|
224
|
+
logger.info('Successfully logged in S3');
|
|
178
225
|
});
|
|
179
226
|
}
|
|
180
227
|
}
|
|
@@ -195,6 +242,17 @@ export function sanitize_customFlavour(customFlavour) {
|
|
|
195
242
|
return fs.existsSync(possiblePath);
|
|
196
243
|
}) || null);
|
|
197
244
|
}
|
|
245
|
+
export function sanitize_mathJax(mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages) {
|
|
246
|
+
if (mathJaxConfig && !mathJaxSource) {
|
|
247
|
+
throw new Error('Option --mathJaxConfig requires --mathJaxSource to also be set');
|
|
248
|
+
}
|
|
249
|
+
if (mathJaxEntryPoint && !mathJaxSource) {
|
|
250
|
+
throw new Error('Option --mathJaxEntryPoint requires --mathJaxSource to also be set');
|
|
251
|
+
}
|
|
252
|
+
if (mathJaxAllPages && !mathJaxSource) {
|
|
253
|
+
throw new Error('Option --mathJaxAllPages requires --mathJaxSource to also be set');
|
|
254
|
+
}
|
|
255
|
+
}
|
|
198
256
|
export function sanitize_forceRender(renderName) {
|
|
199
257
|
const checkRenderName = (arr, val) => {
|
|
200
258
|
return arr.some((arrVal) => val === arrVal);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sanitize-argument.js","sourceRoot":"","sources":["../src/sanitize-argument.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,KAAK,CAAA;AACxC,OAAO,UAAU,MAAM,MAAM,CAAA;AAC7B,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEpC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;AAC1C,MAAM,uBAAuB,GAAG,CAAC,QAAQ,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"sanitize-argument.js","sourceRoot":"","sources":["../src/sanitize-argument.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,KAAK,CAAA;AACxC,OAAO,UAAU,MAAM,MAAM,CAAA;AAC7B,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEpC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;AAC1C,MAAM,uBAAuB,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;AAEzD,6EAA6E;AAC7E,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAS;IAC1C,2BAA2B;IAC3B,MAAM,EACJ,QAAQ,EACR,aAAa,EACb,cAAc,EACd,KAAK,EACL,UAAU,EACV,KAAK,EACL,QAAQ,EACR,wBAAwB,EACxB,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,GAAG,IAAI,CAAA;IAER,4BAA4B,CAAC,IAAI,CAAC,CAAA;IAElC,+BAA+B,CAAC,aAAa,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAA;IAE9E,qBAAqB,CAAC,QAAQ,EAAE,aAAa,EAAE,cAAc,CAAC,CAAA;IAE9D,sBAAsB;IACtB,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAE3B,mBAAmB;IACnB,cAAc,CAAC,KAAK,CAAC,CAAA;IAErB,0BAA0B,CAAC,kBAAkB,CAAC,CAAA;IAE9C,6BAA6B;IAC7B,uBAAuB,CAAC,wBAAwB,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEnF,yBAAyB;IACzB,uBAAuB,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,EAAE,CAAC,CAAA;IAEzE,4BAA4B;IAC5B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAC/D,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,8BAA8B,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;IAEtD,6BAA6B;IAC7B,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAA;IAC9D,CAAC;IAED,0BAA0B;IAC1B,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAA;IACxD,CAAC;IAED,4BAA4B;IAC5B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAC9D,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,kBAAkB;IAClB,cAAc,CAAC,KAAK,CAAC,CAAA;IACrB,wBAAwB;IACxB,mBAAmB,CAAC,UAAU,CAAC,CAAA;IAE/B,sBAAsB;IACtB,IAAI,WAAW,EAAE,CAAC;QAChB,oBAAoB,CAAC,WAAW,CAAC,CAAA;IACnC,CAAC;IAED,6BAA6B;IAC7B,gBAAgB,CAAC,aAAa,EAAE,aAAa,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAA;AACpF,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAS;IACvC,gCAAgC;IAChC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAA;IAEzF,mCAAmC;IACnC,MAAM,uBAAuB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;IAErD,wBAAwB;IACxB,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAA;IAE9C,2BAA2B;IAC3B,6CAA6C;IAC7C,MAAM,WAAW,EAAE,CAAA;IAEnB,wCAAwC;IACxC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,sBAAsB,CAAC,gBAAgB,CAAC,CAAA;IAChD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACvD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAM;IACR,CAAC;IAED,+CAA+C;IAC/C,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,YAAY,GAAG,GAAG,GAAG,YAAY,CAAA;IACnC,CAAC;IAED,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,GAAW,EAAE,MAAc;IAC/E,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wBAAwB,MAAM,cAAc,CAAC,CAAA;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAA4B;IAC5D,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAA2B,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,IAAI,QAAQ,uEAAuE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrI,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,aAAqB,EAAE,iBAAyB,EAAE,WAA8B;IAC9H,IAAI,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAA;IAChI,CAAC;IACD,IAAI,iBAAiB,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACxG,MAAM,IAAI,KAAK,CAAC,iHAAiH,CAAC,CAAA;IACpI,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,aAAqB,EAAE,cAAsB;IACnG,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,QAAQ,IAAI,cAAc,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAA;IACpF,CAAC;IACD,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAA;IACzF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,OAAe;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;IACxD,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/D,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACpH,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,yBAAyB,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAW;IACxC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;QAC/B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8HAA8H,CAAC,CAAA;QACjJ,CAAC;QACD,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;QACrF,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,kBAAuB;IAChE,IAAI,kBAAkB,KAAK,SAAS,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;QACpE,MAAM,qBAAqB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACxD,IAAI,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACzD,CAAC;QACD,IAAI,qBAAqB,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAC3D,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,UAAe,EAAE,UAAe;IAC7E,IAAI,UAAU,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,IAAI,UAAU,kGAAkG,CAAC,CAAA;IACnI,CAAC;IACD,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAAa,EAAE,eAAuB;IAClF,sHAAsH;IACtH,MAAM,WAAW,GAAG,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,oDAAoD,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;IACxK,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5D,MAAM,IAAI,KAAK,CAAC,uCAAuC,WAAW,KAAK,GAAG,gFAAgF,CAAC,CAAA;IAC7J,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACjD,IAAI,CAAE,WAAsB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,+BAA+B,WAAW,qDAAqD,WAAW,EAAE,CAAC,CAAA;IAC/H,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IACnG,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,uEAAuE,GAAG,CAAC,QAAQ,iBAAiB,CAAC,CAAA;IACvH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAe;IACjD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gBAAgB,UAAU,iBAAiB,CAAC,CAAA;IAC9D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/B,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;IACxC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,gBAAqB;IAChE,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,eAAe,EAAE,CAAC;QACpB,wDAAwD;QACxD,MAAM,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,gBAAgB,GAAG,CAAC,CAAA;QACrF,CAAC,CAAC,CAAA;IACJ,CAAC;SAAM,CAAC;QACN,EAAE,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,oBAA4B,EAAE,QAAiB;IAC5E,gBAAgB;IAChB,IAAI,oBAAoB,EAAE,CAAC;QACzB,iDAAiD;QACjD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAA;QAC9C,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QACxG,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAA;QACvE,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AAEH,MAAM,UAAU,sBAAsB,CAAC,aAAqB;IAC1D,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;IACzE,OAAO,CACL,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,aAAa,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,YAAY;QAC7I,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;IACpC,CAAC,CAAC,IAAI,IAAI,CACX,CAAA;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,aAAkB,EAAE,aAAkB,EAAE,iBAAsB,EAAE,eAAoB;IACnH,IAAI,aAAa,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,iBAAiB,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;IACD,IAAI,eAAe,IAAI,CAAC,aAAa,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;IACrF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,MAAM,eAAe,GAAG,CAAC,GAAa,EAAE,GAAW,EAAE,EAAE;QACrD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAA;IAC7C,CAAC,CAAA;IACD,IAAI,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC,EAAE,CAAC;QAChD,OAAO,UAAU,CAAA;IACnB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAA;AACvD,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Creator } from '@openzim/libzim';
|
|
2
|
+
import { Dump } from '../Dump.js';
|
|
3
|
+
declare class FileManager {
|
|
4
|
+
private static instance;
|
|
5
|
+
static getInstance(): FileManager;
|
|
6
|
+
private hosts;
|
|
7
|
+
private get filesToDownloadXPath();
|
|
8
|
+
private constructor();
|
|
9
|
+
/**
|
|
10
|
+
* Reset internal state for a new dump cycle.
|
|
11
|
+
*/
|
|
12
|
+
reset(): void;
|
|
13
|
+
/**
|
|
14
|
+
* Add a single file to download. Checks dedup via filesToDownloadXPath.
|
|
15
|
+
* On first add: writes to both filesToDownloadXPath and host queue.
|
|
16
|
+
* On resolution upgrade: updates filesToDownloadXPath only (queue already has the path).
|
|
17
|
+
*
|
|
18
|
+
* This method is typically called multiple times when the file is an image and the scraper detects
|
|
19
|
+
* a second use of an image already seen but with a higher resolution. Could be that we call this
|
|
20
|
+
* method twice with different urls / data for other usage in the future.
|
|
21
|
+
*/
|
|
22
|
+
addFileToProcess(path: string, detail: FileDetail): Promise<void>;
|
|
23
|
+
private _addFileToProcess;
|
|
24
|
+
/**
|
|
25
|
+
* Batch version of addFileToProcess. Uses getMany for efficiency.
|
|
26
|
+
*/
|
|
27
|
+
addManyFilesToProcess(files: KVS<FileDetail>): Promise<void>;
|
|
28
|
+
_addManyFilesToProcess(files: KVS<FileDetail>): Promise<void>;
|
|
29
|
+
private pushToHostQueue;
|
|
30
|
+
/**
|
|
31
|
+
* Start download workers. Queues are already populated via addFileToProcess/addManyFilesToProcess.
|
|
32
|
+
*/
|
|
33
|
+
startDownloading(zimCreator: Creator, dump: Dump): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
declare const _default: FileManager;
|
|
36
|
+
export default _default;
|