mwoffliner 1.17.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +355 -62
  2. package/docs/functional_architecture.md +34 -38
  3. package/extensions/wiktionary_fr.js +18 -17
  4. package/jest.config.cjs +2 -2
  5. package/lib/DOMUtils.js +0 -1
  6. package/lib/DOMUtils.js.map +1 -1
  7. package/lib/Downloader.d.ts +39 -31
  8. package/lib/Downloader.js +418 -349
  9. package/lib/Downloader.js.map +1 -1
  10. package/lib/Dump.d.ts +24 -10
  11. package/lib/Dump.js +95 -50
  12. package/lib/Dump.js.map +1 -1
  13. package/lib/Gadgets.d.ts +2 -1
  14. package/lib/Gadgets.js +7 -5
  15. package/lib/Gadgets.js.map +1 -1
  16. package/lib/Logger.d.ts +6 -6
  17. package/lib/Logger.js +33 -33
  18. package/lib/Logger.js.map +1 -1
  19. package/lib/MediaWiki.d.ts +14 -31
  20. package/lib/MediaWiki.js +119 -172
  21. package/lib/MediaWiki.js.map +1 -1
  22. package/lib/RedisStore.d.ts +3 -3
  23. package/lib/RedisStore.js +21 -23
  24. package/lib/RedisStore.js.map +1 -1
  25. package/lib/S3.js +1 -1
  26. package/lib/S3.js.map +1 -1
  27. package/lib/Templates.d.ts +3 -10
  28. package/lib/Templates.js +5 -14
  29. package/lib/Templates.js.map +1 -1
  30. package/lib/cli.js +7 -6
  31. package/lib/cli.js.map +1 -1
  32. package/lib/config.d.ts +10 -16
  33. package/lib/config.js +33 -55
  34. package/lib/config.js.map +1 -1
  35. package/lib/error.manager.d.ts +2 -2
  36. package/lib/error.manager.js +42 -49
  37. package/lib/error.manager.js.map +1 -1
  38. package/lib/i18n.d.ts +2 -0
  39. package/lib/i18n.js +53 -0
  40. package/lib/i18n.js.map +1 -0
  41. package/lib/mutex.d.ts +2 -1
  42. package/lib/mutex.js +2 -1
  43. package/lib/mutex.js.map +1 -1
  44. package/lib/mwoffliner.lib.js +370 -222
  45. package/lib/mwoffliner.lib.js.map +1 -1
  46. package/lib/parameterList.d.ts +20 -8
  47. package/lib/parameterList.js +30 -18
  48. package/lib/parameterList.js.map +1 -1
  49. package/lib/renderers/abstract.renderer.d.ts +48 -55
  50. package/lib/renderers/abstract.renderer.js +377 -98
  51. package/lib/renderers/abstract.renderer.js.map +1 -1
  52. package/lib/renderers/action-parse.renderer.d.ts +22 -3
  53. package/lib/renderers/action-parse.renderer.js +123 -81
  54. package/lib/renderers/action-parse.renderer.js.map +1 -1
  55. package/lib/renderers/renderer.builder.js +6 -69
  56. package/lib/renderers/renderer.builder.js.map +1 -1
  57. package/lib/renderers/rendering.context.d.ts +2 -3
  58. package/lib/renderers/rendering.context.js +6 -14
  59. package/lib/renderers/rendering.context.js.map +1 -1
  60. package/lib/sanitize-argument.d.ts +7 -4
  61. package/lib/sanitize-argument.js +99 -41
  62. package/lib/sanitize-argument.js.map +1 -1
  63. package/lib/util/FileManager.d.ts +36 -0
  64. package/lib/util/FileManager.js +246 -0
  65. package/lib/util/FileManager.js.map +1 -0
  66. package/lib/util/RedisKvs.js +4 -4
  67. package/lib/util/RedisKvs.js.map +1 -1
  68. package/lib/util/RedisQueue.js +1 -1
  69. package/lib/util/RedisQueue.js.map +1 -1
  70. package/lib/util/builders/url/action-parse.director.d.ts +2 -2
  71. package/lib/util/builders/url/action-parse.director.js +7 -7
  72. package/lib/util/builders/url/action-parse.director.js.map +1 -1
  73. package/lib/util/builders/url/api.director.d.ts +2 -4
  74. package/lib/util/builders/url/api.director.js +10 -17
  75. package/lib/util/builders/url/api.director.js.map +1 -1
  76. package/lib/util/builders/url/base.director.d.ts +0 -4
  77. package/lib/util/builders/url/base.director.js +0 -25
  78. package/lib/util/builders/url/base.director.js.map +1 -1
  79. package/lib/util/builders/url/basic.director.d.ts +1 -1
  80. package/lib/util/builders/url/basic.director.js +1 -1
  81. package/lib/util/builders/url/web.director.d.ts +1 -1
  82. package/lib/util/builders/url/web.director.js +2 -2
  83. package/lib/util/builders/url/web.director.js.map +1 -1
  84. package/lib/util/categories.d.ts +5 -5
  85. package/lib/util/categories.js +175 -174
  86. package/lib/util/categories.js.map +1 -1
  87. package/lib/util/const.d.ts +4 -8
  88. package/lib/util/const.js +9 -17
  89. package/lib/util/const.js.map +1 -1
  90. package/lib/util/customCssJs.d.ts +6 -0
  91. package/lib/util/customCssJs.js +70 -0
  92. package/lib/util/customCssJs.js.map +1 -0
  93. package/lib/util/dump.d.ts +15 -4
  94. package/lib/util/dump.js +282 -86
  95. package/lib/util/dump.js.map +1 -1
  96. package/lib/util/index.d.ts +1 -1
  97. package/lib/util/index.js +1 -1
  98. package/lib/util/index.js.map +1 -1
  99. package/lib/util/metaData.js.map +1 -1
  100. package/lib/util/misc.d.ts +23 -13
  101. package/lib/util/misc.js +90 -59
  102. package/lib/util/misc.js.map +1 -1
  103. package/lib/util/mw-api.d.ts +7 -5
  104. package/lib/util/mw-api.js +167 -221
  105. package/lib/util/mw-api.js.map +1 -1
  106. package/lib/util/pageListMainPage.d.ts +3 -0
  107. package/lib/util/pageListMainPage.js +10 -0
  108. package/lib/util/pageListMainPage.js.map +1 -0
  109. package/lib/util/pages.d.ts +30 -0
  110. package/lib/util/pages.js +146 -0
  111. package/lib/util/pages.js.map +1 -0
  112. package/lib/util/rewriteUrls.d.ts +2 -2
  113. package/lib/util/rewriteUrls.js +33 -31
  114. package/lib/util/rewriteUrls.js.map +1 -1
  115. package/lib/util/savePages.d.ts +8 -0
  116. package/lib/util/savePages.js +182 -0
  117. package/lib/util/savePages.js.map +1 -0
  118. package/lib/version.d.ts +1 -1
  119. package/lib/version.js +1 -1
  120. package/lib/version.js.map +1 -1
  121. package/offliner-definition.json +178 -90
  122. package/package.json +50 -49
  123. package/res/page_list_home.js +20 -0
  124. package/res/{article_not_found.svg → page_not_found.svg} +1 -1
  125. package/res/script.js +30 -45
  126. package/res/style.css +52 -68
  127. package/res/templates/download_error_placeholder.html +7 -8
  128. package/res/templates/javaScript.html +19 -0
  129. package/res/templates/pageFallback.html +9 -7
  130. package/res/templates/pageVector2022.html +9 -7
  131. package/res/templates/pageVectorLegacy.html +9 -7
  132. package/res/templates/{article_list_home.html → page_list_home.html} +1 -2
  133. package/translation/ar.json +21 -21
  134. package/translation/bn.json +1 -1
  135. package/translation/de.json +46 -19
  136. package/translation/en.json +54 -21
  137. package/translation/es.json +17 -19
  138. package/translation/fi.json +23 -3
  139. package/translation/fr.json +57 -21
  140. package/translation/he.json +8 -6
  141. package/translation/ia.json +19 -19
  142. package/translation/id.json +13 -13
  143. package/translation/it.json +13 -8
  144. package/translation/ko.json +55 -3
  145. package/translation/lb.json +5 -2
  146. package/translation/mk.json +5 -5
  147. package/translation/nl.json +57 -22
  148. package/translation/or.json +2 -2
  149. package/translation/pt.json +2 -2
  150. package/translation/qqq.json +55 -22
  151. package/translation/ru.json +2 -2
  152. package/translation/sc.json +2 -2
  153. package/translation/sk.json +62 -0
  154. package/translation/sl.json +54 -21
  155. package/translation/sv.json +20 -19
  156. package/translation/sw.json +45 -2
  157. package/translation/zh-hans.json +56 -22
  158. package/translation/zh-hant.json +26 -19
  159. package/lib/renderers/abstractDesktop.render.d.ts +0 -12
  160. package/lib/renderers/abstractDesktop.render.js +0 -67
  161. package/lib/renderers/abstractDesktop.render.js.map +0 -1
  162. package/lib/renderers/abstractMobile.render.d.ts +0 -12
  163. package/lib/renderers/abstractMobile.render.js +0 -54
  164. package/lib/renderers/abstractMobile.render.js.map +0 -1
  165. package/lib/renderers/rest-api.renderer.d.ts +0 -7
  166. package/lib/renderers/rest-api.renderer.js +0 -68
  167. package/lib/renderers/rest-api.renderer.js.map +0 -1
  168. package/lib/renderers/visual-editor.renderer.d.ts +0 -7
  169. package/lib/renderers/visual-editor.renderer.js +0 -76
  170. package/lib/renderers/visual-editor.renderer.js.map +0 -1
  171. package/lib/renderers/wikimedia-desktop.renderer.d.ts +0 -4
  172. package/lib/renderers/wikimedia-desktop.renderer.js +0 -7
  173. package/lib/renderers/wikimedia-desktop.renderer.js.map +0 -1
  174. package/lib/renderers/wikimedia-mobile.renderer.d.ts +0 -19
  175. package/lib/renderers/wikimedia-mobile.renderer.js +0 -243
  176. package/lib/renderers/wikimedia-mobile.renderer.js.map +0 -1
  177. package/lib/util/articleListMainPage.d.ts +0 -3
  178. package/lib/util/articleListMainPage.js +0 -10
  179. package/lib/util/articleListMainPage.js.map +0 -1
  180. package/lib/util/articles.d.ts +0 -26
  181. package/lib/util/articles.js +0 -102
  182. package/lib/util/articles.js.map +0 -1
  183. package/lib/util/builders/url/desktop.director.d.ts +0 -8
  184. package/lib/util/builders/url/desktop.director.js +0 -15
  185. package/lib/util/builders/url/desktop.director.js.map +0 -1
  186. package/lib/util/builders/url/mobile.director.d.ts +0 -8
  187. package/lib/util/builders/url/mobile.director.js +0 -15
  188. package/lib/util/builders/url/mobile.director.js.map +0 -1
  189. package/lib/util/builders/url/rest-api.director.d.ts +0 -8
  190. package/lib/util/builders/url/rest-api.director.js +0 -18
  191. package/lib/util/builders/url/rest-api.director.js.map +0 -1
  192. package/lib/util/builders/url/visual-editor.director.d.ts +0 -9
  193. package/lib/util/builders/url/visual-editor.director.js +0 -18
  194. package/lib/util/builders/url/visual-editor.director.js.map +0 -1
  195. package/lib/util/saveArticles.d.ts +0 -8
  196. package/lib/util/saveArticles.js +0 -411
  197. package/lib/util/saveArticles.js.map +0 -1
  198. package/res/article_list_home.js +0 -24
  199. package/res/content.parsoid.css +0 -196
  200. package/res/inserted_style.css +0 -45
  201. package/res/templates/categories.html +0 -11
  202. package/res/templates/lead_section_wrapper.html +0 -6
  203. package/res/templates/pageWikimediaDesktop.html +0 -24
  204. package/res/templates/pageWikimediaMobile.html +0 -24
  205. package/res/templates/section_wrapper.html +0 -5
  206. package/res/templates/subcategories.html +0 -23
  207. package/res/templates/subpages.html +0 -17
  208. package/res/templates/subsection_wrapper.html +0 -5
  209. package/res/webpHandler.js +0 -165
  210. package/res/wm_mobile_override_script.js +0 -15
  211. package/res/wm_mobile_override_style.css +0 -20
  212. package/translation/br.json +0 -8
  213. package/translation/dag.json +0 -9
  214. package/translation/es-formal.json +0 -29
  215. package/translation/ha.json +0 -10
  216. package/translation/hi.json +0 -9
  217. package/translation/ig.json +0 -10
  218. package/translation/kaa.json +0 -9
  219. package/translation/nb.json +0 -9
  220. package/translation/nqo.json +0 -8
  221. package/translation/pt-br.json +0 -10
  222. package/translation/ro.json +0 -9
  223. package/translation/scn.json +0 -8
  224. package/translation/sq.json +0 -9
  225. package/translation/te.json +0 -9
  226. package/translation/tn.json +0 -9
  227. package/translation/tr.json +0 -10
@@ -1,21 +1,10 @@
1
1
  import MediaWiki from './../MediaWiki.js';
2
- import { VisualEditorRenderer } from './visual-editor.renderer.js';
3
- import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js';
4
- import { WikimediaMobileRenderer } from './wikimedia-mobile.renderer.js';
5
- import { RestApiRenderer } from './rest-api.renderer.js';
6
2
  import * as logger from './../Logger.js';
7
3
  import { ActionParseRenderer } from './action-parse.renderer.js';
8
4
  export class RendererBuilder {
9
5
  async createRenderer(options) {
10
6
  const { renderType, renderName } = options;
11
- const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi, hasActionParseApi, hasModuleApi] = await Promise.all([
12
- MediaWiki.hasVisualEditorApi(),
13
- MediaWiki.hasWikimediaDesktopApi(),
14
- MediaWiki.hasWikimediaMobileApi(),
15
- MediaWiki.hasRestApi(),
16
- MediaWiki.hasActionParseApi(),
17
- MediaWiki.hasModuleApi(),
18
- ]);
7
+ const [hasActionParseApi, hasModuleApi] = await Promise.all([MediaWiki.hasActionParseApi(), MediaWiki.hasModuleApi()]);
19
8
  if (!hasModuleApi) {
20
9
  logger.error('Module API not available while mandatory');
21
10
  process.exit(1);
@@ -25,16 +14,6 @@ export class RendererBuilder {
25
14
  if (hasActionParseApi) {
26
15
  return new ActionParseRenderer();
27
16
  }
28
- else if (hasWikimediaDesktopApi) {
29
- // Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
30
- return new WikimediaDesktopRenderer();
31
- }
32
- else if (hasVisualEditorApi) {
33
- return new VisualEditorRenderer();
34
- }
35
- else if (hasRestApi) {
36
- return new RestApiRenderer();
37
- }
38
17
  else {
39
18
  logger.error('No available desktop renderer.');
40
19
  process.exit(1);
@@ -44,29 +23,15 @@ export class RendererBuilder {
44
23
  if (hasActionParseApi) {
45
24
  return new ActionParseRenderer();
46
25
  }
47
- else if (hasWikimediaMobileApi) {
48
- return new WikimediaMobileRenderer();
26
+ else {
27
+ logger.error('No available mobile renderer.');
28
+ process.exit(1);
49
29
  }
50
- logger.error('No available mobile renderer.');
51
- process.exit(1);
52
30
  break; // prettier-ignore
53
31
  case 'auto':
54
32
  if (hasActionParseApi) {
55
33
  return new ActionParseRenderer();
56
34
  }
57
- else if (hasWikimediaDesktopApi) {
58
- // Choose WikimediaDesktopRenderer if it's present, regardless of hasVisualEditorApi value
59
- return new WikimediaDesktopRenderer();
60
- }
61
- else if (hasVisualEditorApi) {
62
- return new VisualEditorRenderer();
63
- }
64
- else if (hasRestApi) {
65
- return new RestApiRenderer();
66
- }
67
- else if (hasWikimediaMobileApi) {
68
- return new WikimediaMobileRenderer();
69
- }
70
35
  else {
71
36
  logger.error('No render available at all.');
72
37
  process.exit(1);
@@ -74,12 +39,12 @@ export class RendererBuilder {
74
39
  break; // prettier-ignore
75
40
  case 'specific':
76
41
  // renderName argument is required for 'specific' mode
77
- return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi, hasActionParseApi);
42
+ return this.handleSpecificRender(renderName, hasActionParseApi);
78
43
  default:
79
44
  throw new Error(`Unknown render: ${renderType}`);
80
45
  }
81
46
  }
82
- handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi, hasActionParseApi) {
47
+ handleSpecificRender(renderName, hasActionParseApi) {
83
48
  // renderName argument is required for 'specific' mode
84
49
  switch (renderName) {
85
50
  case 'ActionParse':
@@ -89,34 +54,6 @@ export class RendererBuilder {
89
54
  logger.error('Cannot create an instance of ActionParse renderer.');
90
55
  process.exit(1);
91
56
  break; // prettier-ignore
92
- case 'WikimediaDesktop':
93
- if (hasWikimediaDesktopApi) {
94
- return new WikimediaDesktopRenderer();
95
- }
96
- logger.error('Cannot create an instance of WikimediaDesktop renderer.');
97
- process.exit(1);
98
- break; // prettier-ignore
99
- case 'VisualEditor':
100
- if (hasVisualEditorApi) {
101
- return new VisualEditorRenderer();
102
- }
103
- logger.error('Cannot create an instance of VisualEditor renderer.');
104
- process.exit(1);
105
- break; // prettier-ignore
106
- case 'WikimediaMobile':
107
- if (hasWikimediaMobileApi) {
108
- return new WikimediaMobileRenderer();
109
- }
110
- logger.error('No available mobile renderer.');
111
- process.exit(1);
112
- break; // prettier-ignore
113
- case 'RestApi':
114
- if (hasRestApi) {
115
- return new RestApiRenderer();
116
- }
117
- logger.error('Cannot create an instance of RestApi renderer.');
118
- process.exit(1);
119
- break; // prettier-ignore
120
57
  default:
121
58
  throw new Error(`Unknown renderName for specific mode: ${renderName}`);
122
59
  }
@@ -1 +1 @@
1
- {"version":3,"file":"renderer.builder.js","sourceRoot":"","sources":["../../src/renderers/renderer.builder.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAA;AAEzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAA;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAA;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,MAAM,OAAO,eAAe;IACnB,KAAK,CAAC,cAAc,CAAC,OAA+B;QACzD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAA;QAE1C,MAAM,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,UAAU,EAAE,iBAAiB,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzI,SAAS,CAAC,kBAAkB,EAAE;YAC9B,SAAS,CAAC,sBAAsB,EAAE;YAClC,SAAS,CAAC,qBAAqB,EAAE;YACjC,SAAS,CAAC,UAAU,EAAE;YACtB,SAAS,CAAC,iBAAiB,EAAE;YAC7B,SAAS,CAAC,YAAY,EAAE;SACzB,CAAC,CAAA;QAEF,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAA;YACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,SAAS;gBACZ,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,IAAI,sBAAsB,EAAE,CAAC;oBAClC,0FAA0F;oBAC1F,OAAO,IAAI,wBAAwB,EAAE,CAAA;gBACvC,CAAC;qBAAM,IAAI,kBAAkB,EAAE,CAAC;oBAC9B,OAAO,IAAI,oBAAoB,EAAE,CAAA;gBACnC,CAAC;qBAAM,IAAI,UAAU,EAAE,CAAC;oBACtB,OAAO,IAAI,eAAe,EAAE,CAAA;gBAC9B,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;oBAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,QAAQ;gBACX,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,IAAI,qBAAqB,EAAE,CAAC;oBACjC,OAAO,IAAI,uBAAuB,EAAE,CAAA;gBACtC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;gBAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B,KAAK,MAAM;gBACT,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,IAAI,sBAAsB,EAAE,CAAC;oBAClC,0FAA0F;oBAC1F,OAAO,IAAI,wBAAwB,EAAE,CAAA;gBACvC,CAAC;qBAAM,IAAI,kBAAkB,EAAE,CAAC;oBAC9B,OAAO,IAAI,oBAAoB,EAAE,CAAA;gBACnC,CAAC;qBAAM,IAAI,UAAU,EAAE,CAAC;oBACtB,OAAO,IAAI,eAAe,EAAE,CAAA;gBAC9B,CAAC;qBAAM,IAAI,qBAAqB,EAAE,CAAC;oBACjC,OAAO,IAAI,uBAAuB,EAAE,CAAA;gBACtC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;oBAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,UAAU;gBACb,sDAAsD;gBACtD,OAAO,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,UAAU,EAAE,iBAAiB,CAAC,CAAA;YAChJ;gBACE,MAAM,IAAI,KAAK,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,UAAkB,EAClB,kBAA2B,EAC3B,sBAA+B,EAC/B,qBAA8B,EAC9B,UAAmB,EACnB,iBAA0B;QAE1B,sDAAsD;QACtD,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,aAAa;gBAChB,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B,KAAK,kBAAkB;gBACrB,IAAI,sBAAsB,EAAE,CAAC;oBAC3B,OAAO,IAAI,wBAAwB,EAAE,CAAA;gBACvC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAA;gBACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B,KAAK,cAAc;gBACjB,IAAI,kBAAkB,EAAE,CAAC;oBACvB,OAAO,IAAI,oBAAoB,EAAE,CAAA;gBACnC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAA;gBACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B,KAAK,iBAAiB;gBACpB,IAAI,qBAAqB,EAAE,CAAC;oBAC1B,OAAO,IAAI,uBAAuB,EAAE,CAAA;gBACtC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;gBAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B,KAAK,SAAS;gBACZ,IAAI,UAAU,EAAE,CAAC;oBACf,OAAO,IAAI,eAAe,EAAE,CAAA;gBAC9B,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;gBAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B;gBACE,MAAM,IAAI,KAAK,CAAC,yCAAyC,UAAU,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;CACF"}
1
+ {"version":3,"file":"renderer.builder.js","sourceRoot":"","sources":["../../src/renderers/renderer.builder.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAA;AAEzC,OAAO,KAAK,MAAM,MAAM,gBAAgB,CAAA;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAEhE,MAAM,OAAO,eAAe;IACnB,KAAK,CAAC,cAAc,CAAC,OAA+B;QACzD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,OAAO,CAAA;QAE1C,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,iBAAiB,EAAE,EAAE,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;QAEtH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAA;YACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,SAAS;gBACZ,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;oBAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,QAAQ;gBACX,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;oBAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,MAAM;gBACT,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;oBAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,MAAK,CAAC,kBAAkB;YAC1B,KAAK,UAAU;gBACb,sDAAsD;gBACtD,OAAO,IAAI,CAAC,oBAAoB,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;YACjE;gBACE,MAAM,IAAI,KAAK,CAAC,mBAAmB,UAAU,EAAE,CAAC,CAAA;QACpD,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,UAAkB,EAAE,iBAA0B;QACzE,sDAAsD;QACtD,QAAQ,UAAU,EAAE,CAAC;YACnB,KAAK,aAAa;gBAChB,IAAI,iBAAiB,EAAE,CAAC;oBACtB,OAAO,IAAI,mBAAmB,EAAE,CAAA;gBAClC,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACf,MAAK,CAAC,kBAAkB;YAC1B;gBACE,MAAM,IAAI,KAAK,CAAC,yCAAyC,UAAU,EAAE,CAAC,CAAA;QAC1E,CAAC;IACH,CAAC;CACF"}
@@ -1,10 +1,9 @@
1
1
  import { Renderer, renderName } from './abstract.renderer.js';
2
2
  declare class RenderingContext {
3
3
  private static instance;
4
- mainPageRenderer: Renderer;
5
- articlesRenderer: Renderer;
4
+ pagesRenderer: Renderer;
6
5
  static getInstance(): RenderingContext;
7
- createRenderers(forceRender: renderName | null, hasWikimediaMobileApi: boolean): Promise<void>;
6
+ createRenderers(forceRender: renderName | null): Promise<void>;
8
7
  }
9
8
  declare const _default: RenderingContext;
10
9
  export default _default;
@@ -3,34 +3,26 @@ import { RendererBuilder } from './renderer.builder.js';
3
3
  import * as logger from '../Logger.js';
4
4
  class RenderingContext {
5
5
  static instance;
6
- mainPageRenderer;
7
- articlesRenderer;
6
+ pagesRenderer;
8
7
  static getInstance() {
9
8
  if (!RenderingContext.instance) {
10
9
  RenderingContext.instance = new RenderingContext();
11
10
  }
12
11
  return RenderingContext.instance;
13
12
  }
14
- async createRenderers(forceRender, hasWikimediaMobileApi) {
13
+ async createRenderers(forceRender) {
15
14
  const rendererBuilder = new RendererBuilder();
16
15
  if (forceRender) {
17
- // All articles and main page will use the same renderer if 'forceRender' is specified
18
- const renderer = await rendererBuilder.createRenderer({
16
+ this.pagesRenderer = await rendererBuilder.createRenderer({
19
17
  renderType: 'specific',
20
18
  renderName: forceRender,
21
19
  });
22
- this.mainPageRenderer = renderer;
23
- this.articlesRenderer = renderer;
24
20
  }
25
21
  else {
26
- this.mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' });
27
- this.articlesRenderer = await rendererBuilder.createRenderer({
28
- renderType: hasWikimediaMobileApi ? 'mobile' : 'auto',
29
- });
22
+ this.pagesRenderer = await rendererBuilder.createRenderer({ renderType: 'auto' });
30
23
  }
31
- logger.log(`Using ${this.mainPageRenderer.constructor.name} for main page renderer`);
32
- logger.log(`Using ${this.articlesRenderer.constructor.name} for articles renderer`);
33
- Downloader.setUrlsDirectors(this.mainPageRenderer, this.articlesRenderer);
24
+ logger.info(`Using ${this.pagesRenderer.constructor.name} for pages renderer`);
25
+ Downloader.setUrlsDirectors(this.pagesRenderer);
34
26
  }
35
27
  }
36
28
  const instance = RenderingContext.getInstance();
@@ -1 +1 @@
1
- {"version":3,"file":"rendering.context.js","sourceRoot":"","sources":["../../src/renderers/rendering.context.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,kBAAkB,CAAA;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AAEtC,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAElC,gBAAgB,CAAU;IAC1B,gBAAgB,CAAU;IAE1B,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YAC/B,gBAAgB,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;QACpD,CAAC;QACD,OAAO,gBAAgB,CAAC,QAAQ,CAAA;IAClC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,WAA8B,EAAE,qBAA8B;QACzF,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAE7C,IAAI,WAAW,EAAE,CAAC;YAChB,sFAAsF;YACtF,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBACpD,UAAU,EAAE,UAAU;gBACtB,UAAU,EAAE,WAAW;aACxB,CAAC,CAAA;YACF,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAA;YAChC,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAA;QAClC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAA;YACvF,IAAI,CAAC,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC3D,UAAU,EAAE,qBAAqB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM;aACtD,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,yBAAyB,CAAC,CAAA;QACpF,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,wBAAwB,CAAC,CAAA;QACnF,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC3E,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAA;AAC/C,eAAe,QAA4B,CAAA"}
1
+ {"version":3,"file":"rendering.context.js","sourceRoot":"","sources":["../../src/renderers/rendering.context.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,kBAAkB,CAAA;AAEzC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,KAAK,MAAM,MAAM,cAAc,CAAA;AAEtC,MAAM,gBAAgB;IACZ,MAAM,CAAC,QAAQ,CAAkB;IAElC,aAAa,CAAU;IAEvB,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YAC/B,gBAAgB,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAA;QACpD,CAAC;QACD,OAAO,gBAAgB,CAAC,QAAQ,CAAA;IAClC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,WAA8B;QACzD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;QAE7C,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC,aAAa,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBACxD,UAAU,EAAE,UAAU;gBACtB,UAAU,EAAE,WAAW;aACxB,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,aAAa,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAA;QACnF,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,qBAAqB,CAAC,CAAA;QAC9E,UAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACjD,CAAC;CACF;AAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,WAAW,EAAE,CAAA;AAC/C,eAAe,QAA4B,CAAA"}
@@ -1,14 +1,16 @@
1
- import * as logger from './Logger.js';
2
1
  export declare function sanitize_all(argv: any): Promise<void>;
3
2
  export declare function check_all(argv: any): Promise<void>;
4
- export declare function sanitizeWikiPath(mwWikiPath?: string): string;
5
3
  export declare function sanitizeApiPathParam(apiPathParam: string): string;
6
4
  export declare function sanitizeStringMaxLength(text: string, key: string, length: number): void;
7
- export declare function sanitize_verbose(verbose: logger.LogLevel | true): void;
8
- export declare function sanitize_articlesList_addNamespaces(articlesList: string, addNamespaces: string): void;
5
+ export declare function sanitize_logLevel(logLevel: string | undefined): void;
6
+ export declare function sanitize_customName_langVariant(customZimName: string, customZimFilename: string, langVariant: string | string[]): void;
7
+ export declare function sanitizePageSelection(pageList: string, addNamespaces: string, onlyNamespaces: string): void;
9
8
  export declare function sanitizeDoubleUsedParameters(options: object): void;
10
9
  export declare function sanitize_speed(_speed: any): void;
10
+ export declare function sanitizeCategoriesPageSize(categoriesPageSize: any): void;
11
+ export declare function sanitize_javaScript_addModules(javaScript: any, addModules: any): void;
11
12
  export declare function check_mwApiReachability(mwUrl: string, mwActionApiPath: string): Promise<void>;
13
+ export declare function sanitize_mwUrl(mwUrl: string): void;
12
14
  export declare function sanitize_adminEmail(adminEmail: any): void;
13
15
  export declare function check_redis(): Promise<void>;
14
16
  export declare function check_customZimFavicon(customZimFavicon: any): Promise<void>;
@@ -25,4 +27,5 @@ export declare function check_s3(optimisationCacheUrl: string, insecure: boolean
25
27
  * path resolution.
26
28
  */
27
29
  export declare function sanitize_customFlavour(customFlavour: string): string;
30
+ export declare function sanitize_mathJax(mathJaxSource: any, mathJaxConfig: any, mathJaxEntryPoint: any, mathJaxAllPages: any): void;
28
31
  export declare function sanitize_forceRender(renderName: string): string;
@@ -13,18 +13,20 @@ import MediaWiki from './MediaWiki.js';
13
13
  import { config } from './config.js';
14
14
  const __filename = fileURLToPath(import.meta.url);
15
15
  const __dirname = path.dirname(__filename);
16
- const parametersWithArrayType = ['format'];
16
+ const parametersWithArrayType = ['format', 'langVariant'];
17
17
  // Perform "static check" of arguments not depending on external connectivity
18
18
  // Stack is not yet sufficiently ready to perform live external connectivity checks
19
19
  export async function sanitize_all(argv) {
20
20
  // extracting all arguments
21
- const { articleList, addNamespaces, speed, adminEmail, verbose, customZimLongDescription, customZimDescription, forceRender, mwWikiPath, mwIndexPhpPath, mwActionApiPath, mwRestApiPath, mwModulePath, } = argv;
21
+ const { pageList, addNamespaces, onlyNamespaces, speed, adminEmail, mwUrl, logLevel, customZimLongDescription, customZimDescription, customZimName, customZimFilename, forceRender, javaScript, addModules, langVariant, mwActionApiPath, mwModulePath, categoriesPageSize, mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages, } = argv;
22
22
  sanitizeDoubleUsedParameters(argv);
23
- sanitize_articlesList_addNamespaces(articleList, addNamespaces);
24
- // sanitizing verbose
25
- sanitize_verbose(verbose);
23
+ sanitize_customName_langVariant(customZimName, customZimFilename, langVariant);
24
+ sanitizePageSelection(pageList, addNamespaces, onlyNamespaces);
25
+ // sanitizing logLevel
26
+ sanitize_logLevel(logLevel);
26
27
  // sanitizing speed
27
28
  sanitize_speed(speed);
29
+ sanitizeCategoriesPageSize(categoriesPageSize);
28
30
  // sanitizing longDescription
29
31
  sanitizeStringMaxLength(customZimLongDescription, 'customZimLongDescription', 4000);
30
32
  // sanitizing description
@@ -36,18 +38,8 @@ export async function sanitize_all(argv) {
36
38
  throw new Error('Custom Flavour not found');
37
39
  }
38
40
  }
39
- // sanitizing mwWikiPath
40
- if (mwWikiPath) {
41
- argv.mwWikiPath = sanitizeWikiPath(mwWikiPath);
42
- }
43
- // sanitizing mwIndexPhpPath
44
- if (mwIndexPhpPath) {
45
- argv.mwIndexPhpPath = sanitizeApiPathParam(mwIndexPhpPath);
46
- }
47
- // sanitizing mwRestApiPath
48
- if (mwRestApiPath) {
49
- argv.mwRestApiPath = sanitizeApiPathParam(mwRestApiPath);
50
- }
41
+ // sanitizing javaScript
42
+ sanitize_javaScript_addModules(javaScript, addModules);
51
43
  // sanitizing mwActionApiPath
52
44
  if (mwActionApiPath) {
53
45
  argv.mwActionApiPath = sanitizeApiPathParam(mwActionApiPath);
@@ -58,14 +50,24 @@ export async function sanitize_all(argv) {
58
50
  }
59
51
  // sanitize Custom Main Page
60
52
  if (argv.customMainPage) {
61
- argv.customMainPage = argv.customMainPage.replace(/ /g, '_');
53
+ argv.customMainPage = argv.customMainPage.replace(/_/g, ' ');
54
+ }
55
+ if (!mwUrl || mwUrl.trim() === '') {
56
+ throw new Error('Option --mwUrl is required and cannot be empty');
62
57
  }
58
+ if (!adminEmail || adminEmail.trim() === '') {
59
+ throw new Error('Option --adminEmail is required and cannot be empty');
60
+ }
61
+ //sanitizing mwUrl
62
+ sanitize_mwUrl(mwUrl);
63
63
  // sanitizing adminEmail
64
64
  sanitize_adminEmail(adminEmail);
65
65
  // sanitizing renderer
66
66
  if (forceRender) {
67
67
  sanitize_forceRender(forceRender);
68
68
  }
69
+ // sanitizing MathJax options
70
+ sanitize_mathJax(mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages);
69
71
  }
70
72
  // perform live checks of arguments needing a connection "somewhere"
71
73
  export async function check_all(argv) {
@@ -83,18 +85,6 @@ export async function check_all(argv) {
83
85
  await check_customZimFavicon(customZimFavicon);
84
86
  }
85
87
  }
86
- export function sanitizeWikiPath(mwWikiPath = '') {
87
- mwWikiPath = sanitizeApiPathParam(mwWikiPath);
88
- // Remove trailing $1 since we don't need it
89
- if (mwWikiPath?.endsWith('$1')) {
90
- mwWikiPath = mwWikiPath.substring(0, mwWikiPath.length - 3);
91
- }
92
- // Make sure wikiPath always has forward slash at the end for the correct URL building
93
- if (!mwWikiPath?.endsWith('/')) {
94
- mwWikiPath += '/';
95
- }
96
- return mwWikiPath;
97
- }
98
88
  export function sanitizeApiPathParam(apiPathParam) {
99
89
  if (!apiPathParam) {
100
90
  return;
@@ -110,14 +100,28 @@ export function sanitizeStringMaxLength(text, key, length) {
110
100
  throw new Error(`${key} should be less than ${length} characters.`);
111
101
  }
112
102
  }
113
- export function sanitize_verbose(verbose) {
114
- if (verbose && verbose !== true && !logger.logLevels.includes(verbose)) {
115
- throw new Error(`"${verbose}" is not a valid value for option verbose. It should be empty or one of [info, log, warn, error, quiet].`);
103
+ export function sanitize_logLevel(logLevel) {
104
+ if (logLevel !== undefined && !logger.logLevels.includes(logLevel)) {
105
+ throw new Error(`"${logLevel}" is not a valid value for option --log-level. It should be one of [${logger.logLevels.join(', ')}].`);
116
106
  }
117
107
  }
118
- export function sanitize_articlesList_addNamespaces(articlesList, addNamespaces) {
119
- if (articlesList && addNamespaces) {
120
- throw new Error('options --articlesList and --addNamespaces cannot be used together');
108
+ export function sanitize_customName_langVariant(customZimName, customZimFilename, langVariant) {
109
+ if (customZimName && Array.isArray(langVariant) && !customZimName.includes('{lang_or_variant}')) {
110
+ throw new Error('option --customZimName needs to include the {lang_or_variant} placeholder when using multiple --langVariant');
111
+ }
112
+ if (customZimFilename && Array.isArray(langVariant) && !customZimFilename.includes('{lang_or_variant}')) {
113
+ throw new Error('option --customZimFilename needs to include the {lang_or_variant} placeholder when using multiple --langVariant');
114
+ }
115
+ }
116
+ export function sanitizePageSelection(pageList, addNamespaces, onlyNamespaces) {
117
+ if (pageList && addNamespaces) {
118
+ throw new Error('options --pageList and --addNamespaces cannot be used together');
119
+ }
120
+ if (pageList && onlyNamespaces) {
121
+ throw new Error('options --pageList and --onlyNamespaces cannot be used together');
122
+ }
123
+ if (addNamespaces && onlyNamespaces) {
124
+ throw new Error('options --addNamespaces and --onlyNamespaces cannot be used together');
121
125
  }
122
126
  }
123
127
  export function sanitizeDoubleUsedParameters(options) {
@@ -129,21 +133,64 @@ export function sanitizeDoubleUsedParameters(options) {
129
133
  }
130
134
  }
131
135
  export function sanitize_speed(_speed) {
132
- if (_speed && isNaN(_speed)) {
133
- throw new Error('speed is not a number, please give a number value to --speed.');
136
+ if (_speed !== undefined && _speed !== null) {
137
+ const numSpeed = Number(_speed);
138
+ if (isNaN(numSpeed) || numSpeed <= 0) {
139
+ throw new Error('speed must be a positive number. Values >= 1 set the number of parallel workers (integer). Values < 1 slow down the scraper.');
140
+ }
141
+ if (numSpeed >= 1 && !Number.isInteger(numSpeed)) {
142
+ throw new Error('speed values >= 1 must be integers (number of parallel workers).');
143
+ }
144
+ }
145
+ }
146
+ export function sanitizeCategoriesPageSize(categoriesPageSize) {
147
+ if (categoriesPageSize !== undefined && categoriesPageSize !== null) {
148
+ const numCategoriesPageSize = Number(categoriesPageSize);
149
+ if (isNaN(numCategoriesPageSize)) {
150
+ throw new Error('categoriesPageSize must be a number.');
151
+ }
152
+ if (numCategoriesPageSize <= 0) {
153
+ throw new Error('categoriesPageSize must be a positive.');
154
+ }
155
+ if (!Number.isInteger(numCategoriesPageSize)) {
156
+ throw new Error('categoriesPageSize must be integer.');
157
+ }
158
+ }
159
+ }
160
+ export function sanitize_javaScript_addModules(javaScript, addModules) {
161
+ if (javaScript && !['none', 'trusted', 'all'].includes(javaScript)) {
162
+ throw new Error(`"${javaScript}" is not a valid value for option javaScript. It should be empty or one of [none, trusted, all].`);
163
+ }
164
+ if (javaScript === 'none' && addModules) {
165
+ throw new Error('options --javaScript=none and --addModules cannot be used together');
134
166
  }
135
167
  }
136
168
  export async function check_mwApiReachability(mwUrl, mwActionApiPath) {
137
169
  // We are building an API query "by-hand" because we've not yet initialized the whole URL director / URL builder stack
138
170
  const apiQueryUrl = `${new URL(mwActionApiPath || MediaWiki.actionApiPath, mwUrl).toString()}?action=query&format=json&formatversion=2&maxlag=${config.defaults.maxlag}`;
139
171
  const value = await Downloader.get(apiQueryUrl).catch((err) => {
140
- throw new Error(`Mediawiki API is not reachable with ${apiQueryUrl}\n${err}`);
172
+ throw new Error(`Mediawiki API is not reachable with ${apiQueryUrl}\n${err}\nYou may need to pass a (different) value for --mwActionApiPath, see the FAQ.`);
141
173
  });
142
174
  const contentType = value.headers['content-type'];
143
175
  if (!contentType.includes('application/json')) {
144
176
  throw new Error(`Mediawiki API is returning '${contentType}' Content-Type instead of 'application/json' with ${apiQueryUrl}`);
145
177
  }
146
178
  }
179
+ export function sanitize_mwUrl(mwUrl) {
180
+ let url;
181
+ try {
182
+ url = new URL(mwUrl);
183
+ }
184
+ catch {
185
+ throw new Error(`mwUrl is not a valid URL.`);
186
+ }
187
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
188
+ throw new Error(`mwUrl must use http or https protocol, got '${url.protocol.replace(':', '')}'.`);
189
+ }
190
+ if (url.pathname !== '/') {
191
+ throw new Error(`mwUrl must not contain a path (only the hostname is expected), got '${url.pathname}'. See the FAQ.`);
192
+ }
193
+ }
147
194
  export function sanitize_adminEmail(adminEmail) {
148
195
  if (!isValidEmail(adminEmail)) {
149
196
  throw new Error(`Admin email [${adminEmail}] is not valid.`);
@@ -151,7 +198,7 @@ export function sanitize_adminEmail(adminEmail) {
151
198
  }
152
199
  export async function check_redis() {
153
200
  await RedisStore.connect(false);
154
- logger.log('closing sanitize redis DB');
201
+ logger.info('closing sanitize redis DB');
155
202
  await RedisStore.close();
156
203
  }
157
204
  export async function check_customZimFavicon(customZimFavicon) {
@@ -174,7 +221,7 @@ export async function check_s3(optimisationCacheUrl, insecure) {
174
221
  const s3Url = (s3UrlObj.protocol || 'https:') + '//' + (s3UrlObj.host || '') + (s3UrlObj.pathname || '');
175
222
  const s3Obj = new S3(s3Url, s3UrlObj.searchParams, 1000 * 60, insecure);
176
223
  await s3Obj.initialise().then(() => {
177
- logger.log('Successfully logged in S3');
224
+ logger.info('Successfully logged in S3');
178
225
  });
179
226
  }
180
227
  }
@@ -195,6 +242,17 @@ export function sanitize_customFlavour(customFlavour) {
195
242
  return fs.existsSync(possiblePath);
196
243
  }) || null);
197
244
  }
245
+ export function sanitize_mathJax(mathJaxSource, mathJaxConfig, mathJaxEntryPoint, mathJaxAllPages) {
246
+ if (mathJaxConfig && !mathJaxSource) {
247
+ throw new Error('Option --mathJaxConfig requires --mathJaxSource to also be set');
248
+ }
249
+ if (mathJaxEntryPoint && !mathJaxSource) {
250
+ throw new Error('Option --mathJaxEntryPoint requires --mathJaxSource to also be set');
251
+ }
252
+ if (mathJaxAllPages && !mathJaxSource) {
253
+ throw new Error('Option --mathJaxAllPages requires --mathJaxSource to also be set');
254
+ }
255
+ }
198
256
  export function sanitize_forceRender(renderName) {
199
257
  const checkRenderName = (arr, val) => {
200
258
  return arr.some((arrVal) => val === arrVal);
@@ -1 +1 @@
1
- {"version":3,"file":"sanitize-argument.js","sourceRoot":"","sources":["../src/sanitize-argument.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,KAAK,CAAA;AACxC,OAAO,UAAU,MAAM,MAAM,CAAA;AAC7B,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEpC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;AAC1C,MAAM,uBAAuB,GAAG,CAAC,QAAQ,CAAC,CAAA;AAE1C,6EAA6E;AAC7E,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAS;IAC1C,2BAA2B;IAC3B,MAAM,EACJ,WAAW,EACX,aAAa,EACb,KAAK,EACL,UAAU,EACV,OAAO,EACP,wBAAwB,EACxB,oBAAoB,EACpB,WAAW,EACX,UAAU,EACV,cAAc,EACd,eAAe,EACf,aAAa,EACb,YAAY,GACb,GAAG,IAAI,CAAA;IAER,4BAA4B,CAAC,IAAI,CAAC,CAAA;IAElC,mCAAmC,CAAC,WAAW,EAAE,aAAa,CAAC,CAAA;IAE/D,qBAAqB;IACrB,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAEzB,mBAAmB;IACnB,cAAc,CAAC,KAAK,CAAC,CAAA;IAErB,6BAA6B;IAC7B,uBAAuB,CAAC,wBAAwB,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEnF,yBAAyB;IACzB,uBAAuB,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,EAAE,CAAC,CAAA;IAEzE,4BAA4B;IAC5B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAC/D,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAA;IAChD,CAAC;IAED,4BAA4B;IAC5B,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;IAC5D,CAAC;IAED,2BAA2B;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC,aAAa,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAA;IAC1D,CAAC;IAED,6BAA6B;IAC7B,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAA;IAC9D,CAAC;IAED,0BAA0B;IAC1B,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAA;IACxD,CAAC;IAED,4BAA4B;IAC5B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAC9D,CAAC;IAED,wBAAwB;IACxB,mBAAmB,CAAC,UAAU,CAAC,CAAA;IAE/B,sBAAsB;IACtB,IAAI,WAAW,EAAE,CAAC;QAChB,oBAAoB,CAAC,WAAW,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAS;IACvC,gCAAgC;IAChC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAA;IAEzF,mCAAmC;IACnC,MAAM,uBAAuB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;IAErD,wBAAwB;IACxB,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAA;IAE9C,2BAA2B;IAC3B,6CAA6C;IAC7C,MAAM,WAAW,EAAE,CAAA;IAEnB,wCAAwC;IACxC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,sBAAsB,CAAC,gBAAgB,CAAC,CAAA;IAChD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAU,GAAG,EAAE;IAC9C,UAAU,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAA;IAE7C,4CAA4C;IAC5C,IAAI,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC7D,CAAC;IAED,sFAAsF;IACtF,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,UAAU,IAAI,GAAG,CAAA;IACnB,CAAC;IAED,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACvD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAM;IACR,CAAC;IAED,+CAA+C;IAC/C,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,YAAY,GAAG,GAAG,GAAG,YAAY,CAAA;IACnC,CAAC;IAED,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,GAAW,EAAE,MAAc;IAC/E,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wBAAwB,MAAM,cAAc,CAAC,CAAA;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA+B;IAC9D,IAAI,OAAO,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,IAAI,OAAO,0GAA0G,CAAC,CAAA;IACxI,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,YAAoB,EAAE,aAAqB;IAC7F,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,OAAe;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;IACxD,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/D,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACpH,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,yBAAyB,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAW;IACxC,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAA;IAClF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAAa,EAAE,eAAuB;IAClF,sHAAsH;IACtH,MAAM,WAAW,GAAG,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,oDAAoD,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;IACxK,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5D,MAAM,IAAI,KAAK,CAAC,uCAAuC,WAAW,KAAK,GAAG,EAAE,CAAC,CAAA;IAC/E,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACjD,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,+BAA+B,WAAW,qDAAqD,WAAW,EAAE,CAAC,CAAA;IAC/H,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAe;IACjD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gBAAgB,UAAU,iBAAiB,CAAC,CAAA;IAC9D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/B,MAAM,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAA;IACvC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,gBAAqB;IAChE,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,eAAe,EAAE,CAAC;QACpB,wDAAwD;QACxD,MAAM,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,gBAAgB,GAAG,CAAC,CAAA;QACrF,CAAC,CAAC,CAAA;IACJ,CAAC;SAAM,CAAC;QACN,EAAE,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,oBAA4B,EAAE,QAAiB;IAC5E,gBAAgB;IAChB,IAAI,oBAAoB,EAAE,CAAC;QACzB,iDAAiD;QACjD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAA;QAC9C,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QACxG,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAA;QACvE,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YACjC,MAAM,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AAEH,MAAM,UAAU,sBAAsB,CAAC,aAAqB;IAC1D,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;IACzE,OAAO,CACL,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,aAAa,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,YAAY;QAC7I,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;IACpC,CAAC,CAAC,IAAI,IAAI,CACX,CAAA;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,MAAM,eAAe,GAAG,CAAC,GAAa,EAAE,GAAW,EAAE,EAAE;QACrD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAA;IAC7C,CAAC,CAAA;IACD,IAAI,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC,EAAE,CAAC;QAChD,OAAO,UAAU,CAAA;IACnB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAA;AACvD,CAAC"}
1
+ {"version":3,"file":"sanitize-argument.js","sourceRoot":"","sources":["../src/sanitize-argument.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,KAAK,CAAA;AACxC,OAAO,UAAU,MAAM,MAAM,CAAA;AAC7B,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AACrC,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,UAAU,MAAM,iBAAiB,CAAA;AACxC,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEpC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACjD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;AAC1C,MAAM,uBAAuB,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;AAEzD,6EAA6E;AAC7E,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAS;IAC1C,2BAA2B;IAC3B,MAAM,EACJ,QAAQ,EACR,aAAa,EACb,cAAc,EACd,KAAK,EACL,UAAU,EACV,KAAK,EACL,QAAQ,EACR,wBAAwB,EACxB,oBAAoB,EACpB,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,iBAAiB,EACjB,eAAe,GAChB,GAAG,IAAI,CAAA;IAER,4BAA4B,CAAC,IAAI,CAAC,CAAA;IAElC,+BAA+B,CAAC,aAAa,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAA;IAE9E,qBAAqB,CAAC,QAAQ,EAAE,aAAa,EAAE,cAAc,CAAC,CAAA;IAE9D,sBAAsB;IACtB,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAE3B,mBAAmB;IACnB,cAAc,CAAC,KAAK,CAAC,CAAA;IAErB,0BAA0B,CAAC,kBAAkB,CAAC,CAAA;IAE9C,6BAA6B;IAC7B,uBAAuB,CAAC,wBAAwB,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEnF,yBAAyB;IACzB,uBAAuB,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,EAAE,CAAC,CAAA;IAEzE,4BAA4B;IAC5B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QAC/D,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC7C,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,8BAA8B,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;IAEtD,6BAA6B;IAC7B,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,oBAAoB,CAAC,eAAe,CAAC,CAAA;IAC9D,CAAC;IAED,0BAA0B;IAC1B,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAA;IACxD,CAAC;IAED,4BAA4B;IAC5B,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAC9D,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAA;IACxE,CAAC;IAED,kBAAkB;IAClB,cAAc,CAAC,KAAK,CAAC,CAAA;IACrB,wBAAwB;IACxB,mBAAmB,CAAC,UAAU,CAAC,CAAA;IAE/B,sBAAsB;IACtB,IAAI,WAAW,EAAE,CAAC;QAChB,oBAAoB,CAAC,WAAW,CAAC,CAAA;IACnC,CAAC;IAED,6BAA6B;IAC7B,gBAAgB,CAAC,aAAa,EAAE,aAAa,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAA;AACpF,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAS;IACvC,gCAAgC;IAChC,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAA;IAEzF,mCAAmC;IACnC,MAAM,uBAAuB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAA;IAErD,wBAAwB;IACxB,MAAM,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAA;IAE9C,2BAA2B;IAC3B,6CAA6C;IAC7C,MAAM,WAAW,EAAE,CAAA;IAEnB,wCAAwC;IACxC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,sBAAsB,CAAC,gBAAgB,CAAC,CAAA;IAChD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,YAAoB;IACvD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAM;IACR,CAAC;IAED,+CAA+C;IAC/C,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,YAAY,GAAG,GAAG,GAAG,YAAY,CAAA;IACnC,CAAC;IAED,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,GAAW,EAAE,MAAc;IAC/E,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wBAAwB,MAAM,cAAc,CAAC,CAAA;IACrE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAA4B;IAC5D,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAA2B,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,IAAI,QAAQ,uEAAuE,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrI,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,aAAqB,EAAE,iBAAyB,EAAE,WAA8B;IAC9H,IAAI,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAA;IAChI,CAAC;IACD,IAAI,iBAAiB,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACxG,MAAM,IAAI,KAAK,CAAC,iHAAiH,CAAC,CAAA;IACpI,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAgB,EAAE,aAAqB,EAAE,cAAsB;IACnG,IAAI,QAAQ,IAAI,aAAa,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,QAAQ,IAAI,cAAc,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAA;IACpF,CAAC;IACD,IAAI,aAAa,IAAI,cAAc,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAA;IACzF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,OAAe;IAC1D,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;IACxD,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/D,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACpH,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,yBAAyB,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAW;IACxC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;QAC/B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8HAA8H,CAAC,CAAA;QACjJ,CAAC;QACD,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;QACrF,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,kBAAuB;IAChE,IAAI,kBAAkB,KAAK,SAAS,IAAI,kBAAkB,KAAK,IAAI,EAAE,CAAC;QACpE,MAAM,qBAAqB,GAAG,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACxD,IAAI,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QACzD,CAAC;QACD,IAAI,qBAAqB,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAC3D,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,UAAe,EAAE,UAAe;IAC7E,IAAI,UAAU,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,IAAI,UAAU,kGAAkG,CAAC,CAAA;IACnI,CAAC;IACD,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAAa,EAAE,eAAuB;IAClF,sHAAsH;IACtH,MAAM,WAAW,GAAG,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,oDAAoD,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;IACxK,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5D,MAAM,IAAI,KAAK,CAAC,uCAAuC,WAAW,KAAK,GAAG,gFAAgF,CAAC,CAAA;IAC7J,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACjD,IAAI,CAAE,WAAsB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,+BAA+B,WAAW,qDAAqD,WAAW,EAAE,CAAC,CAAA;IAC/H,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,GAAQ,CAAA;IACZ,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IACnG,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,uEAAuE,GAAG,CAAC,QAAQ,iBAAiB,CAAC,CAAA;IACvH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAe;IACjD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gBAAgB,UAAU,iBAAiB,CAAC,CAAA;IAC9D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/B,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;IACxC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAA;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,gBAAqB;IAChE,MAAM,eAAe,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACzD,IAAI,eAAe,EAAE,CAAC;QACpB,wDAAwD;QACxD,MAAM,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,gBAAgB,GAAG,CAAC,CAAA;QACrF,CAAC,CAAC,CAAA;IACJ,CAAC;SAAM,CAAC;QACN,EAAE,CAAC,YAAY,CAAC,gBAAgB,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,oBAA4B,EAAE,QAAiB;IAC5E,gBAAgB;IAChB,IAAI,oBAAoB,EAAE,CAAC;QACzB,iDAAiD;QACjD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAA;QAC9C,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QACxG,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,YAAY,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAA;QACvE,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YACjC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AAEH,MAAM,UAAU,sBAAsB,CAAC,aAAqB;IAC1D,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;IACzE,OAAO,CACL,CAAC,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,aAAa,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,YAAY;QAC7I,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;IACpC,CAAC,CAAC,IAAI,IAAI,CACX,CAAA;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,aAAkB,EAAE,aAAkB,EAAE,iBAAsB,EAAE,eAAoB;IACnH,IAAI,aAAa,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;IACnF,CAAC;IACD,IAAI,iBAAiB,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;IACvF,CAAC;IACD,IAAI,eAAe,IAAI,CAAC,aAAa,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;IACrF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,MAAM,eAAe,GAAG,CAAC,GAAa,EAAE,GAAW,EAAE,EAAE;QACrD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC,CAAA;IAC7C,CAAC,CAAA;IACD,IAAI,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC,EAAE,CAAC;QAChD,OAAO,UAAU,CAAA;IACnB,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAA;AACvD,CAAC"}
@@ -0,0 +1,36 @@
1
+ import { Creator } from '@openzim/libzim';
2
+ import { Dump } from '../Dump.js';
3
+ declare class FileManager {
4
+ private static instance;
5
+ static getInstance(): FileManager;
6
+ private hosts;
7
+ private get filesToDownloadXPath();
8
+ private constructor();
9
+ /**
10
+ * Reset internal state for a new dump cycle.
11
+ */
12
+ reset(): void;
13
+ /**
14
+ * Add a single file to download. Checks dedup via filesToDownloadXPath.
15
+ * On first add: writes to both filesToDownloadXPath and host queue.
16
+ * On resolution upgrade: updates filesToDownloadXPath only (queue already has the path).
17
+ *
18
+ * This method is typically called multiple times when the file is an image and the scraper detects
19
+ * a second use of an image already seen but with a higher resolution. Could be that we call this
20
+ * method twice with different urls / data for other usage in the future.
21
+ */
22
+ addFileToProcess(path: string, detail: FileDetail): Promise<void>;
23
+ private _addFileToProcess;
24
+ /**
25
+ * Batch version of addFileToProcess. Uses getMany for efficiency.
26
+ */
27
+ addManyFilesToProcess(files: KVS<FileDetail>): Promise<void>;
28
+ _addManyFilesToProcess(files: KVS<FileDetail>): Promise<void>;
29
+ private pushToHostQueue;
30
+ /**
31
+ * Start download workers. Queues are already populated via addFileToProcess/addManyFilesToProcess.
32
+ */
33
+ startDownloading(zimCreator: Creator, dump: Dump): Promise<void>;
34
+ }
35
+ declare const _default: FileManager;
36
+ export default _default;