hydra-crawler 1.4.6 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (305) hide show
  1. package/dist/apis/autocomplete.api.d.ts +7 -0
  2. package/dist/apis/autocomplete.api.js +15 -9
  3. package/dist/apis/autocomplete.api.js.map +1 -0
  4. package/dist/apis/bugs.api.d.ts +7 -0
  5. package/dist/apis/bugs.api.js +21 -15
  6. package/dist/apis/bugs.api.js.map +1 -0
  7. package/dist/apis/crawl.api.d.ts +7 -0
  8. package/dist/apis/crawl.api.js +15 -9
  9. package/dist/apis/crawl.api.js.map +1 -0
  10. package/dist/apis/domains.api.d.ts +7 -0
  11. package/dist/apis/domains.api.js +24 -19
  12. package/dist/apis/domains.api.js.map +1 -0
  13. package/dist/apis/images.api.d.ts +7 -0
  14. package/dist/apis/images.api.js +20 -14
  15. package/dist/apis/images.api.js.map +1 -0
  16. package/dist/apis/statistics.api.d.ts +8 -0
  17. package/dist/apis/statistics.api.js +27 -20
  18. package/dist/apis/statistics.api.js.map +1 -0
  19. package/dist/apis/test.api.d.ts +5 -0
  20. package/dist/apis/test.api.js +15 -9
  21. package/dist/apis/test.api.js.map +1 -0
  22. package/dist/apis/urls.api.d.ts +7 -0
  23. package/dist/apis/urls.api.js +21 -15
  24. package/dist/apis/urls.api.js.map +1 -0
  25. package/dist/apps/cleanup.app.d.ts +19 -0
  26. package/dist/apps/cleanup.app.js +118 -100
  27. package/dist/apps/cleanup.app.js.map +1 -0
  28. package/dist/apps/cross-populate-export.app.d.ts +12 -0
  29. package/dist/apps/cross-populate-export.app.js +60 -47
  30. package/dist/apps/cross-populate-export.app.js.map +1 -0
  31. package/dist/apps/cross-populate-import.app.d.ts +12 -0
  32. package/dist/apps/cross-populate-import.app.js +64 -51
  33. package/dist/apps/cross-populate-import.app.js.map +1 -0
  34. package/dist/apps/denylist.app.d.ts +17 -0
  35. package/dist/apps/denylist.app.js +115 -98
  36. package/dist/apps/denylist.app.js.map +1 -0
  37. package/dist/apps/expire.app.d.ts +19 -0
  38. package/dist/apps/expire.app.js +44 -31
  39. package/dist/apps/expire.app.js.map +1 -0
  40. package/dist/apps/extract-text.app.d.ts +8 -0
  41. package/dist/apps/extract-text.app.js +43 -35
  42. package/dist/apps/extract-text.app.js.map +1 -0
  43. package/dist/apps/hydra.app.d.ts +34 -0
  44. package/dist/apps/hydra.app.js +150 -137
  45. package/dist/apps/hydra.app.js.map +1 -0
  46. package/dist/apps/import.app.d.ts +11 -0
  47. package/dist/apps/import.app.js +44 -32
  48. package/dist/apps/import.app.js.map +1 -0
  49. package/dist/apps/internal-hydra-common.app.d.ts +28 -0
  50. package/dist/apps/internal-hydra-common.app.js +5 -11
  51. package/dist/apps/internal-hydra-common.app.js.map +1 -0
  52. package/dist/apps/query.app.d.ts +20 -0
  53. package/dist/apps/query.app.js +63 -49
  54. package/dist/apps/query.app.js.map +1 -0
  55. package/dist/apps/reattempt.app.d.ts +17 -0
  56. package/dist/apps/reattempt.app.js +66 -53
  57. package/dist/apps/reattempt.app.js.map +1 -0
  58. package/dist/apps/requeue-domain.app.d.ts +13 -0
  59. package/dist/apps/requeue-domain.app.js +50 -37
  60. package/dist/apps/requeue-domain.app.js.map +1 -0
  61. package/dist/apps/seed.app.d.ts +15 -0
  62. package/dist/apps/seed.app.js +53 -40
  63. package/dist/apps/seed.app.js.map +1 -0
  64. package/dist/apps/startup.app.d.ts +11 -0
  65. package/dist/apps/startup.app.js +51 -38
  66. package/dist/apps/startup.app.js.map +1 -0
  67. package/dist/apps/unarchive.app.d.ts +15 -0
  68. package/dist/apps/unarchive.app.js +67 -54
  69. package/dist/apps/unarchive.app.js.map +1 -0
  70. package/dist/classes/cleaner.d.ts +12 -0
  71. package/dist/classes/cleaner.js +227 -207
  72. package/dist/classes/cleaner.js.map +1 -0
  73. package/dist/classes/crawler.d.ts +34 -0
  74. package/dist/classes/crawler.js +248 -241
  75. package/dist/classes/crawler.js.map +1 -0
  76. package/dist/classes/dns.d.ts +3 -0
  77. package/dist/classes/dns.js +10 -13
  78. package/dist/classes/dns.js.map +1 -0
  79. package/dist/classes/expirer.d.ts +10 -0
  80. package/dist/classes/expirer.js +107 -94
  81. package/dist/classes/expirer.js.map +1 -0
  82. package/dist/classes/expiry.d.ts +8 -0
  83. package/dist/classes/expiry.js +16 -19
  84. package/dist/classes/expiry.js.map +1 -0
  85. package/dist/classes/lists.d.ts +9 -0
  86. package/dist/classes/lists.js +13 -18
  87. package/dist/classes/lists.js.map +1 -0
  88. package/dist/classes/robot.d.ts +15 -0
  89. package/dist/classes/robot.js +40 -30
  90. package/dist/classes/robot.js.map +1 -0
  91. package/dist/classes/tracker.d.ts +25 -0
  92. package/dist/classes/tracker.js +82 -64
  93. package/dist/classes/tracker.js.map +1 -0
  94. package/dist/cli.d.ts +1 -0
  95. package/dist/cli.js +72 -65
  96. package/dist/cli.js.map +1 -0
  97. package/dist/enums/eavailable-strategy.d.ts +4 -0
  98. package/dist/enums/eavailable-strategy.js +3 -5
  99. package/dist/enums/eavailable-strategy.js.map +1 -0
  100. package/dist/enums/elist.d.ts +7 -0
  101. package/dist/enums/elist.js +7 -11
  102. package/dist/enums/elist.js.map +1 -0
  103. package/dist/enums/eserver.d.ts +8 -0
  104. package/dist/enums/eserver.js +3 -5
  105. package/dist/enums/eserver.js.map +1 -0
  106. package/dist/enums/ex-powered-by.d.ts +6 -0
  107. package/dist/enums/ex-powered-by.js +3 -5
  108. package/dist/enums/ex-powered-by.js.map +1 -0
  109. package/dist/helpers/matcher.d.ts +5 -0
  110. package/dist/helpers/matcher.js +2 -5
  111. package/dist/helpers/matcher.js.map +1 -0
  112. package/dist/helpers/random.d.ts +4 -0
  113. package/dist/helpers/random.js +2 -5
  114. package/dist/helpers/random.js.map +1 -0
  115. package/dist/helpers/utf-decoder.d.ts +4 -0
  116. package/dist/helpers/utf-decoder.js +3 -6
  117. package/dist/helpers/utf-decoder.js.map +1 -0
  118. package/dist/interfaces/iexpiry.d.ts +7 -0
  119. package/dist/interfaces/iexpiry.js +9 -13
  120. package/dist/interfaces/iexpiry.js.map +1 -0
  121. package/dist/interfaces/imatch.d.ts +6 -0
  122. package/dist/interfaces/imatch.js +6 -9
  123. package/dist/interfaces/imatch.js.map +1 -0
  124. package/dist/interfaces/iparser-config.d.ts +4 -0
  125. package/dist/interfaces/iparser-config.js +4 -7
  126. package/dist/interfaces/iparser-config.js.map +1 -0
  127. package/dist/interfaces/iparser.d.ts +8 -0
  128. package/dist/interfaces/iparser.js +2 -2
  129. package/dist/interfaces/iparser.js.map +1 -0
  130. package/dist/interfaces/irequest-outcome.d.ts +11 -0
  131. package/dist/interfaces/irequest-outcome.js +2 -2
  132. package/dist/interfaces/irequest-outcome.js.map +1 -0
  133. package/dist/interfaces/iserver.d.ts +4 -0
  134. package/dist/interfaces/iserver.js +2 -2
  135. package/dist/interfaces/iserver.js.map +1 -0
  136. package/dist/parsers/accessibility-metrics.parser.d.ts +11 -0
  137. package/dist/parsers/accessibility-metrics.parser.js +34 -26
  138. package/dist/parsers/accessibility-metrics.parser.js.map +1 -0
  139. package/dist/parsers/asp-error.parser.d.ts +12 -0
  140. package/dist/parsers/asp-error.parser.js +36 -28
  141. package/dist/parsers/asp-error.parser.js.map +1 -0
  142. package/dist/parsers/bad-words.parser.d.ts +10 -0
  143. package/dist/parsers/bad-words.parser.js +21 -13
  144. package/dist/parsers/bad-words.parser.js.map +1 -0
  145. package/dist/parsers/complex-english.parser.d.ts +15 -0
  146. package/dist/parsers/complex-english.parser.js +33 -25
  147. package/dist/parsers/complex-english.parser.js.map +1 -0
  148. package/dist/parsers/data.parser.d.ts +14 -0
  149. package/dist/parsers/data.parser.js +12 -16
  150. package/dist/parsers/data.parser.js.map +1 -0
  151. package/dist/parsers/dictionary.parser.d.ts +19 -0
  152. package/dist/parsers/dictionary.parser.js +47 -39
  153. package/dist/parsers/dictionary.parser.js.map +1 -0
  154. package/dist/parsers/html.parser.d.ts +13 -0
  155. package/dist/parsers/html.parser.js +4 -8
  156. package/dist/parsers/html.parser.js.map +1 -0
  157. package/dist/parsers/hyperlinks.parser.d.ts +20 -0
  158. package/dist/parsers/hyperlinks.parser.js +82 -77
  159. package/dist/parsers/hyperlinks.parser.js.map +1 -0
  160. package/dist/parsers/image-tags.parser.d.ts +19 -0
  161. package/dist/parsers/image-tags.parser.js +31 -35
  162. package/dist/parsers/image-tags.parser.js.map +1 -0
  163. package/dist/parsers/jpeg.parser.d.ts +11 -0
  164. package/dist/parsers/jpeg.parser.js +28 -20
  165. package/dist/parsers/jpeg.parser.js.map +1 -0
  166. package/dist/parsers/paragraphs.parser.d.ts +13 -0
  167. package/dist/parsers/paragraphs.parser.js +33 -40
  168. package/dist/parsers/paragraphs.parser.js.map +1 -0
  169. package/dist/parsers/parser.d.ts +19 -0
  170. package/dist/parsers/parser.js +30 -17
  171. package/dist/parsers/parser.js.map +1 -0
  172. package/dist/parsers/php-error.parser.d.ts +12 -0
  173. package/dist/parsers/php-error.parser.js +42 -34
  174. package/dist/parsers/php-error.parser.js.map +1 -0
  175. package/dist/parsers/phrase.parser.d.ts +8 -0
  176. package/dist/parsers/phrase.parser.js +16 -11
  177. package/dist/parsers/phrase.parser.js.map +1 -0
  178. package/dist/parsers/regex.parser.d.ts +10 -0
  179. package/dist/parsers/regex.parser.js +30 -22
  180. package/dist/parsers/regex.parser.js.map +1 -0
  181. package/dist/parsers/server.parser.d.ts +11 -0
  182. package/dist/parsers/server.parser.js +58 -57
  183. package/dist/parsers/server.parser.js.map +1 -0
  184. package/dist/parsers/spelling.parser.d.ts +10 -0
  185. package/dist/parsers/spelling.parser.js +21 -13
  186. package/dist/parsers/spelling.parser.js.map +1 -0
  187. package/dist/parsers/string.parser.d.ts +8 -0
  188. package/dist/parsers/string.parser.js +5 -8
  189. package/dist/parsers/string.parser.js.map +1 -0
  190. package/dist/parsers/text.parser.d.ts +8 -0
  191. package/dist/parsers/text.parser.js +24 -18
  192. package/dist/parsers/text.parser.js.map +1 -0
  193. package/dist/parsers/words.parser.d.ts +11 -0
  194. package/dist/parsers/words.parser.js +32 -28
  195. package/dist/parsers/words.parser.js.map +1 -0
  196. package/dist/queries/complex-english.query.d.ts +2 -0
  197. package/dist/queries/complex-english.query.js +37 -38
  198. package/dist/queries/complex-english.query.js.map +1 -0
  199. package/dist/queries/flash-content.query.d.ts +2 -0
  200. package/dist/queries/flash-content.query.js +39 -30
  201. package/dist/queries/flash-content.query.js.map +1 -0
  202. package/dist/queries/linking-to-domains.query.d.ts +2 -0
  203. package/dist/queries/linking-to-domains.query.js +35 -27
  204. package/dist/queries/linking-to-domains.query.js.map +1 -0
  205. package/dist/queries/readability-score.query.d.ts +2 -0
  206. package/dist/queries/readability-score.query.js +21 -13
  207. package/dist/queries/readability-score.query.js.map +1 -0
  208. package/dist/servers/crawl.server.d.ts +35 -0
  209. package/dist/servers/crawl.server.js +133 -121
  210. package/dist/servers/crawl.server.js.map +1 -0
  211. package/dist/servers/express.server.d.ts +8 -0
  212. package/dist/servers/express.server.js +7 -10
  213. package/dist/servers/express.server.js.map +1 -0
  214. package/dist/servers/maintenance.server.d.ts +22 -0
  215. package/dist/servers/maintenance.server.js +42 -36
  216. package/dist/servers/maintenance.server.js.map +1 -0
  217. package/dist/servers/rest.server.d.ts +7 -0
  218. package/dist/servers/rest.server.js +40 -51
  219. package/dist/servers/rest.server.js.map +1 -0
  220. package/dist/servers/socket-io.server.d.ts +12 -0
  221. package/dist/servers/socket-io.server.js +48 -15
  222. package/dist/servers/socket-io.server.js.map +1 -0
  223. package/dist/services/database.service.d.ts +68 -0
  224. package/dist/services/database.service.js +527 -462
  225. package/dist/services/database.service.js.map +1 -0
  226. package/dist/types/tcrawl-config.d.ts +14 -0
  227. package/dist/types/tcrawl-config.js +14 -17
  228. package/dist/types/tcrawl-config.js.map +1 -0
  229. package/dist/types/thydra-config.d.ts +4 -0
  230. package/dist/types/thydra-config.js +4 -7
  231. package/dist/types/thydra-config.js.map +1 -0
  232. package/dist/types/tparser-ctor.d.ts +7 -0
  233. package/dist/types/tparser-ctor.js +2 -2
  234. package/dist/types/tparser-ctor.js.map +1 -0
  235. package/dist/types/tquery.d.ts +7 -0
  236. package/dist/types/tquery.js +2 -2
  237. package/dist/types/tquery.js.map +1 -0
  238. package/dist/types/trobots-config.d.ts +4 -0
  239. package/dist/types/trobots-config.js +4 -7
  240. package/dist/types/trobots-config.js.map +1 -0
  241. package/package.json +41 -29
  242. package/angular/10-es2015.bacd4ae5dd7913ce55f0.js +0 -1
  243. package/angular/10-es5.bacd4ae5dd7913ce55f0.js +0 -1
  244. package/angular/11-es2015.0f031dcf752d1e8eda6b.js +0 -1
  245. package/angular/11-es5.0f031dcf752d1e8eda6b.js +0 -1
  246. package/angular/3rdpartylicenses.txt +0 -1127
  247. package/angular/5-es2015.951498ca9c1bc74e57bf.js +0 -1
  248. package/angular/5-es5.951498ca9c1bc74e57bf.js +0 -1
  249. package/angular/6-es2015.65f680261a3506b88381.js +0 -1
  250. package/angular/6-es5.65f680261a3506b88381.js +0 -1
  251. package/angular/7-es2015.625197f3af1dbf3e805d.js +0 -1
  252. package/angular/7-es5.625197f3af1dbf3e805d.js +0 -1
  253. package/angular/8-es2015.55518901987a5b834309.js +0 -1
  254. package/angular/8-es5.55518901987a5b834309.js +0 -1
  255. package/angular/9-es2015.6cc9bde262564e7836f2.js +0 -1
  256. package/angular/9-es5.6cc9bde262564e7836f2.js +0 -1
  257. package/angular/Roboto-Black.41ed1105a6ebb8ffe34e.woff2 +0 -0
  258. package/angular/Roboto-Black.937491dfcbe64ca9a9f1.woff +0 -0
  259. package/angular/Roboto-BlackItalic.2e1ee657996854c6f427.woff +0 -0
  260. package/angular/Roboto-BlackItalic.50ca4c51ebc27e7e7d2f.woff2 +0 -0
  261. package/angular/Roboto-Bold.73288d91c325e82a5b92.woff +0 -0
  262. package/angular/Roboto-Bold.92fbd4e93cf0a5dbebaa.woff2 +0 -0
  263. package/angular/Roboto-BoldItalic.5f600d98a73d800ae575.woff2 +0 -0
  264. package/angular/Roboto-BoldItalic.6d89acbd21d7e3fbecb2.woff +0 -0
  265. package/angular/Roboto-Light.c27d89ac77468ae18f28.woff2 +0 -0
  266. package/angular/Roboto-Light.d923dfafc0c5183b59aa.woff +0 -0
  267. package/angular/Roboto-LightItalic.506274c7228cf81cae4d.woff2 +0 -0
  268. package/angular/Roboto-LightItalic.d4b8c137518d9d92bb28.woff +0 -0
  269. package/angular/Roboto-Medium.092c6130df8fd2199888.woff +0 -0
  270. package/angular/Roboto-Medium.1d3bced88509b0838984.woff2 +0 -0
  271. package/angular/Roboto-MediumItalic.18ff1628c628080166c1.woff +0 -0
  272. package/angular/Roboto-MediumItalic.d620b8f53f75966fe42e.woff2 +0 -0
  273. package/angular/Roboto-Regular.64cfb66c866ea50cad47.woff2 +0 -0
  274. package/angular/Roboto-Regular.e02e9d6ff5547f7e9962.woff +0 -0
  275. package/angular/Roboto-RegularItalic.4dd2af1e8df532f41db8.woff2 +0 -0
  276. package/angular/Roboto-RegularItalic.5ea38fff9eebef99c5df.woff +0 -0
  277. package/angular/Roboto-Thin.dbd56bd3357dc3617fe5.woff2 +0 -0
  278. package/angular/Roboto-Thin.e7f7c82374bd0ebef14b.woff +0 -0
  279. package/angular/Roboto-ThinItalic.5dd9349c940073834e9a.woff +0 -0
  280. package/angular/Roboto-ThinItalic.a8cef84f735ef887abdc.woff2 +0 -0
  281. package/angular/assets/config/app-config.json +0 -16
  282. package/angular/assets/images/splashbg.jpg +0 -0
  283. package/angular/assets/web-app-commons/fonts/material-icons/MaterialDesignIcons-Community-2.7.94.woff +0 -0
  284. package/angular/assets/web-app-commons/fonts/material-icons/MaterialDesignIcons-Community-2.7.94.woff2 +0 -0
  285. package/angular/assets/web-app-commons/fonts/material-icons/material-design-icons-community.css +0 -11293
  286. package/angular/favicon.ico +0 -0
  287. package/angular/flUhRq6tzZclQEJ-Vdg-IuiaDsNa.f2a0933406f783065152.woff +0 -0
  288. package/angular/flUhRq6tzZclQEJ-Vdg-IuiaDsNc.6467d9a24f234e8e8e07.woff2 +0 -0
  289. package/angular/index.html +0 -16
  290. package/angular/main-es2015.3a582572476c7f292e52.js +0 -1
  291. package/angular/main-es5.3a582572476c7f292e52.js +0 -1
  292. package/angular/polyfills-es2015.7df68534018bc2f6cb09.js +0 -1
  293. package/angular/polyfills-es5.e79468f406fae2989221.js +0 -1
  294. package/angular/runtime-es2015.6d2cff76cdb2790d3308.js +0 -1
  295. package/angular/runtime-es5.6d2cff76cdb2790d3308.js +0 -1
  296. package/angular/styles.c5c6c2534225b85c4ff0.css +0 -1
  297. package/config/bad-words.json +0 -1
  298. package/config/complex-english.json +0 -400
  299. package/config/hydra-auth.json +0 -8
  300. package/config/hydra-crawler.json +0 -84
  301. package/config/list-allow.json +0 -171
  302. package/config/list-deny.json +0 -248
  303. package/config/list-expiry.json +0 -7
  304. package/config/schedule.json +0 -25
  305. package/config/spelling.json +0 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utf-decoder.js","sourceRoot":"","sources":["../../src/helpers/utf-decoder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,OAAgB,UAAU;IACxB,MAAM,CAAC,UAAU,CAAC,QAAgB;QACxC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAE7C,IAAI;YACH,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAE5B,MAAM,KAAK,GAAW,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACjD,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YACnC,QAAQ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBAC9B,KAAK,UAAU;oBACd,OAAO,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBACrC,KAAK,UAAU;oBACd,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAC3D;oBACC,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;aAC5B;SACD;QAAC,OAAO,EAAE,EAAE;YACZ,kDAAkD;YAClD,IAAI;gBACH,OAAO,QAAQ,CAAC,QAAQ,EAAE,CAAC;aAC3B;YAAC,OAAO,GAAG,EAAE;gBACb,SAAS;aACT;SACD;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;CACD"}
@@ -0,0 +1,7 @@
1
+ import { IMatch } from './imatch';
2
+ export interface IExpiry {
3
+ match: IMatch;
4
+ expiry: number;
5
+ }
6
+ export declare function isIExpiry(test: unknown): test is IExpiry;
7
+ export declare function toIExpiry(data: unknown): IExpiry;
@@ -1,19 +1,15 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.toIExpiry = exports.isIExpiry = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- const imatch_1 = require("./imatch");
6
- function isIExpiry(test) {
1
+ import { commonsTypeHasPropertyNumber, commonsTypeHasPropertyObject, commonsTypeHasPropertyString } from 'tscommons-es-core';
2
+ import { isIMatch } from './imatch';
3
+ export function isIExpiry(test) {
7
4
  // Can't use hasPropertyT as we need to pass a true parameter
8
- if (!tscommons_core_1.CommonsType.hasPropertyObject(test, 'match'))
5
+ if (!commonsTypeHasPropertyObject(test, 'match'))
9
6
  return false;
10
- if (!imatch_1.isIMatch(test.match, true))
7
+ if (!isIMatch(test.match, true))
11
8
  return false;
12
- if (!tscommons_core_1.CommonsType.hasPropertyNumber(test, 'expiry'))
9
+ if (!commonsTypeHasPropertyNumber(test, 'expiry'))
13
10
  return false;
14
11
  return true;
15
12
  }
16
- exports.isIExpiry = isIExpiry;
17
13
  var EInterval;
18
14
  (function (EInterval) {
19
15
  EInterval["SECOND"] = "second";
@@ -58,12 +54,12 @@ function calculateExpiry(expiry) {
58
54
  throw new Error(`Remaining expiry: ${expiry.trim()}`);
59
55
  return total;
60
56
  }
61
- function toIExpiry(data) {
62
- if (tscommons_core_1.CommonsType.hasPropertyString(data, 'expiry')) {
57
+ export function toIExpiry(data) {
58
+ if (commonsTypeHasPropertyString(data, 'expiry')) {
63
59
  data['expiry'] = calculateExpiry(data['expiry']);
64
60
  }
65
61
  if (!isIExpiry(data))
66
62
  throw new Error('Object is not an instance of IExpiry');
67
63
  return data;
68
64
  }
69
- exports.toIExpiry = toIExpiry;
65
+ //# sourceMappingURL=iexpiry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iexpiry.js","sourceRoot":"","sources":["../../src/interfaces/iexpiry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,4BAA4B,EAAe,MAAM,mBAAmB,CAAC;AAE1I,OAAO,EAAU,QAAQ,EAAE,MAAM,UAAU,CAAC;AAO5C,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,6DAA6D;IAC7D,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhE,OAAO,IAAI,CAAC;AACb,CAAC;AAED,IAAK,SAQJ;AARD,WAAK,SAAS;IACZ,8BAAiB,CAAA;IACjB,8BAAiB,CAAA;IACjB,0BAAa,CAAA;IACb,wBAAW,CAAA;IACX,0BAAa,CAAA;IACb,4BAAe,CAAA;IACf,0BAAa,CAAA;AACf,CAAC,EARI,SAAS,KAAT,SAAS,QAQb;AAED,MAAM,eAAe,GAAW,CAAC,CAAC;AAClC,MAAM,eAAe,GAAW,eAAe,GAAG,EAAE,CAAC;AACrD,MAAM,aAAa,GAAW,eAAe,GAAG,EAAE,CAAC;AACnD,MAAM,YAAY,GAAW,aAAa,GAAG,EAAE,CAAC;AAChD,MAAM,aAAa,GAAW,YAAY,GAAG,CAAC,CAAC;AAC/C,MAAM,aAAa,GAAW,YAAY,GAAG,MAAM,CAAC;AACpD,MAAM,cAAc,GAAW,aAAa,GAAG,EAAE,CAAC;AAElD,MAAM,cAAc,GAAmC;IACrD,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe;IACnC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,YAAY;IAC7B,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;IAC/B,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc;IACjC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa;CAChC,CAAC;AAEF,SAAS,eAAe,CAAC,MAAc;IACtC,IAAI,KAAK,GAAW,CAAC,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;QAC/C,OAAO,IAAI,EAAE;YACZ,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,4BAA4B,IAAI,mBAAmB,EAAE,GAAG,CAAC,CAAC;YAC3F,MAAM,MAAM,GAAyB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,MAAM,KAAK,IAAI;gBAAE,MAAM;YAE3B,IAAI,KAAK,GAAW,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5C,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC;YAC9B,KAAK,IAAI,KAAK,CAAC;YAEf,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SACxC;KACD;IAED,IAAI,EAAE,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAEhF,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAa;IACtC,IAAI,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAW,CAAC,CAAC;KAC3D;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAE9E,OAAO,IAAI,CAAC;AACb,CAAC"}
@@ -0,0 +1,6 @@
1
+ export interface IMatch {
2
+ hostname?: string;
3
+ pathname?: string;
4
+ search?: string;
5
+ }
6
+ export declare function isIMatch(test: unknown, allowBlank?: boolean): test is IMatch;
@@ -1,16 +1,13 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.isIMatch = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- function isIMatch(test, allowBlank = false) {
6
- if (!tscommons_core_1.CommonsType.hasPropertyStringOrUndefined(test, 'hostname'))
1
+ import { commonsTypeHasPropertyStringOrUndefined } from 'tscommons-es-core';
2
+ export function isIMatch(test, allowBlank = false) {
3
+ if (!commonsTypeHasPropertyStringOrUndefined(test, 'hostname'))
7
4
  return false;
8
- if (!tscommons_core_1.CommonsType.hasPropertyStringOrUndefined(test, 'pathname'))
5
+ if (!commonsTypeHasPropertyStringOrUndefined(test, 'pathname'))
9
6
  return false;
10
- if (!tscommons_core_1.CommonsType.hasPropertyStringOrUndefined(test, 'search'))
7
+ if (!commonsTypeHasPropertyStringOrUndefined(test, 'search'))
11
8
  return false;
12
9
  if (!test.hostname && !test.pathname && !test.search && !allowBlank)
13
10
  return false;
14
11
  return true;
15
12
  }
16
- exports.isIMatch = isIMatch;
13
+ //# sourceMappingURL=imatch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"imatch.js","sourceRoot":"","sources":["../../src/interfaces/imatch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uCAAuC,EAAE,MAAM,mBAAmB,CAAC;AAQ5E,MAAM,UAAU,QAAQ,CAAC,IAAa,EAAE,aAAsB,KAAK;IAClE,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,UAAU,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAE3E,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU;QAAE,OAAO,KAAK,CAAC;IAElF,OAAO,IAAI,CAAC;AACb,CAAC"}
@@ -0,0 +1,4 @@
1
+ export interface IParserConfig {
2
+ enabled: boolean;
3
+ }
4
+ export declare function isIParserConfig(test: unknown): test is IParserConfig;
@@ -1,10 +1,7 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.isIParserConfig = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- function isIParserConfig(test) {
6
- if (!tscommons_core_1.CommonsType.hasPropertyBoolean(test, 'enabled'))
1
+ import { commonsTypeHasPropertyBoolean } from 'tscommons-es-core';
2
+ export function isIParserConfig(test) {
3
+ if (!commonsTypeHasPropertyBoolean(test, 'enabled'))
7
4
  return false;
8
5
  return true;
9
6
  }
10
- exports.isIParserConfig = isIParserConfig;
7
+ //# sourceMappingURL=iparser-config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iparser-config.js","sourceRoot":"","sources":["../../src/interfaces/iparser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AAQlE,MAAM,UAAU,eAAe,CAAC,IAAa;IAC5C,IAAI,CAAC,6BAA6B,CAAC,IAAI,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IAElE,OAAO,IAAI,CAAC;AACb,CAAC"}
@@ -0,0 +1,8 @@
1
+ import { DatabaseService } from '../services/database.service';
2
+ export interface IParser {
3
+ isEnabled(): boolean;
4
+ supports(contentType: string, isAllow: boolean): boolean;
5
+ init(database: DatabaseService): Promise<void>;
6
+ parse(database: DatabaseService): Promise<void>;
7
+ links(): Promise<string[]>;
8
+ }
@@ -1,2 +1,2 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
2
+ //# sourceMappingURL=iparser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iparser.js","sourceRoot":"","sources":["../../src/interfaces/iparser.ts"],"names":[],"mappings":""}
@@ -0,0 +1,11 @@
1
+ /// <reference types="node" />
2
+ /// <reference types="node" />
3
+ import { IncomingHttpHeaders } from 'http';
4
+ export interface IRequestOutcome {
5
+ latency: number;
6
+ ip: string;
7
+ statusCode: number;
8
+ headers: IncomingHttpHeaders;
9
+ data?: Buffer;
10
+ exceeded: boolean;
11
+ }
@@ -1,2 +1,2 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
2
+ //# sourceMappingURL=irequest-outcome.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"irequest-outcome.js","sourceRoot":"","sources":["../../src/interfaces/irequest-outcome.ts"],"names":[],"mappings":""}
@@ -0,0 +1,4 @@
1
+ export interface IServer {
2
+ server?: string;
3
+ xPoweredBy?: string;
4
+ }
@@ -1,2 +1,2 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
1
+ export {};
2
+ //# sourceMappingURL=iserver.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iserver.js","sourceRoot":"","sources":["../../src/interfaces/iserver.ts"],"names":[],"mappings":""}
@@ -0,0 +1,11 @@
1
+ import { TKeyObject } from 'tscommons-es-core';
2
+ import { DatabaseService } from '../services/database.service';
3
+ import { IParserConfig } from '../interfaces/iparser-config';
4
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
5
+ import { TextParser } from './text.parser';
6
+ import { IDataConfig } from './data.parser';
7
+ export declare class AccessibilityMetricsParser extends TextParser<IDataConfig> {
8
+ private url?;
9
+ constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
10
+ protected parseText(database: DatabaseService, text: string): Promise<void>;
11
+ }
@@ -1,32 +1,40 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AccessibilityMetricsParser = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- const nodecommons_cli_1 = require("nodecommons-cli");
6
- const text_parser_1 = require("./text.parser");
7
- class AccessibilityMetricsParser extends text_parser_1.TextParser {
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { commonsStringAutomatedReadabilityIndex, commonsStringFleschKincaidReadingEase, commonsStringSplitSentences, commonsStringSplitWords } from 'tscommons-es-core';
11
+ import { commonsOutputDebug } from 'nodecommons-es-cli';
12
+ import { TextParser } from './text.parser';
13
+ export class AccessibilityMetricsParser extends TextParser {
8
14
  constructor(url, outcome, config) {
9
15
  super(outcome, config, 'accessibilityMetrics');
10
16
  this.url = url;
11
17
  }
12
- async parseText(database, text) {
13
- if (!this.url)
14
- return;
15
- if (text.length === 0) {
16
- await database.unsetData(this.url, 'accessibilityMetrics');
17
- return;
18
- }
19
- const paragraphs = text.split('\n');
20
- const all = paragraphs.join('. ');
21
- const metrics = {
22
- paragraphs: paragraphs.length,
23
- sentences: tscommons_core_1.CommonsString.splitSentences(all).length,
24
- words: tscommons_core_1.CommonsString.splitWords(all).length,
25
- fkre: tscommons_core_1.CommonsString.fleschKincaidReadingEase(all),
26
- ari: tscommons_core_1.CommonsString.automatedReadabilityIndex(all)
27
- };
28
- nodecommons_cli_1.CommonsOutput.debug(`Accessibility metrics are: ${JSON.stringify(metrics)}`);
29
- await database.setData(this.url, 'accessibilityMetrics', metrics);
18
+ parseText(database, text) {
19
+ return __awaiter(this, void 0, void 0, function* () {
20
+ if (!this.url)
21
+ return;
22
+ if (text.length === 0) {
23
+ yield database.unsetData(this.url, 'accessibilityMetrics');
24
+ return;
25
+ }
26
+ const paragraphs = text.split('\n');
27
+ const all = paragraphs.join('. ');
28
+ const metrics = {
29
+ paragraphs: paragraphs.length,
30
+ sentences: commonsStringSplitSentences(all).length,
31
+ words: commonsStringSplitWords(all).length,
32
+ fkre: commonsStringFleschKincaidReadingEase(all),
33
+ ari: commonsStringAutomatedReadabilityIndex(all)
34
+ };
35
+ commonsOutputDebug(`Accessibility metrics are: ${JSON.stringify(metrics)}`);
36
+ yield database.setData(this.url, 'accessibilityMetrics', metrics);
37
+ });
30
38
  }
31
39
  }
32
- exports.AccessibilityMetricsParser = AccessibilityMetricsParser;
40
+ //# sourceMappingURL=accessibility-metrics.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"accessibility-metrics.parser.js","sourceRoot":"","sources":["../../src/parsers/accessibility-metrics.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,sCAAsC,EAAE,qCAAqC,EAAE,2BAA2B,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAEpL,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAOxD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAW3C,MAAM,OAAO,0BAA2B,SAAQ,UAAuB;IACtE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CACH,OAAO,EACP,MAAM,EACN,sBAAsB,CACvB,CAAC;QARO,QAAG,GAAH,GAAG,CAAS;IAStB,CAAC;IAEe,SAAS,CAAC,QAAyB,EAAE,IAAY;;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;gBAC3D,OAAO;aACP;YAED,MAAM,UAAU,GAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,GAAG,GAAW,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1C,MAAM,OAAO,GAA0B;gBACrC,UAAU,EAAE,UAAU,CAAC,MAAM;gBAC7B,SAAS,EAAE,2BAA2B,CAAC,GAAG,CAAC,CAAC,MAAM;gBAClD,KAAK,EAAE,uBAAuB,CAAC,GAAG,CAAC,CAAC,MAAM;gBAC1C,IAAI,EAAE,qCAAqC,CAAC,GAAG,CAAC;gBAChD,GAAG,EAAE,sCAAsC,CAAC,GAAG,CAAC;aACjD,CAAC;YAEF,kBAAkB,CAAC,8BAA8B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC5E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,sBAAsB,EAAE,OAAO,CAAC,CAAC;QACnE,CAAC;KAAA;CACD"}
@@ -0,0 +1,12 @@
1
+ import { TKeyObject } from 'tscommons-es-core';
2
+ import { DatabaseService } from '../services/database.service';
3
+ import { IParserConfig } from '../interfaces/iparser-config';
4
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
5
+ import { StringParser } from './string.parser';
6
+ import { IDataConfig } from './data.parser';
7
+ export declare class AspErrorParser extends StringParser<IDataConfig> {
8
+ private url?;
9
+ constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
10
+ parse(database: DatabaseService): Promise<void>;
11
+ supports(_contentType: string, isAllow: boolean): boolean;
12
+ }
@@ -1,40 +1,48 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.AspErrorParser = void 0;
4
- const string_parser_1 = require("./string.parser");
5
- const ASP_ERROR = `Server Error in '[^']+' Application[.]`;
6
- const ASP_ERROR_PARSE = `Server Error in '([^']+)' Application[.].*<h2>\\s*<i>(.+?)</i>\\s*</h2>.+?<code><pre>(.+?)</pre></code>`;
7
- class AspErrorParser extends string_parser_1.StringParser {
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { StringParser } from './string.parser';
11
+ const ASP_ERROR = 'Server Error in \'[^\']+\' Application[.]';
12
+ const ASP_ERROR_PARSE = 'Server Error in \'([^\']+)\' Application[.].*<h2>\\s*<i>(.+?)</i>\\s*</h2>.+?<code><pre>(.+?)</pre></code>';
13
+ export class AspErrorParser extends StringParser {
8
14
  constructor(url, outcome, config) {
9
15
  super(outcome, config, 'aspError');
10
16
  this.url = url;
11
17
  }
12
- async parse(database) {
13
- if (!this.url || !this.stringData)
14
- return;
15
- const singlified = this.stringData.replace(/[\t\n\r]/g, ' ');
16
- const pattern = new RegExp(ASP_ERROR, 'g');
17
- const result = pattern.exec(singlified);
18
- if (result === null) {
19
- await database.unsetData(this.url, 'aspErrors');
20
- }
21
- else {
22
- const detail = new RegExp(ASP_ERROR_PARSE, 'g');
23
- const result2 = detail.exec(singlified);
24
- if (result2 === null) {
25
- await database.setData(this.url, 'aspErrors', true);
18
+ parse(database) {
19
+ return __awaiter(this, void 0, void 0, function* () {
20
+ if (!this.url || !this.stringData)
21
+ return;
22
+ const singlified = this.stringData.replace(/[\t\n\r]/g, ' ');
23
+ const pattern = new RegExp(ASP_ERROR, 'g');
24
+ const result = pattern.exec(singlified);
25
+ if (result === null) {
26
+ yield database.unsetData(this.url, 'aspErrors');
26
27
  }
27
28
  else {
28
- await database.setData(this.url, 'aspErrors', {
29
- application: result2[1],
30
- message: result2[2],
31
- stack: result2[3]
32
- });
29
+ const detail = new RegExp(ASP_ERROR_PARSE, 'g');
30
+ const result2 = detail.exec(singlified);
31
+ if (result2 === null) {
32
+ yield database.setData(this.url, 'aspErrors', true);
33
+ }
34
+ else {
35
+ yield database.setData(this.url, 'aspErrors', {
36
+ application: result2[1],
37
+ message: result2[2],
38
+ stack: result2[3]
39
+ });
40
+ }
33
41
  }
34
- }
42
+ });
35
43
  }
36
44
  supports(_contentType, isAllow) {
37
45
  return isAllow; // everything allowlisted
38
46
  }
39
47
  }
40
- exports.AspErrorParser = AspErrorParser;
48
+ //# sourceMappingURL=asp-error.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"asp-error.parser.js","sourceRoot":"","sources":["../../src/parsers/asp-error.parser.ts"],"names":[],"mappings":";;;;;;;;;AAOA,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,SAAS,GAAW,2CAA2C,CAAC;AACtE,MAAM,eAAe,GAAW,4GAA4G,CAAC;AAE7I,MAAM,OAAO,cAAe,SAAQ,YAAyB;IAC5D,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEY,KAAK,CAAC,QAAyB;;YAC3C,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU;gBAAE,OAAO;YAE1C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;YAErE,MAAM,OAAO,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAEnD,MAAM,MAAM,GAAyB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9D,IAAI,MAAM,KAAK,IAAI,EAAE;gBACpB,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;aAChD;iBAAM;gBACN,MAAM,MAAM,GAAW,IAAI,MAAM,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBACxD,MAAM,OAAO,GAAyB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE9D,IAAI,OAAO,KAAK,IAAI,EAAE;oBACrB,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC;iBACpD;qBAAM;oBACN,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,EAAE;wBAC5C,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;wBACvB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;wBACnB,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;qBAClB,CAAC,CAAC;iBACH;aACD;QACF,CAAC;KAAA;IAEM,QAAQ,CAAC,YAAoB,EAAE,OAAgB;QACrD,OAAO,OAAO,CAAC,CAAC,yBAAyB;IAC1C,CAAC;CACD"}
@@ -0,0 +1,10 @@
1
+ import { TKeyObject } from 'tscommons-es-core';
2
+ import { DatabaseService } from '../services/database.service';
3
+ import { IParserConfig } from '../interfaces/iparser-config';
4
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
5
+ import { DictionaryParser, IDictionaryConfig } from './dictionary.parser';
6
+ export declare class BadWordsParser extends DictionaryParser<IDictionaryConfig> {
7
+ private url?;
8
+ constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
9
+ protected parseMatches(database: DatabaseService, matches: string[], _nonMatches: string[]): Promise<void>;
10
+ }
@@ -1,19 +1,27 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.BadWordsParser = void 0;
4
- const dictionary_parser_1 = require("./dictionary.parser");
5
- class BadWordsParser extends dictionary_parser_1.DictionaryParser {
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { DictionaryParser } from './dictionary.parser';
11
+ export class BadWordsParser extends DictionaryParser {
6
12
  constructor(url, outcome, config) {
7
13
  super(outcome, config, 'badWords');
8
14
  this.url = url;
9
15
  }
10
- async parseMatches(database, matches, _nonMatches) {
11
- if (!this.url)
12
- return;
13
- if (matches.length > 0)
14
- await database.setData(this.url, 'badWords', matches);
15
- else
16
- await database.unsetData(this.url, 'badWords');
16
+ parseMatches(database, matches, _nonMatches) {
17
+ return __awaiter(this, void 0, void 0, function* () {
18
+ if (!this.url)
19
+ return;
20
+ if (matches.length > 0)
21
+ yield database.setData(this.url, 'badWords', matches);
22
+ else
23
+ yield database.unsetData(this.url, 'badWords');
24
+ });
17
25
  }
18
26
  }
19
- exports.BadWordsParser = BadWordsParser;
27
+ //# sourceMappingURL=bad-words.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bad-words.parser.js","sourceRoot":"","sources":["../../src/parsers/bad-words.parser.ts"],"names":[],"mappings":";;;;;;;;;AAOA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,qBAAqB,CAAC;AAE1E,MAAM,OAAO,cAAe,SAAQ,gBAAmC;IACtE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAJ1B,QAAG,GAAH,GAAG,CAAS;IAKtB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB,EAAE,WAAqB;;YAC/F,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;;gBACzE,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,CAAC;KAAA;CACD"}
@@ -0,0 +1,15 @@
1
+ import { TKeyObject } from 'tscommons-es-core';
2
+ import { DatabaseService } from '../services/database.service';
3
+ import { IParserConfig } from '../interfaces/iparser-config';
4
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
5
+ import { IDictionaryConfig } from './dictionary.parser';
6
+ import { PhraseParser } from './phrase.parser';
7
+ export declare class ComplexEnglishParser extends PhraseParser<IDictionaryConfig> {
8
+ private url?;
9
+ static loadKeyObject(config: IDictionaryConfig): TKeyObject<string[]>;
10
+ static loadDictionary(config: IDictionaryConfig): string[];
11
+ private dictionary;
12
+ constructor(url?: string | undefined, outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>);
13
+ getDictionary(): TKeyObject<string[]> | undefined;
14
+ protected parsePhrases(database: DatabaseService, matches: string[]): Promise<void>;
15
+ }
@@ -1,20 +1,26 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ComplexEnglishParser = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- const nodecommons_cli_1 = require("nodecommons-cli");
6
- const nodecommons_file_1 = require("nodecommons-file");
7
- const dictionary_parser_1 = require("./dictionary.parser");
8
- const phrase_parser_1 = require("./phrase.parser");
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { commonsTypeHasPropertyT, commonsTypeIsStringArrayKeyObject } from 'tscommons-es-core';
11
+ import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
12
+ import { commonsFileCwdRelativeOrAbsolutePath, commonsFileReadJsonFile } from 'nodecommons-es-file';
13
+ import { isIDictionaryConfig } from './dictionary.parser';
14
+ import { PhraseParser } from './phrase.parser';
9
15
  let CACHE;
10
- class ComplexEnglishParser extends phrase_parser_1.PhraseParser {
16
+ export class ComplexEnglishParser extends PhraseParser {
11
17
  constructor(url, outcome, config) {
12
18
  super(outcome, config, 'complexEnglish');
13
19
  this.url = url;
14
20
  if (!config)
15
21
  return;
16
- if (!tscommons_core_1.CommonsType.hasPropertyT(config, 'complexEnglish', dictionary_parser_1.isIDictionaryConfig)) {
17
- nodecommons_cli_1.CommonsOutput.error(`Invalid config for ComplexEnglishParser`);
22
+ if (!commonsTypeHasPropertyT(config, 'complexEnglish', isIDictionaryConfig)) {
23
+ commonsOutputError('Invalid config for ComplexEnglishParser');
18
24
  }
19
25
  const dictionaryConfig = config['complexEnglish'];
20
26
  if (!CACHE)
@@ -24,11 +30,11 @@ class ComplexEnglishParser extends phrase_parser_1.PhraseParser {
24
30
  this.setPhrases(keys, dictionaryConfig.caseSensitive || false);
25
31
  }
26
32
  static loadKeyObject(config) {
27
- const filename = `${__dirname}/../../config/${config.dictionary}`;
28
- const json = nodecommons_file_1.CommonsFile.readJsonFile(filename);
33
+ const filename = commonsFileCwdRelativeOrAbsolutePath(`config/${config.dictionary}`);
34
+ const json = commonsFileReadJsonFile(filename);
29
35
  if (json === undefined)
30
36
  throw new Error('Unable to read dictionary file');
31
- if (!tscommons_core_1.CommonsType.isStringArrayKeyObject(json))
37
+ if (!commonsTypeIsStringArrayKeyObject(json))
32
38
  throw new Error('Dictionary file is not a JSON array');
33
39
  return json;
34
40
  }
@@ -38,16 +44,18 @@ class ComplexEnglishParser extends phrase_parser_1.PhraseParser {
38
44
  getDictionary() {
39
45
  return this.dictionary;
40
46
  }
41
- async parsePhrases(database, matches) {
42
- if (!this.url)
43
- return;
44
- if (matches.length > 0) {
45
- nodecommons_cli_1.CommonsOutput.debug(`Found ${matches.length} potentially complex phrases`);
46
- await database.setData(this.url, 'complexEnglish', matches);
47
- }
48
- else {
49
- await database.unsetData(this.url, 'complexEnglish');
50
- }
47
+ parsePhrases(database, matches) {
48
+ return __awaiter(this, void 0, void 0, function* () {
49
+ if (!this.url)
50
+ return;
51
+ if (matches.length > 0) {
52
+ commonsOutputDebug(`Found ${matches.length} potentially complex phrases`);
53
+ yield database.setData(this.url, 'complexEnglish', matches);
54
+ }
55
+ else {
56
+ yield database.unsetData(this.url, 'complexEnglish');
57
+ }
58
+ });
51
59
  }
52
60
  }
53
- exports.ComplexEnglishParser = ComplexEnglishParser;
61
+ //# sourceMappingURL=complex-english.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"complex-english.parser.js","sourceRoot":"","sources":["../../src/parsers/complex-english.parser.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAE,iCAAiC,EAAc,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAOpG,OAAO,EAAqB,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,IAAI,KAAqC,CAAC;AAE1C,MAAM,OAAO,oBAAqB,SAAQ,YAA+B;IAkBxE,YACU,GAAY,EACpB,OAAyB,EACzB,MAAkC;QAEnC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAJhC,QAAG,GAAH,GAAG,CAAS;QAMrB,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,CAAC,uBAAuB,CAAoB,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,EAAE;YAC/F,kBAAkB,CAAC,yCAAyC,CAAC,CAAC;SAC9D;QACD,MAAM,gBAAgB,GAAsB,MAAM,CAAC,gBAAgB,CAAsB,CAAC;QAE1F,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,oBAAoB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,MAAM,IAAI,GAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,gBAAgB,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC;IAChE,CAAC;IApCM,MAAM,CAAC,aAAa,CAAC,MAAyB;QACpD,MAAM,QAAQ,GAAW,oCAAoC,CAAC,UAAU,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE7F,MAAM,IAAI,GAAsB,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QAClE,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAE1E,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAErG,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,MAAyB;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;IAyBM,aAAa;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAEe,YAAY,CAAC,QAAyB,EAAE,OAAiB;;YACxE,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEtB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,kBAAkB,CAAC,SAAS,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAC;gBAC1E,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;aAC5D;iBAAM;gBACN,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;aACrD;QACF,CAAC;KAAA;CACD"}
@@ -0,0 +1,14 @@
1
+ /// <reference types="node" />
2
+ import { TKeyObject } from 'tscommons-es-core';
3
+ import { IRequestOutcome } from '../interfaces/irequest-outcome';
4
+ import { IParserConfig } from '../interfaces/iparser-config';
5
+ import { Parser } from './parser';
6
+ export interface IDataConfig extends IParserConfig {
7
+ maxDataBytes?: number;
8
+ allowExceeded?: boolean;
9
+ }
10
+ export declare function isIDataConfig(test: unknown): test is IDataConfig;
11
+ export declare abstract class DataParser<T extends IDataConfig> extends Parser<T> {
12
+ protected data: Buffer | undefined;
13
+ constructor(outcome?: IRequestOutcome, config?: TKeyObject<IParserConfig>, configKey?: string);
14
+ }
@@ -1,22 +1,18 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DataParser = exports.isIDataConfig = void 0;
4
- const tscommons_core_1 = require("tscommons-core");
5
- const nodecommons_cli_1 = require("nodecommons-cli");
6
- const iparser_config_1 = require("../interfaces/iparser-config");
7
- const parser_1 = require("./parser");
8
- function isIDataConfig(test) {
9
- if (!iparser_config_1.isIParserConfig(test))
1
+ import { commonsTypeHasPropertyBooleanOrUndefined, commonsTypeHasPropertyNumberOrUndefined } from 'tscommons-es-core';
2
+ import { commonsOutputError } from 'nodecommons-es-cli';
3
+ import { isIParserConfig } from '../interfaces/iparser-config';
4
+ import { Parser } from './parser';
5
+ export function isIDataConfig(test) {
6
+ if (!isIParserConfig(test))
10
7
  return false;
11
- if (!tscommons_core_1.CommonsType.hasPropertyNumberOrUndefined(test, 'maxDataBytes'))
8
+ if (!commonsTypeHasPropertyNumberOrUndefined(test, 'maxDataBytes'))
12
9
  return false;
13
- if (!tscommons_core_1.CommonsType.hasPropertyBooleanOrUndefined(test, 'allowExceeded'))
10
+ if (!commonsTypeHasPropertyBooleanOrUndefined(test, 'allowExceeded'))
14
11
  return false;
15
12
  return true;
16
13
  }
17
- exports.isIDataConfig = isIDataConfig;
18
14
  const MAX_DATA_BYTES_HARD_LIMIT = 1024 * 1024 * 1024; // 1Gb
19
- class DataParser extends parser_1.Parser {
15
+ export class DataParser extends Parser {
20
16
  constructor(outcome, config, configKey) {
21
17
  super(outcome, config, configKey);
22
18
  if (!outcome || !outcome.data)
@@ -24,17 +20,17 @@ class DataParser extends parser_1.Parser {
24
20
  const dataConfig = this.getConfig(isIDataConfig);
25
21
  const maxByteSize = (dataConfig && dataConfig.maxDataBytes) ? dataConfig.maxDataBytes : MAX_DATA_BYTES_HARD_LIMIT;
26
22
  if (outcome.data.byteLength >= maxByteSize) {
27
- nodecommons_cli_1.CommonsOutput.error(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
23
+ commonsOutputError(`Data to be parsed exceeded parser limit of ${maxByteSize}. Skipping parse.`);
28
24
  this.data = undefined;
29
25
  return;
30
26
  }
31
27
  if ((!dataConfig || !dataConfig.allowExceeded) && outcome.exceeded) {
32
28
  // by default don't parse data which has exceeded Hydra's upper download limit, so will be corrupt. But there is a parser option to force it to do so if desired for some reason
33
- nodecommons_cli_1.CommonsOutput.error(`Data to be parsed exceeded Hydra's upper download limit and allowExceed it set. Skipping parse.`);
29
+ commonsOutputError('Data to be parsed exceeded Hydra\'s upper download limit and allowExceed it set. Skipping parse.');
34
30
  this.data = undefined;
35
31
  return;
36
32
  }
37
33
  this.data = outcome.data;
38
34
  }
39
35
  }
40
- exports.DataParser = DataParser;
36
+ //# sourceMappingURL=data.parser.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"data.parser.js","sourceRoot":"","sources":["../../src/parsers/data.parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAE,uCAAuC,EAAc,MAAM,mBAAmB,CAAC;AAElI,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGxD,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAE9E,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAOlC,MAAM,UAAU,aAAa,CAAC,IAAa;IAC1C,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzC,IAAI,CAAC,uCAAuC,CAAC,IAAI,EAAE,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IACjF,IAAI,CAAC,wCAAwC,CAAC,IAAI,EAAE,eAAe,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnF,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,yBAAyB,GAAW,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAEpE,MAAM,OAAgB,UAAkC,SAAQ,MAAS;IAGxE,YACE,OAAyB,EACzB,MAAkC,EAClC,SAAkB;QAEnB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAElC,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAEtC,MAAM,UAAU,GAA0B,IAAI,CAAC,SAAS,CAAc,aAAa,CAAC,CAAC;QACrF,MAAM,WAAW,GAAW,CAAC,UAAU,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,yBAAyB,CAAC;QAC1H,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,IAAI,WAAW,EAAE;YAC3C,kBAAkB,CAAC,8CAA8C,WAAW,mBAAmB,CAAC,CAAC;YACjG,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;SACP;QAED,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE;YACnE,gLAAgL;YAEhL,kBAAkB,CAAC,kGAAkG,CAAC,CAAC;YACvH,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;SACP;QAED,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,CAAC;CACD"}