hydra-crawler 2.8.4 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (468) hide show
  1. package/dist/apis/{autocomplete.api.d.ts → autocomplete.api.d.mts} +4 -4
  2. package/dist/apis/autocomplete.api.mjs +14 -0
  3. package/dist/apis/autocomplete.api.mjs.map +1 -0
  4. package/dist/apis/{bugs.api.d.ts → bugs.api.d.mts} +4 -4
  5. package/dist/apis/bugs.api.mjs +18 -0
  6. package/dist/apis/bugs.api.mjs.map +1 -0
  7. package/dist/apis/{crawl.api.d.ts → crawl.api.d.mts} +4 -4
  8. package/dist/apis/crawl.api.mjs +17 -0
  9. package/dist/apis/crawl.api.mjs.map +1 -0
  10. package/dist/apis/{domains.api.d.ts → domains.api.d.mts} +4 -4
  11. package/dist/apis/{domains.api.js → domains.api.mjs} +10 -18
  12. package/dist/apis/domains.api.mjs.map +1 -0
  13. package/dist/apis/{images.api.d.ts → images.api.d.mts} +4 -4
  14. package/dist/apis/images.api.mjs +19 -0
  15. package/dist/apis/images.api.mjs.map +1 -0
  16. package/dist/apis/{statistics.api.d.ts → statistics.api.d.mts} +4 -4
  17. package/dist/apis/statistics.api.mjs +33 -0
  18. package/dist/apis/statistics.api.mjs.map +1 -0
  19. package/dist/apis/{test.api.d.ts → test.api.d.mts} +3 -3
  20. package/dist/apis/test.api.mjs +10 -0
  21. package/dist/apis/test.api.mjs.map +1 -0
  22. package/dist/apis/{urls.api.d.ts → urls.api.d.mts} +4 -4
  23. package/dist/apis/urls.api.mjs +21 -0
  24. package/dist/apis/urls.api.mjs.map +1 -0
  25. package/dist/apps/{cleanup.app.d.ts → cleanup.app.d.mts} +5 -5
  26. package/dist/apps/cleanup.app.mjs +129 -0
  27. package/dist/apps/cleanup.app.mjs.map +1 -0
  28. package/dist/apps/{cross-populate-export.app.d.ts → cross-populate-export.app.d.mts} +3 -3
  29. package/dist/apps/cross-populate-export.app.mjs +61 -0
  30. package/dist/apps/cross-populate-export.app.mjs.map +1 -0
  31. package/dist/apps/{cross-populate-import.app.d.ts → cross-populate-import.app.d.mts} +3 -3
  32. package/dist/apps/cross-populate-import.app.mjs +61 -0
  33. package/dist/apps/cross-populate-import.app.mjs.map +1 -0
  34. package/dist/apps/{denylist.app.d.ts → denylist.app.d.mts} +5 -5
  35. package/dist/apps/denylist.app.mjs +122 -0
  36. package/dist/apps/denylist.app.mjs.map +1 -0
  37. package/dist/apps/{expire.app.d.ts → expire.app.d.mts} +8 -7
  38. package/dist/apps/expire.app.mjs +51 -0
  39. package/dist/apps/expire.app.mjs.map +1 -0
  40. package/dist/apps/{export-domain-urls.d.ts → export-domain-urls.d.mts} +3 -3
  41. package/dist/apps/export-domain-urls.mjs +85 -0
  42. package/dist/apps/export-domain-urls.mjs.map +1 -0
  43. package/dist/apps/{extract-text.app.d.ts → extract-text.app.d.mts} +2 -2
  44. package/dist/apps/extract-text.app.mjs +43 -0
  45. package/dist/apps/extract-text.app.mjs.map +1 -0
  46. package/dist/apps/{hydra.app.d.ts → hydra.app.d.mts} +10 -10
  47. package/dist/apps/hydra.app.mjs +222 -0
  48. package/dist/apps/hydra.app.mjs.map +1 -0
  49. package/dist/apps/{import.app.d.ts → import.app.d.mts} +3 -3
  50. package/dist/apps/import.app.mjs +48 -0
  51. package/dist/apps/import.app.mjs.map +1 -0
  52. package/dist/apps/{internal-hydra-common.app.d.ts → internal-hydra-common.app.d.mts} +6 -6
  53. package/dist/apps/{internal-hydra-common.app.js → internal-hydra-common.app.mjs} +1 -1
  54. package/dist/apps/internal-hydra-common.app.mjs.map +1 -0
  55. package/dist/apps/{move-to-archive.app.d.ts → move-to-archive.app.d.mts} +2 -2
  56. package/dist/apps/move-to-archive.app.mjs +31 -0
  57. package/dist/apps/move-to-archive.app.mjs.map +1 -0
  58. package/dist/apps/{prune-archive.app.d.ts → prune-archive.app.d.mts} +2 -2
  59. package/dist/apps/prune-archive.app.mjs +40 -0
  60. package/dist/apps/prune-archive.app.mjs.map +1 -0
  61. package/dist/apps/{query.app.d.ts → query.app.d.mts} +6 -6
  62. package/dist/apps/query.app.mjs +69 -0
  63. package/dist/apps/query.app.mjs.map +1 -0
  64. package/dist/apps/{reattempt.app.d.ts → reattempt.app.d.mts} +6 -6
  65. package/dist/apps/reattempt.app.mjs +74 -0
  66. package/dist/apps/reattempt.app.mjs.map +1 -0
  67. package/dist/apps/{requeue-domain.app.d.ts → requeue-domain.app.d.mts} +3 -3
  68. package/dist/apps/requeue-domain.app.mjs +52 -0
  69. package/dist/apps/requeue-domain.app.mjs.map +1 -0
  70. package/dist/apps/{seed.app.d.ts → seed.app.d.mts} +5 -5
  71. package/dist/apps/seed.app.mjs +59 -0
  72. package/dist/apps/seed.app.mjs.map +1 -0
  73. package/dist/apps/{startup.app.d.ts → startup.app.d.mts} +3 -3
  74. package/dist/apps/startup.app.mjs +55 -0
  75. package/dist/apps/startup.app.mjs.map +1 -0
  76. package/dist/apps/{unarchive-urls.app.d.ts → unarchive-urls.app.d.mts} +5 -5
  77. package/dist/apps/unarchive-urls.app.mjs +71 -0
  78. package/dist/apps/unarchive-urls.app.mjs.map +1 -0
  79. package/dist/classes/{cleaner.d.ts → cleaner.d.mts} +2 -2
  80. package/dist/classes/cleaner.mjs +257 -0
  81. package/dist/classes/cleaner.mjs.map +1 -0
  82. package/dist/classes/{crawler.d.ts → crawler.d.mts} +10 -12
  83. package/dist/classes/crawler.mjs +418 -0
  84. package/dist/classes/crawler.mjs.map +1 -0
  85. package/dist/classes/{dns.js → dns.mjs} +4 -4
  86. package/dist/classes/dns.mjs.map +1 -0
  87. package/dist/classes/{expirer.d.ts → expirer.d.mts} +2 -2
  88. package/dist/classes/expirer.mjs +117 -0
  89. package/dist/classes/expirer.mjs.map +1 -0
  90. package/dist/classes/{expiry.d.ts → expiry.d.mts} +1 -1
  91. package/dist/classes/{expiry.js → expiry.mjs} +7 -9
  92. package/dist/classes/expiry.mjs.map +1 -0
  93. package/dist/classes/{lists.d.ts → lists.d.mts} +2 -2
  94. package/dist/classes/{lists.js → lists.mjs} +4 -4
  95. package/dist/classes/lists.mjs.map +1 -0
  96. package/dist/classes/{robot.d.ts → robot.d.mts} +3 -3
  97. package/dist/classes/robot.mjs +74 -0
  98. package/dist/classes/robot.mjs.map +1 -0
  99. package/dist/classes/{tracker.d.ts → tracker.d.mts} +3 -3
  100. package/dist/classes/tracker.mjs +101 -0
  101. package/dist/classes/tracker.mjs.map +1 -0
  102. package/dist/{cli.js → cli.mjs} +46 -58
  103. package/dist/cli.mjs.map +1 -0
  104. package/dist/enums/{eavailable-strategy.js → eavailable-strategy.mjs} +1 -1
  105. package/dist/enums/eavailable-strategy.mjs.map +1 -0
  106. package/dist/enums/{elist.js → elist.mjs} +1 -1
  107. package/dist/enums/elist.mjs.map +1 -0
  108. package/dist/enums/{eserver.js → eserver.mjs} +1 -1
  109. package/dist/enums/eserver.mjs.map +1 -0
  110. package/dist/enums/{ex-powered-by.js → ex-powered-by.mjs} +1 -1
  111. package/dist/enums/ex-powered-by.mjs.map +1 -0
  112. package/dist/helpers/{matcher.d.ts → matcher.d.mts} +1 -1
  113. package/dist/helpers/{matcher.js → matcher.mjs} +1 -1
  114. package/dist/helpers/matcher.mjs.map +1 -0
  115. package/dist/helpers/{random.d.ts → random.d.mts} +1 -1
  116. package/dist/helpers/{random.js → random.mjs} +1 -1
  117. package/dist/helpers/random.mjs.map +1 -0
  118. package/dist/helpers/{utf-decoder.d.ts → utf-decoder.d.mts} +0 -1
  119. package/dist/helpers/{utf-decoder.js → utf-decoder.mjs} +3 -3
  120. package/dist/helpers/utf-decoder.mjs.map +1 -0
  121. package/dist/interfaces/{iexpiry.d.ts → iexpiry.d.mts} +1 -1
  122. package/dist/interfaces/{iexpiry.js → iexpiry.mjs} +3 -3
  123. package/dist/interfaces/iexpiry.mjs.map +1 -0
  124. package/dist/interfaces/{imatch.js → imatch.mjs} +2 -2
  125. package/dist/interfaces/imatch.mjs.map +1 -0
  126. package/dist/interfaces/{iparser-config.js → iparser-config.mjs} +2 -2
  127. package/dist/interfaces/iparser-config.mjs.map +1 -0
  128. package/dist/interfaces/{iparser.d.ts → iparser.d.mts} +1 -1
  129. package/dist/interfaces/iparser.mjs +2 -0
  130. package/dist/interfaces/iparser.mjs.map +1 -0
  131. package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts} +0 -2
  132. package/dist/interfaces/irequest-outcome.mjs +2 -0
  133. package/dist/interfaces/irequest-outcome.mjs.map +1 -0
  134. package/dist/interfaces/iserver.mjs +2 -0
  135. package/dist/interfaces/iserver.mjs.map +1 -0
  136. package/dist/parsers/accessibility-metrics.parser.d.mts +11 -0
  137. package/dist/parsers/accessibility-metrics.parser.mjs +30 -0
  138. package/dist/parsers/accessibility-metrics.parser.mjs.map +1 -0
  139. package/dist/parsers/asp-error.parser.d.mts +12 -0
  140. package/dist/parsers/asp-error.parser.mjs +38 -0
  141. package/dist/parsers/asp-error.parser.mjs.map +1 -0
  142. package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts} +6 -6
  143. package/dist/parsers/bad-words.parser.mjs +17 -0
  144. package/dist/parsers/bad-words.parser.mjs.map +1 -0
  145. package/dist/parsers/complex-english.parser.d.mts +15 -0
  146. package/dist/parsers/complex-english.parser.mjs +52 -0
  147. package/dist/parsers/complex-english.parser.mjs.map +1 -0
  148. package/dist/parsers/data.parser.d.mts +13 -0
  149. package/dist/parsers/{data.parser.js → data.parser.mjs} +8 -7
  150. package/dist/parsers/data.parser.mjs.map +1 -0
  151. package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts} +6 -6
  152. package/dist/parsers/dictionary.parser.mjs +63 -0
  153. package/dist/parsers/dictionary.parser.mjs.map +1 -0
  154. package/dist/parsers/html.parser.d.mts +13 -0
  155. package/dist/parsers/{html.parser.js → html.parser.mjs} +4 -3
  156. package/dist/parsers/html.parser.mjs.map +1 -0
  157. package/dist/parsers/hyperlinks.parser.d.mts +20 -0
  158. package/dist/parsers/hyperlinks.parser.mjs +104 -0
  159. package/dist/parsers/hyperlinks.parser.mjs.map +1 -0
  160. package/dist/parsers/image-tags.parser.d.mts +19 -0
  161. package/dist/parsers/image-tags.parser.mjs +42 -0
  162. package/dist/parsers/image-tags.parser.mjs.map +1 -0
  163. package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts} +7 -7
  164. package/dist/parsers/interest.parser.mjs +60 -0
  165. package/dist/parsers/interest.parser.mjs.map +1 -0
  166. package/dist/parsers/jpeg.parser.d.mts +11 -0
  167. package/dist/parsers/jpeg.parser.mjs +29 -0
  168. package/dist/parsers/jpeg.parser.mjs.map +1 -0
  169. package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts} +7 -7
  170. package/dist/parsers/llama-guard.parser.mjs +56 -0
  171. package/dist/parsers/llama-guard.parser.mjs.map +1 -0
  172. package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts} +7 -7
  173. package/dist/parsers/offence.parser.mjs +60 -0
  174. package/dist/parsers/offence.parser.mjs.map +1 -0
  175. package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts} +6 -6
  176. package/dist/parsers/ollama.parser.mjs +43 -0
  177. package/dist/parsers/ollama.parser.mjs.map +1 -0
  178. package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts} +5 -5
  179. package/dist/parsers/paragraphs.parser.mjs +38 -0
  180. package/dist/parsers/paragraphs.parser.mjs.map +1 -0
  181. package/dist/parsers/{parser.d.ts → parser.d.mts} +6 -6
  182. package/dist/parsers/parser.mjs +45 -0
  183. package/dist/parsers/parser.mjs.map +1 -0
  184. package/dist/parsers/php-error.parser.d.mts +12 -0
  185. package/dist/parsers/php-error.parser.mjs +42 -0
  186. package/dist/parsers/php-error.parser.mjs.map +1 -0
  187. package/dist/parsers/{phrase.parser.d.ts → phrase.parser.d.mts} +3 -3
  188. package/dist/parsers/phrase.parser.mjs +15 -0
  189. package/dist/parsers/phrase.parser.mjs.map +1 -0
  190. package/dist/parsers/{regex.parser.d.ts → regex.parser.d.mts} +3 -3
  191. package/dist/parsers/regex.parser.mjs +29 -0
  192. package/dist/parsers/regex.parser.mjs.map +1 -0
  193. package/dist/parsers/server.parser.d.mts +11 -0
  194. package/dist/parsers/server.parser.mjs +57 -0
  195. package/dist/parsers/server.parser.mjs.map +1 -0
  196. package/dist/parsers/{spelling.parser.d.ts → spelling.parser.d.mts} +6 -6
  197. package/dist/parsers/spelling.parser.mjs +17 -0
  198. package/dist/parsers/spelling.parser.mjs.map +1 -0
  199. package/dist/parsers/string.parser.d.mts +8 -0
  200. package/dist/parsers/{string.parser.js → string.parser.mjs} +5 -4
  201. package/dist/parsers/string.parser.mjs.map +1 -0
  202. package/dist/parsers/{text.parser.d.ts → text.parser.d.mts} +3 -3
  203. package/dist/parsers/text.parser.mjs +30 -0
  204. package/dist/parsers/text.parser.mjs.map +1 -0
  205. package/dist/parsers/{words.parser.d.ts → words.parser.d.mts} +3 -3
  206. package/dist/parsers/words.parser.mjs +29 -0
  207. package/dist/parsers/words.parser.mjs.map +1 -0
  208. package/dist/queries/complex-english.query.d.mts +2 -0
  209. package/dist/queries/{complex-english.query.js → complex-english.query.mjs} +18 -27
  210. package/dist/queries/complex-english.query.mjs.map +1 -0
  211. package/dist/queries/flash-content.query.d.mts +2 -0
  212. package/dist/queries/flash-content.query.mjs +80 -0
  213. package/dist/queries/flash-content.query.mjs.map +1 -0
  214. package/dist/queries/linking-to-domains.query.d.mts +2 -0
  215. package/dist/queries/linking-to-domains.query.mjs +128 -0
  216. package/dist/queries/linking-to-domains.query.mjs.map +1 -0
  217. package/dist/queries/llamaguard-unsafe-content.query.d.mts +2 -0
  218. package/dist/queries/llamaguard-unsafe-content.query.mjs +90 -0
  219. package/dist/queries/llamaguard-unsafe-content.query.mjs.map +1 -0
  220. package/dist/queries/readability-score.query.d.mts +2 -0
  221. package/dist/queries/{readability-score.query.js → readability-score.query.mjs} +11 -20
  222. package/dist/queries/readability-score.query.mjs.map +1 -0
  223. package/dist/servers/{crawl.server.d.ts → crawl.server.d.mts} +10 -10
  224. package/dist/servers/crawl.server.mjs +192 -0
  225. package/dist/servers/crawl.server.mjs.map +1 -0
  226. package/dist/servers/{express.server.d.ts → express.server.d.mts} +2 -3
  227. package/dist/servers/express.server.mjs +13 -0
  228. package/dist/servers/express.server.mjs.map +1 -0
  229. package/dist/servers/{maintenance.server.d.ts → maintenance.server.d.mts} +5 -5
  230. package/dist/servers/maintenance.server.mjs +97 -0
  231. package/dist/servers/maintenance.server.mjs.map +1 -0
  232. package/dist/servers/{rest.server.d.ts → rest.server.d.mts} +4 -4
  233. package/dist/servers/rest.server.mjs +62 -0
  234. package/dist/servers/rest.server.mjs.map +1 -0
  235. package/dist/servers/{socket-io.server.d.ts → socket-io.server.d.mts} +4 -4
  236. package/dist/servers/socket-io.server.mjs +22 -0
  237. package/dist/servers/socket-io.server.mjs.map +1 -0
  238. package/dist/services/{database.service.d.ts → database.service.d.mts} +12 -12
  239. package/dist/services/database.service.mjs +645 -0
  240. package/dist/services/database.service.mjs.map +1 -0
  241. package/dist/services/{ollama-rest.service.d.ts → ollama-rest.service.d.mts} +2 -2
  242. package/dist/services/ollama-rest.service.mjs +27 -0
  243. package/dist/services/ollama-rest.service.mjs.map +1 -0
  244. package/dist/services/{rig-llama-guard.service.d.ts → rig-llama-guard.service.d.mts} +2 -2
  245. package/dist/services/{rig-llama-guard.service.js → rig-llama-guard.service.mjs} +33 -43
  246. package/dist/services/rig-llama-guard.service.mjs.map +1 -0
  247. package/dist/services/{rig-queue-length.service.d.ts → rig-queue-length.service.d.mts} +1 -1
  248. package/dist/services/rig-queue-length.service.mjs +12 -0
  249. package/dist/services/rig-queue-length.service.mjs.map +1 -0
  250. package/dist/services/{rig-subjectivity-scale.service.d.ts → rig-subjectivity-scale.service.d.mts} +2 -2
  251. package/dist/services/rig-subjectivity-scale.service.mjs +82 -0
  252. package/dist/services/rig-subjectivity-scale.service.mjs.map +1 -0
  253. package/dist/services/{rig-ticket-rest.service.d.ts → rig-ticket-rest.service.d.mts} +3 -3
  254. package/dist/services/rig-ticket-rest.service.mjs +15 -0
  255. package/dist/services/rig-ticket-rest.service.mjs.map +1 -0
  256. package/dist/services/{rig-ticketed-promise.service.d.ts → rig-ticketed-promise.service.d.mts} +4 -6
  257. package/dist/services/{rig-ticketed-promise.service.js → rig-ticketed-promise.service.mjs} +7 -6
  258. package/dist/services/rig-ticketed-promise.service.mjs.map +1 -0
  259. package/dist/types/{tcrawl-config.js → tcrawl-config.mjs} +2 -2
  260. package/dist/types/tcrawl-config.mjs.map +1 -0
  261. package/dist/types/{thydra-config.js → thydra-config.mjs} +2 -2
  262. package/dist/types/thydra-config.mjs.map +1 -0
  263. package/dist/types/tparser-ctor.d.mts +7 -0
  264. package/dist/types/tparser-ctor.mjs +2 -0
  265. package/dist/types/tparser-ctor.mjs.map +1 -0
  266. package/dist/types/tquery.d.mts +7 -0
  267. package/dist/types/tquery.mjs +2 -0
  268. package/dist/types/tquery.mjs.map +1 -0
  269. package/dist/types/tqueue-length.mjs +2 -0
  270. package/dist/types/tqueue-length.mjs.map +1 -0
  271. package/dist/types/{trobots-config.js → trobots-config.mjs} +2 -2
  272. package/dist/types/trobots-config.mjs.map +1 -0
  273. package/package.json +37 -43
  274. package/dist/apis/autocomplete.api.js +0 -22
  275. package/dist/apis/autocomplete.api.js.map +0 -1
  276. package/dist/apis/bugs.api.js +0 -26
  277. package/dist/apis/bugs.api.js.map +0 -1
  278. package/dist/apis/crawl.api.js +0 -25
  279. package/dist/apis/crawl.api.js.map +0 -1
  280. package/dist/apis/domains.api.js.map +0 -1
  281. package/dist/apis/images.api.js +0 -27
  282. package/dist/apis/images.api.js.map +0 -1
  283. package/dist/apis/statistics.api.js +0 -41
  284. package/dist/apis/statistics.api.js.map +0 -1
  285. package/dist/apis/test.api.js +0 -19
  286. package/dist/apis/test.api.js.map +0 -1
  287. package/dist/apis/urls.api.js +0 -29
  288. package/dist/apis/urls.api.js.map +0 -1
  289. package/dist/apps/cleanup.app.js +0 -151
  290. package/dist/apps/cleanup.app.js.map +0 -1
  291. package/dist/apps/cross-populate-export.app.js +0 -75
  292. package/dist/apps/cross-populate-export.app.js.map +0 -1
  293. package/dist/apps/cross-populate-import.app.js +0 -100
  294. package/dist/apps/cross-populate-import.app.js.map +0 -1
  295. package/dist/apps/denylist.app.js +0 -132
  296. package/dist/apps/denylist.app.js.map +0 -1
  297. package/dist/apps/expire.app.js +0 -63
  298. package/dist/apps/expire.app.js.map +0 -1
  299. package/dist/apps/export-domain-urls.js +0 -99
  300. package/dist/apps/export-domain-urls.js.map +0 -1
  301. package/dist/apps/extract-text.app.js +0 -55
  302. package/dist/apps/extract-text.app.js.map +0 -1
  303. package/dist/apps/hydra.app.js +0 -218
  304. package/dist/apps/hydra.app.js.map +0 -1
  305. package/dist/apps/import.app.js +0 -57
  306. package/dist/apps/import.app.js.map +0 -1
  307. package/dist/apps/internal-hydra-common.app.js.map +0 -1
  308. package/dist/apps/move-to-archive.app.js +0 -46
  309. package/dist/apps/move-to-archive.app.js.map +0 -1
  310. package/dist/apps/prune-archive.app.js +0 -55
  311. package/dist/apps/prune-archive.app.js.map +0 -1
  312. package/dist/apps/query.app.js +0 -80
  313. package/dist/apps/query.app.js.map +0 -1
  314. package/dist/apps/reattempt.app.js +0 -83
  315. package/dist/apps/reattempt.app.js.map +0 -1
  316. package/dist/apps/requeue-domain.app.js +0 -64
  317. package/dist/apps/requeue-domain.app.js.map +0 -1
  318. package/dist/apps/seed.app.js +0 -69
  319. package/dist/apps/seed.app.js.map +0 -1
  320. package/dist/apps/startup.app.js +0 -64
  321. package/dist/apps/startup.app.js.map +0 -1
  322. package/dist/apps/unarchive-urls.app.js +0 -83
  323. package/dist/apps/unarchive-urls.app.js.map +0 -1
  324. package/dist/classes/cleaner.js +0 -266
  325. package/dist/classes/cleaner.js.map +0 -1
  326. package/dist/classes/crawler.js +0 -422
  327. package/dist/classes/crawler.js.map +0 -1
  328. package/dist/classes/dns.js.map +0 -1
  329. package/dist/classes/expirer.js +0 -121
  330. package/dist/classes/expirer.js.map +0 -1
  331. package/dist/classes/expiry.js.map +0 -1
  332. package/dist/classes/lists.js.map +0 -1
  333. package/dist/classes/robot.js +0 -82
  334. package/dist/classes/robot.js.map +0 -1
  335. package/dist/classes/tracker.js +0 -120
  336. package/dist/classes/tracker.js.map +0 -1
  337. package/dist/cli.js.map +0 -1
  338. package/dist/enums/eavailable-strategy.js.map +0 -1
  339. package/dist/enums/elist.js.map +0 -1
  340. package/dist/enums/eserver.js.map +0 -1
  341. package/dist/enums/ex-powered-by.js.map +0 -1
  342. package/dist/helpers/matcher.js.map +0 -1
  343. package/dist/helpers/random.js.map +0 -1
  344. package/dist/helpers/utf-decoder.js.map +0 -1
  345. package/dist/interfaces/iexpiry.js.map +0 -1
  346. package/dist/interfaces/imatch.js.map +0 -1
  347. package/dist/interfaces/iparser-config.js.map +0 -1
  348. package/dist/interfaces/iparser.js +0 -2
  349. package/dist/interfaces/iparser.js.map +0 -1
  350. package/dist/interfaces/irequest-outcome.js +0 -2
  351. package/dist/interfaces/irequest-outcome.js.map +0 -1
  352. package/dist/interfaces/iserver.js +0 -2
  353. package/dist/interfaces/iserver.js.map +0 -1
  354. package/dist/parsers/accessibility-metrics.parser.d.ts +0 -11
  355. package/dist/parsers/accessibility-metrics.parser.js +0 -40
  356. package/dist/parsers/accessibility-metrics.parser.js.map +0 -1
  357. package/dist/parsers/asp-error.parser.d.ts +0 -12
  358. package/dist/parsers/asp-error.parser.js +0 -48
  359. package/dist/parsers/asp-error.parser.js.map +0 -1
  360. package/dist/parsers/bad-words.parser.js +0 -27
  361. package/dist/parsers/bad-words.parser.js.map +0 -1
  362. package/dist/parsers/complex-english.parser.d.ts +0 -15
  363. package/dist/parsers/complex-english.parser.js +0 -61
  364. package/dist/parsers/complex-english.parser.js.map +0 -1
  365. package/dist/parsers/data.parser.d.ts +0 -14
  366. package/dist/parsers/data.parser.js.map +0 -1
  367. package/dist/parsers/dictionary.parser.js +0 -73
  368. package/dist/parsers/dictionary.parser.js.map +0 -1
  369. package/dist/parsers/html.parser.d.ts +0 -13
  370. package/dist/parsers/html.parser.js.map +0 -1
  371. package/dist/parsers/hyperlinks.parser.d.ts +0 -20
  372. package/dist/parsers/hyperlinks.parser.js +0 -115
  373. package/dist/parsers/hyperlinks.parser.js.map +0 -1
  374. package/dist/parsers/image-tags.parser.d.ts +0 -19
  375. package/dist/parsers/image-tags.parser.js +0 -52
  376. package/dist/parsers/image-tags.parser.js.map +0 -1
  377. package/dist/parsers/interest.parser.js +0 -69
  378. package/dist/parsers/interest.parser.js.map +0 -1
  379. package/dist/parsers/jpeg.parser.d.ts +0 -11
  380. package/dist/parsers/jpeg.parser.js +0 -39
  381. package/dist/parsers/jpeg.parser.js.map +0 -1
  382. package/dist/parsers/llama-guard.parser.js +0 -65
  383. package/dist/parsers/llama-guard.parser.js.map +0 -1
  384. package/dist/parsers/offence.parser.js +0 -69
  385. package/dist/parsers/offence.parser.js.map +0 -1
  386. package/dist/parsers/ollama.parser.js +0 -51
  387. package/dist/parsers/ollama.parser.js.map +0 -1
  388. package/dist/parsers/paragraphs.parser.js +0 -49
  389. package/dist/parsers/paragraphs.parser.js.map +0 -1
  390. package/dist/parsers/parser.js +0 -57
  391. package/dist/parsers/parser.js.map +0 -1
  392. package/dist/parsers/php-error.parser.d.ts +0 -12
  393. package/dist/parsers/php-error.parser.js +0 -52
  394. package/dist/parsers/php-error.parser.js.map +0 -1
  395. package/dist/parsers/phrase.parser.js +0 -26
  396. package/dist/parsers/phrase.parser.js.map +0 -1
  397. package/dist/parsers/regex.parser.js +0 -43
  398. package/dist/parsers/regex.parser.js.map +0 -1
  399. package/dist/parsers/server.parser.d.ts +0 -11
  400. package/dist/parsers/server.parser.js +0 -67
  401. package/dist/parsers/server.parser.js.map +0 -1
  402. package/dist/parsers/spelling.parser.js +0 -27
  403. package/dist/parsers/spelling.parser.js.map +0 -1
  404. package/dist/parsers/string.parser.d.ts +0 -8
  405. package/dist/parsers/string.parser.js.map +0 -1
  406. package/dist/parsers/text.parser.js +0 -41
  407. package/dist/parsers/text.parser.js.map +0 -1
  408. package/dist/parsers/words.parser.js +0 -40
  409. package/dist/parsers/words.parser.js.map +0 -1
  410. package/dist/queries/complex-english.query.d.ts +0 -2
  411. package/dist/queries/complex-english.query.js.map +0 -1
  412. package/dist/queries/flash-content.query.d.ts +0 -2
  413. package/dist/queries/flash-content.query.js +0 -82
  414. package/dist/queries/flash-content.query.js.map +0 -1
  415. package/dist/queries/linking-to-domains.query.d.ts +0 -2
  416. package/dist/queries/linking-to-domains.query.js +0 -130
  417. package/dist/queries/linking-to-domains.query.js.map +0 -1
  418. package/dist/queries/llamaguard-unsafe-content.query.d.ts +0 -2
  419. package/dist/queries/llamaguard-unsafe-content.query.js +0 -92
  420. package/dist/queries/llamaguard-unsafe-content.query.js.map +0 -1
  421. package/dist/queries/readability-score.query.d.ts +0 -2
  422. package/dist/queries/readability-score.query.js.map +0 -1
  423. package/dist/servers/crawl.server.js +0 -198
  424. package/dist/servers/crawl.server.js.map +0 -1
  425. package/dist/servers/express.server.js +0 -13
  426. package/dist/servers/express.server.js.map +0 -1
  427. package/dist/servers/maintenance.server.js +0 -103
  428. package/dist/servers/maintenance.server.js.map +0 -1
  429. package/dist/servers/rest.server.js +0 -46
  430. package/dist/servers/rest.server.js.map +0 -1
  431. package/dist/servers/socket-io.server.js +0 -61
  432. package/dist/servers/socket-io.server.js.map +0 -1
  433. package/dist/services/database.service.js +0 -795
  434. package/dist/services/database.service.js.map +0 -1
  435. package/dist/services/ollama-rest.service.js +0 -46
  436. package/dist/services/ollama-rest.service.js.map +0 -1
  437. package/dist/services/rig-llama-guard.service.js.map +0 -1
  438. package/dist/services/rig-queue-length.service.js +0 -22
  439. package/dist/services/rig-queue-length.service.js.map +0 -1
  440. package/dist/services/rig-subjectivity-scale.service.js +0 -96
  441. package/dist/services/rig-subjectivity-scale.service.js.map +0 -1
  442. package/dist/services/rig-ticket-rest.service.js +0 -29
  443. package/dist/services/rig-ticket-rest.service.js.map +0 -1
  444. package/dist/services/rig-ticketed-promise.service.js.map +0 -1
  445. package/dist/types/tcrawl-config.js.map +0 -1
  446. package/dist/types/thydra-config.js.map +0 -1
  447. package/dist/types/tparser-ctor.d.ts +0 -7
  448. package/dist/types/tparser-ctor.js +0 -2
  449. package/dist/types/tparser-ctor.js.map +0 -1
  450. package/dist/types/tquery.d.ts +0 -7
  451. package/dist/types/tquery.js +0 -2
  452. package/dist/types/tquery.js.map +0 -1
  453. package/dist/types/tqueue-length.js +0 -2
  454. package/dist/types/tqueue-length.js.map +0 -1
  455. package/dist/types/trobots-config.js.map +0 -1
  456. /package/dist/classes/{dns.d.ts → dns.d.mts} +0 -0
  457. /package/dist/{cli.d.ts → cli.d.mts} +0 -0
  458. /package/dist/enums/{eavailable-strategy.d.ts → eavailable-strategy.d.mts} +0 -0
  459. /package/dist/enums/{elist.d.ts → elist.d.mts} +0 -0
  460. /package/dist/enums/{eserver.d.ts → eserver.d.mts} +0 -0
  461. /package/dist/enums/{ex-powered-by.d.ts → ex-powered-by.d.mts} +0 -0
  462. /package/dist/interfaces/{imatch.d.ts → imatch.d.mts} +0 -0
  463. /package/dist/interfaces/{iparser-config.d.ts → iparser-config.d.mts} +0 -0
  464. /package/dist/interfaces/{iserver.d.ts → iserver.d.mts} +0 -0
  465. /package/dist/types/{tcrawl-config.d.ts → tcrawl-config.d.mts} +0 -0
  466. /package/dist/types/{thydra-config.d.ts → thydra-config.d.mts} +0 -0
  467. /package/dist/types/{tqueue-length.d.ts → tqueue-length.d.mts} +0 -0
  468. /package/dist/types/{trobots-config.d.ts → trobots-config.d.mts} +0 -0
@@ -0,0 +1,97 @@
1
+ import { commonsAsyncTimeout, CommonsSchedule } from 'tscommons-esm-async';
2
+ import { commonsLogAlert, commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
3
+ import { commonsGracefulAbortAddCallback } from 'nodecommons-esm-process';
4
+ import { Expirer } from '../classes/expirer.mjs';
5
+ import { Cleaner } from '../classes/cleaner.mjs';
6
+ var EAction;
7
+ (function (EAction) {
8
+ EAction["EXPIRE"] = "expire";
9
+ EAction["PURGE_EMPTY_DOMAINS"] = "purge-empty-domains";
10
+ EAction["PURGE_ORPHAN_URLS"] = "purge-orphan-urls";
11
+ })(EAction || (EAction = {}));
12
+ export function toEAction(value) {
13
+ switch (value) {
14
+ case EAction.EXPIRE.toString():
15
+ return EAction.EXPIRE;
16
+ case EAction.PURGE_EMPTY_DOMAINS.toString():
17
+ return EAction.PURGE_EMPTY_DOMAINS;
18
+ case EAction.PURGE_ORPHAN_URLS.toString():
19
+ return EAction.PURGE_ORPHAN_URLS;
20
+ }
21
+ return undefined;
22
+ }
23
+ export class MaintenanceServer {
24
+ crawl;
25
+ hardLimit;
26
+ schedule;
27
+ expirer;
28
+ cleaner;
29
+ isPaused = false;
30
+ isRunning = false;
31
+ constructor(times, expiry, lists, database, crawl, hardLimit) {
32
+ this.crawl = crawl;
33
+ this.hardLimit = hardLimit;
34
+ this.schedule = new CommonsSchedule('hydra-maintenance');
35
+ this.schedule.parse(times, (action) => toEAction(action), async (action) => {
36
+ // prevent duplicate runs
37
+ if (this.isRunning) {
38
+ commonsLogDebug('Maintenance is already running. Not re-running');
39
+ return;
40
+ }
41
+ this.isRunning = true;
42
+ try {
43
+ let claimedPause = false;
44
+ if (!this.isPaused) {
45
+ this.isPaused = true;
46
+ claimedPause = true;
47
+ this.crawl.pause();
48
+ for (let i = 30; i-- > 0;) {
49
+ commonsLogAlert(`Going down for maintenance ... ${i}`);
50
+ try {
51
+ await commonsAsyncTimeout(1000);
52
+ }
53
+ catch (_ex) {
54
+ /* do nothing */
55
+ }
56
+ }
57
+ }
58
+ await this.perform(action);
59
+ if (claimedPause) {
60
+ commonsLogAlert('Resuming from maintenance');
61
+ this.isPaused = false;
62
+ this.crawl.resume();
63
+ }
64
+ }
65
+ catch (e) {
66
+ console.log(e);
67
+ commonsLogError(e.message);
68
+ }
69
+ finally {
70
+ this.isRunning = false;
71
+ }
72
+ });
73
+ this.expirer = new Expirer(expiry, database);
74
+ this.cleaner = new Cleaner(lists, database);
75
+ commonsGracefulAbortAddCallback(() => {
76
+ commonsLogAlert('SIGINT abort flag is set. Aborting maintenance server.');
77
+ this.schedule.stop();
78
+ });
79
+ }
80
+ async perform(action) {
81
+ switch (action) {
82
+ case EAction.EXPIRE:
83
+ await this.expirer.expire();
84
+ break;
85
+ case EAction.PURGE_EMPTY_DOMAINS:
86
+ await this.cleaner.purgeEmptyDomains();
87
+ break;
88
+ case EAction.PURGE_ORPHAN_URLS:
89
+ await this.cleaner.purgeOrphanUrls(this.hardLimit);
90
+ break;
91
+ }
92
+ }
93
+ start() {
94
+ this.schedule.start();
95
+ }
96
+ }
97
+ //# sourceMappingURL=maintenance.server.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"maintenance.server.mjs","sourceRoot":"","sources":["../../src/servers/maintenance.server.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,qBAAqB,CAAC;AAEjG,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAC;AAI1E,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAMjD,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;IACnC,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAanB;IACA;IAbF,QAAQ,CAAkB;IAC1B,OAAO,CAAU;IACjB,OAAO,CAAU;IAEjB,QAAQ,GAAY,KAAK,CAAC;IAC1B,SAAS,GAAY,KAAK,CAAC;IAEnC,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB,EAClB,SAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAS;QAE3B,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,KAAK,EAAE,MAAe,EAAiB,EAAE;YACxC,yBAAyB;YACzB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,eAAe,CAAC,gDAAgD,CAAC,CAAC;gBAClE,OAAO;YACR,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YAEtB,IAAI,CAAC;gBACJ,IAAI,YAAY,GAAY,KAAK,CAAC;gBAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACrB,YAAY,GAAG,IAAI,CAAC;oBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAEnB,KAAK,IAAI,CAAC,GAAW,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;wBACnC,eAAe,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;wBAEvD,IAAI,CAAC;4BACJ,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBACjC,CAAC;wBAAC,OAAO,GAAG,EAAE,CAAC;4BACd,gBAAgB;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE3B,IAAI,YAAY,EAAE,CAAC;oBAClB,eAAe,CAAC,2BAA2B,CAAC,CAAC;oBAE7C,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;oBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,CAAC;YACF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,eAAe,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACxB,CAAC;QACF,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,eAAe,CAAC,wDAAwD,CAAC,CAAC;YAC1E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,MAAe;QACpC,QAAQ,MAAM,EAAE,CAAC;YAChB,KAAK,OAAO,CAAC,MAAM;gBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM;YACP,KAAK,OAAO,CAAC,mBAAmB;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBACvC,MAAM;YACP,KAAK,OAAO,CAAC,iBAAiB;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACnD,MAAM;QACR,CAAC;IACF,CAAC;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}
@@ -1,8 +1,8 @@
1
1
  import * as express from 'express';
2
- import { CommonsConfig } from 'tscommons-es-config';
3
- import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-es-express';
4
- import { CommonsRestServer } from 'nodecommons-es-rest';
5
- import { DatabaseService } from '../services/database.service';
2
+ import { CommonsConfig } from 'tscommons-esm-config';
3
+ import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
4
+ import { CommonsRestServer } from 'nodecommons-esm-rest';
5
+ import { DatabaseService } from '../services/database.service.mjs';
6
6
  export declare class RestServer extends CommonsRestServer<ICommonsRequestWithStrictParams, express.Response> {
7
7
  constructor(expressServer: CommonsStrictExpressServer, database: DatabaseService, config: CommonsConfig);
8
8
  }
@@ -0,0 +1,62 @@
1
+ import { isICommonsExpressConfig } from 'nodecommons-esm-express';
2
+ import { CommonsRestServer } from 'nodecommons-esm-rest';
3
+ import { commonsLogDoing } from 'nodecommons-esm-log';
4
+ import { commonsNginxProxyPath } from 'nodecommons-esm-http';
5
+ import { TestApi } from '../apis/test.api.mjs';
6
+ import { StatisticsApi } from '../apis/statistics.api.mjs';
7
+ import { CrawlApi } from '../apis/crawl.api.mjs';
8
+ import { BugsApi } from '../apis/bugs.api.mjs';
9
+ import { DomainsApi } from '../apis/domains.api.mjs';
10
+ import { UrlsApi } from '../apis/urls.api.mjs';
11
+ import { ImagesApi } from '../apis/images.api.mjs';
12
+ import { AutocompleteApi } from '../apis/autocomplete.api.mjs';
13
+ export class RestServer extends CommonsRestServer {
14
+ constructor(expressServer, database, config) {
15
+ super(expressServer);
16
+ const expressConfig = config.getObject('express');
17
+ if (!isICommonsExpressConfig(expressConfig))
18
+ throw new Error('Express config is not valid');
19
+ const path = commonsNginxProxyPath(expressConfig.path);
20
+ { // scope
21
+ const log = commonsLogDoing('rest-server', 'Installing REST Test API');
22
+ new TestApi(this, path);
23
+ log.success();
24
+ }
25
+ { // scope
26
+ const log = commonsLogDoing('rest-server', 'Installing REST Statistics API');
27
+ new StatisticsApi(this, database, path);
28
+ log.success();
29
+ }
30
+ { // scope
31
+ const log = commonsLogDoing('rest-server', 'Installing REST Crawl API');
32
+ new CrawlApi(this, database, path);
33
+ log.success();
34
+ }
35
+ { // scope
36
+ const log = commonsLogDoing('rest-server', 'Installing REST Bugs API');
37
+ new BugsApi(this, database, path);
38
+ log.success();
39
+ }
40
+ { // scope
41
+ const log = commonsLogDoing('rest-server', 'Installing REST Domains API');
42
+ new DomainsApi(this, database, path);
43
+ log.success();
44
+ }
45
+ { // scope
46
+ const log = commonsLogDoing('rest-server', 'Installing REST URLs API');
47
+ new UrlsApi(this, database, path);
48
+ log.success();
49
+ }
50
+ { // scope
51
+ const log = commonsLogDoing('rest-server', 'Installing REST Images API');
52
+ new ImagesApi(this, database, path);
53
+ log.success();
54
+ }
55
+ { // scope
56
+ const log = commonsLogDoing('rest-server', 'Installing REST Autocomplete API');
57
+ new AutocompleteApi(this, database, path);
58
+ log.success();
59
+ }
60
+ }
61
+ }
62
+ //# sourceMappingURL=rest.server.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rest.server.mjs","sourceRoot":"","sources":["../../src/servers/rest.server.mts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI/D,MAAM,OAAO,UAAW,SAAQ,iBAAoE;IACnG,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE/D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACxB,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,gCAAgC,CAAC,CAAC;YACjG,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACxC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,2BAA2B,CAAC,CAAC;YAC5F,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACnC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,6BAA6B,CAAC,CAAC;YAC9F,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACrC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;YAC7F,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACpC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,kCAAkC,CAAC,CAAC;YACnG,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1C,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
@@ -1,7 +1,7 @@
1
- import { TStatusTallies } from 'hydra-crawler-ts-assets';
2
- import { TFetch } from 'hydra-crawler-ts-assets';
3
- import { TOutcome } from 'hydra-crawler-ts-assets';
4
- import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
1
+ import { TStatusTallies } from 'hydra-crawler-esm-assets';
2
+ import { TFetch } from 'hydra-crawler-esm-assets';
3
+ import { TOutcome } from 'hydra-crawler-esm-assets';
4
+ import { CommonsAppSocketIoServer } from 'nodecommons-esm-app-socket-io';
5
5
  export declare class SocketIoServer extends CommonsAppSocketIoServer {
6
6
  fetching(fetch: TFetch): Promise<void>;
7
7
  outcome(outcome: TOutcome): Promise<void>;
@@ -0,0 +1,22 @@
1
+ import { CommonsAppSocketIoServer } from 'nodecommons-esm-app-socket-io';
2
+ export class SocketIoServer extends CommonsAppSocketIoServer {
3
+ async fetching(fetch) {
4
+ await super.broadcast('fetching', fetch);
5
+ }
6
+ async outcome(outcome) {
7
+ await super.broadcast('outcome', outcome);
8
+ }
9
+ async statusTallies(tallies) {
10
+ await super.broadcast('statusTallies', tallies);
11
+ }
12
+ async linkTallies(tally) {
13
+ await super.broadcast('linkTallies', tally);
14
+ }
15
+ async domainTallies(tally) {
16
+ await super.broadcast('domainTallies', tally);
17
+ }
18
+ async bandwidth(bps) {
19
+ await super.broadcast('bandwidth', bps);
20
+ }
21
+ }
22
+ //# sourceMappingURL=socket-io.server.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"socket-io.server.mjs","sourceRoot":"","sources":["../../src/servers/socket-io.server.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,OAAO,cAAe,SAAQ,wBAAwB;IACpD,KAAK,CAAC,QAAQ,CAAC,KAAa;QAClC,MAAM,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,OAAiB;QACrC,MAAM,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,OAAuB;QACjD,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,KAAa;QACrC,MAAM,KAAK,CAAC,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,KAAa;QACvC,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,MAAM,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;CACD"}
@@ -1,16 +1,16 @@
1
1
  import mongodb from 'mongodb';
2
- import { IUrlErrors } from 'hydra-crawler-ts-assets';
3
- import { IImage } from 'hydra-crawler-ts-assets';
4
- import { IUrl } from 'hydra-crawler-ts-assets';
5
- import { TPhpError } from 'hydra-crawler-ts-assets';
6
- import { TAspError } from 'hydra-crawler-ts-assets';
7
- import { TDomain } from 'hydra-crawler-ts-assets';
8
- import { TLink } from 'hydra-crawler-ts-assets';
9
- import { EStatus } from 'hydra-crawler-ts-assets';
10
- import { EComparator } from 'hydra-crawler-ts-assets';
11
- import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
12
- import { TParserCtor } from '../types/tparser-ctor';
13
- import { EAvailableStrategy } from '../enums/eavailable-strategy';
2
+ import { IUrlErrors } from 'hydra-crawler-esm-assets';
3
+ import { IImage } from 'hydra-crawler-esm-assets';
4
+ import { IUrl } from 'hydra-crawler-esm-assets';
5
+ import { TPhpError } from 'hydra-crawler-esm-assets';
6
+ import { TAspError } from 'hydra-crawler-esm-assets';
7
+ import { TDomain } from 'hydra-crawler-esm-assets';
8
+ import { TLink } from 'hydra-crawler-esm-assets';
9
+ import { EStatus } from 'hydra-crawler-esm-assets';
10
+ import { EComparator } from 'hydra-crawler-esm-assets';
11
+ import { CommonsMongodbService } from 'nodecommons-esm-database-mongodb';
12
+ import { TParserCtor } from '../types/tparser-ctor.mjs';
13
+ import { EAvailableStrategy } from '../enums/eavailable-strategy.mjs';
14
14
  export type TMongoIdRow = {
15
15
  _id: string;
16
16
  };