hydra-crawler 2.8.4 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (468) hide show
  1. package/dist/apis/{autocomplete.api.d.ts → autocomplete.api.d.mts} +4 -4
  2. package/dist/apis/autocomplete.api.mjs +14 -0
  3. package/dist/apis/autocomplete.api.mjs.map +1 -0
  4. package/dist/apis/{bugs.api.d.ts → bugs.api.d.mts} +4 -4
  5. package/dist/apis/bugs.api.mjs +18 -0
  6. package/dist/apis/bugs.api.mjs.map +1 -0
  7. package/dist/apis/{crawl.api.d.ts → crawl.api.d.mts} +4 -4
  8. package/dist/apis/crawl.api.mjs +17 -0
  9. package/dist/apis/crawl.api.mjs.map +1 -0
  10. package/dist/apis/{domains.api.d.ts → domains.api.d.mts} +4 -4
  11. package/dist/apis/{domains.api.js → domains.api.mjs} +10 -18
  12. package/dist/apis/domains.api.mjs.map +1 -0
  13. package/dist/apis/{images.api.d.ts → images.api.d.mts} +4 -4
  14. package/dist/apis/images.api.mjs +19 -0
  15. package/dist/apis/images.api.mjs.map +1 -0
  16. package/dist/apis/{statistics.api.d.ts → statistics.api.d.mts} +4 -4
  17. package/dist/apis/statistics.api.mjs +33 -0
  18. package/dist/apis/statistics.api.mjs.map +1 -0
  19. package/dist/apis/{test.api.d.ts → test.api.d.mts} +3 -3
  20. package/dist/apis/test.api.mjs +10 -0
  21. package/dist/apis/test.api.mjs.map +1 -0
  22. package/dist/apis/{urls.api.d.ts → urls.api.d.mts} +4 -4
  23. package/dist/apis/urls.api.mjs +21 -0
  24. package/dist/apis/urls.api.mjs.map +1 -0
  25. package/dist/apps/{cleanup.app.d.ts → cleanup.app.d.mts} +5 -5
  26. package/dist/apps/cleanup.app.mjs +129 -0
  27. package/dist/apps/cleanup.app.mjs.map +1 -0
  28. package/dist/apps/{cross-populate-export.app.d.ts → cross-populate-export.app.d.mts} +3 -3
  29. package/dist/apps/cross-populate-export.app.mjs +61 -0
  30. package/dist/apps/cross-populate-export.app.mjs.map +1 -0
  31. package/dist/apps/{cross-populate-import.app.d.ts → cross-populate-import.app.d.mts} +3 -3
  32. package/dist/apps/cross-populate-import.app.mjs +61 -0
  33. package/dist/apps/cross-populate-import.app.mjs.map +1 -0
  34. package/dist/apps/{denylist.app.d.ts → denylist.app.d.mts} +5 -5
  35. package/dist/apps/denylist.app.mjs +122 -0
  36. package/dist/apps/denylist.app.mjs.map +1 -0
  37. package/dist/apps/{expire.app.d.ts → expire.app.d.mts} +8 -7
  38. package/dist/apps/expire.app.mjs +51 -0
  39. package/dist/apps/expire.app.mjs.map +1 -0
  40. package/dist/apps/{export-domain-urls.d.ts → export-domain-urls.d.mts} +3 -3
  41. package/dist/apps/export-domain-urls.mjs +85 -0
  42. package/dist/apps/export-domain-urls.mjs.map +1 -0
  43. package/dist/apps/{extract-text.app.d.ts → extract-text.app.d.mts} +2 -2
  44. package/dist/apps/extract-text.app.mjs +43 -0
  45. package/dist/apps/extract-text.app.mjs.map +1 -0
  46. package/dist/apps/{hydra.app.d.ts → hydra.app.d.mts} +10 -10
  47. package/dist/apps/hydra.app.mjs +222 -0
  48. package/dist/apps/hydra.app.mjs.map +1 -0
  49. package/dist/apps/{import.app.d.ts → import.app.d.mts} +3 -3
  50. package/dist/apps/import.app.mjs +48 -0
  51. package/dist/apps/import.app.mjs.map +1 -0
  52. package/dist/apps/{internal-hydra-common.app.d.ts → internal-hydra-common.app.d.mts} +6 -6
  53. package/dist/apps/{internal-hydra-common.app.js → internal-hydra-common.app.mjs} +1 -1
  54. package/dist/apps/internal-hydra-common.app.mjs.map +1 -0
  55. package/dist/apps/{move-to-archive.app.d.ts → move-to-archive.app.d.mts} +2 -2
  56. package/dist/apps/move-to-archive.app.mjs +31 -0
  57. package/dist/apps/move-to-archive.app.mjs.map +1 -0
  58. package/dist/apps/{prune-archive.app.d.ts → prune-archive.app.d.mts} +2 -2
  59. package/dist/apps/prune-archive.app.mjs +40 -0
  60. package/dist/apps/prune-archive.app.mjs.map +1 -0
  61. package/dist/apps/{query.app.d.ts → query.app.d.mts} +6 -6
  62. package/dist/apps/query.app.mjs +69 -0
  63. package/dist/apps/query.app.mjs.map +1 -0
  64. package/dist/apps/{reattempt.app.d.ts → reattempt.app.d.mts} +6 -6
  65. package/dist/apps/reattempt.app.mjs +74 -0
  66. package/dist/apps/reattempt.app.mjs.map +1 -0
  67. package/dist/apps/{requeue-domain.app.d.ts → requeue-domain.app.d.mts} +3 -3
  68. package/dist/apps/requeue-domain.app.mjs +52 -0
  69. package/dist/apps/requeue-domain.app.mjs.map +1 -0
  70. package/dist/apps/{seed.app.d.ts → seed.app.d.mts} +5 -5
  71. package/dist/apps/seed.app.mjs +59 -0
  72. package/dist/apps/seed.app.mjs.map +1 -0
  73. package/dist/apps/{startup.app.d.ts → startup.app.d.mts} +3 -3
  74. package/dist/apps/startup.app.mjs +55 -0
  75. package/dist/apps/startup.app.mjs.map +1 -0
  76. package/dist/apps/{unarchive-urls.app.d.ts → unarchive-urls.app.d.mts} +5 -5
  77. package/dist/apps/unarchive-urls.app.mjs +71 -0
  78. package/dist/apps/unarchive-urls.app.mjs.map +1 -0
  79. package/dist/classes/{cleaner.d.ts → cleaner.d.mts} +2 -2
  80. package/dist/classes/cleaner.mjs +257 -0
  81. package/dist/classes/cleaner.mjs.map +1 -0
  82. package/dist/classes/{crawler.d.ts → crawler.d.mts} +10 -12
  83. package/dist/classes/crawler.mjs +418 -0
  84. package/dist/classes/crawler.mjs.map +1 -0
  85. package/dist/classes/{dns.js → dns.mjs} +4 -4
  86. package/dist/classes/dns.mjs.map +1 -0
  87. package/dist/classes/{expirer.d.ts → expirer.d.mts} +2 -2
  88. package/dist/classes/expirer.mjs +117 -0
  89. package/dist/classes/expirer.mjs.map +1 -0
  90. package/dist/classes/{expiry.d.ts → expiry.d.mts} +1 -1
  91. package/dist/classes/{expiry.js → expiry.mjs} +7 -9
  92. package/dist/classes/expiry.mjs.map +1 -0
  93. package/dist/classes/{lists.d.ts → lists.d.mts} +2 -2
  94. package/dist/classes/{lists.js → lists.mjs} +4 -4
  95. package/dist/classes/lists.mjs.map +1 -0
  96. package/dist/classes/{robot.d.ts → robot.d.mts} +3 -3
  97. package/dist/classes/robot.mjs +74 -0
  98. package/dist/classes/robot.mjs.map +1 -0
  99. package/dist/classes/{tracker.d.ts → tracker.d.mts} +3 -3
  100. package/dist/classes/tracker.mjs +101 -0
  101. package/dist/classes/tracker.mjs.map +1 -0
  102. package/dist/{cli.js → cli.mjs} +46 -58
  103. package/dist/cli.mjs.map +1 -0
  104. package/dist/enums/{eavailable-strategy.js → eavailable-strategy.mjs} +1 -1
  105. package/dist/enums/eavailable-strategy.mjs.map +1 -0
  106. package/dist/enums/{elist.js → elist.mjs} +1 -1
  107. package/dist/enums/elist.mjs.map +1 -0
  108. package/dist/enums/{eserver.js → eserver.mjs} +1 -1
  109. package/dist/enums/eserver.mjs.map +1 -0
  110. package/dist/enums/{ex-powered-by.js → ex-powered-by.mjs} +1 -1
  111. package/dist/enums/ex-powered-by.mjs.map +1 -0
  112. package/dist/helpers/{matcher.d.ts → matcher.d.mts} +1 -1
  113. package/dist/helpers/{matcher.js → matcher.mjs} +1 -1
  114. package/dist/helpers/matcher.mjs.map +1 -0
  115. package/dist/helpers/{random.d.ts → random.d.mts} +1 -1
  116. package/dist/helpers/{random.js → random.mjs} +1 -1
  117. package/dist/helpers/random.mjs.map +1 -0
  118. package/dist/helpers/{utf-decoder.d.ts → utf-decoder.d.mts} +0 -1
  119. package/dist/helpers/{utf-decoder.js → utf-decoder.mjs} +3 -3
  120. package/dist/helpers/utf-decoder.mjs.map +1 -0
  121. package/dist/interfaces/{iexpiry.d.ts → iexpiry.d.mts} +1 -1
  122. package/dist/interfaces/{iexpiry.js → iexpiry.mjs} +3 -3
  123. package/dist/interfaces/iexpiry.mjs.map +1 -0
  124. package/dist/interfaces/{imatch.js → imatch.mjs} +2 -2
  125. package/dist/interfaces/imatch.mjs.map +1 -0
  126. package/dist/interfaces/{iparser-config.js → iparser-config.mjs} +2 -2
  127. package/dist/interfaces/iparser-config.mjs.map +1 -0
  128. package/dist/interfaces/{iparser.d.ts → iparser.d.mts} +1 -1
  129. package/dist/interfaces/iparser.mjs +2 -0
  130. package/dist/interfaces/iparser.mjs.map +1 -0
  131. package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts} +0 -2
  132. package/dist/interfaces/irequest-outcome.mjs +2 -0
  133. package/dist/interfaces/irequest-outcome.mjs.map +1 -0
  134. package/dist/interfaces/iserver.mjs +2 -0
  135. package/dist/interfaces/iserver.mjs.map +1 -0
  136. package/dist/parsers/accessibility-metrics.parser.d.mts +11 -0
  137. package/dist/parsers/accessibility-metrics.parser.mjs +30 -0
  138. package/dist/parsers/accessibility-metrics.parser.mjs.map +1 -0
  139. package/dist/parsers/asp-error.parser.d.mts +12 -0
  140. package/dist/parsers/asp-error.parser.mjs +38 -0
  141. package/dist/parsers/asp-error.parser.mjs.map +1 -0
  142. package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts} +6 -6
  143. package/dist/parsers/bad-words.parser.mjs +17 -0
  144. package/dist/parsers/bad-words.parser.mjs.map +1 -0
  145. package/dist/parsers/complex-english.parser.d.mts +15 -0
  146. package/dist/parsers/complex-english.parser.mjs +52 -0
  147. package/dist/parsers/complex-english.parser.mjs.map +1 -0
  148. package/dist/parsers/data.parser.d.mts +13 -0
  149. package/dist/parsers/{data.parser.js → data.parser.mjs} +8 -7
  150. package/dist/parsers/data.parser.mjs.map +1 -0
  151. package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts} +6 -6
  152. package/dist/parsers/dictionary.parser.mjs +63 -0
  153. package/dist/parsers/dictionary.parser.mjs.map +1 -0
  154. package/dist/parsers/html.parser.d.mts +13 -0
  155. package/dist/parsers/{html.parser.js → html.parser.mjs} +4 -3
  156. package/dist/parsers/html.parser.mjs.map +1 -0
  157. package/dist/parsers/hyperlinks.parser.d.mts +20 -0
  158. package/dist/parsers/hyperlinks.parser.mjs +104 -0
  159. package/dist/parsers/hyperlinks.parser.mjs.map +1 -0
  160. package/dist/parsers/image-tags.parser.d.mts +19 -0
  161. package/dist/parsers/image-tags.parser.mjs +42 -0
  162. package/dist/parsers/image-tags.parser.mjs.map +1 -0
  163. package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts} +7 -7
  164. package/dist/parsers/interest.parser.mjs +60 -0
  165. package/dist/parsers/interest.parser.mjs.map +1 -0
  166. package/dist/parsers/jpeg.parser.d.mts +11 -0
  167. package/dist/parsers/jpeg.parser.mjs +29 -0
  168. package/dist/parsers/jpeg.parser.mjs.map +1 -0
  169. package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts} +7 -7
  170. package/dist/parsers/llama-guard.parser.mjs +56 -0
  171. package/dist/parsers/llama-guard.parser.mjs.map +1 -0
  172. package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts} +7 -7
  173. package/dist/parsers/offence.parser.mjs +60 -0
  174. package/dist/parsers/offence.parser.mjs.map +1 -0
  175. package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts} +6 -6
  176. package/dist/parsers/ollama.parser.mjs +43 -0
  177. package/dist/parsers/ollama.parser.mjs.map +1 -0
  178. package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts} +5 -5
  179. package/dist/parsers/paragraphs.parser.mjs +38 -0
  180. package/dist/parsers/paragraphs.parser.mjs.map +1 -0
  181. package/dist/parsers/{parser.d.ts → parser.d.mts} +6 -6
  182. package/dist/parsers/parser.mjs +45 -0
  183. package/dist/parsers/parser.mjs.map +1 -0
  184. package/dist/parsers/php-error.parser.d.mts +12 -0
  185. package/dist/parsers/php-error.parser.mjs +42 -0
  186. package/dist/parsers/php-error.parser.mjs.map +1 -0
  187. package/dist/parsers/{phrase.parser.d.ts → phrase.parser.d.mts} +3 -3
  188. package/dist/parsers/phrase.parser.mjs +15 -0
  189. package/dist/parsers/phrase.parser.mjs.map +1 -0
  190. package/dist/parsers/{regex.parser.d.ts → regex.parser.d.mts} +3 -3
  191. package/dist/parsers/regex.parser.mjs +29 -0
  192. package/dist/parsers/regex.parser.mjs.map +1 -0
  193. package/dist/parsers/server.parser.d.mts +11 -0
  194. package/dist/parsers/server.parser.mjs +57 -0
  195. package/dist/parsers/server.parser.mjs.map +1 -0
  196. package/dist/parsers/{spelling.parser.d.ts → spelling.parser.d.mts} +6 -6
  197. package/dist/parsers/spelling.parser.mjs +17 -0
  198. package/dist/parsers/spelling.parser.mjs.map +1 -0
  199. package/dist/parsers/string.parser.d.mts +8 -0
  200. package/dist/parsers/{string.parser.js → string.parser.mjs} +5 -4
  201. package/dist/parsers/string.parser.mjs.map +1 -0
  202. package/dist/parsers/{text.parser.d.ts → text.parser.d.mts} +3 -3
  203. package/dist/parsers/text.parser.mjs +30 -0
  204. package/dist/parsers/text.parser.mjs.map +1 -0
  205. package/dist/parsers/{words.parser.d.ts → words.parser.d.mts} +3 -3
  206. package/dist/parsers/words.parser.mjs +29 -0
  207. package/dist/parsers/words.parser.mjs.map +1 -0
  208. package/dist/queries/complex-english.query.d.mts +2 -0
  209. package/dist/queries/{complex-english.query.js → complex-english.query.mjs} +18 -27
  210. package/dist/queries/complex-english.query.mjs.map +1 -0
  211. package/dist/queries/flash-content.query.d.mts +2 -0
  212. package/dist/queries/flash-content.query.mjs +80 -0
  213. package/dist/queries/flash-content.query.mjs.map +1 -0
  214. package/dist/queries/linking-to-domains.query.d.mts +2 -0
  215. package/dist/queries/linking-to-domains.query.mjs +128 -0
  216. package/dist/queries/linking-to-domains.query.mjs.map +1 -0
  217. package/dist/queries/llamaguard-unsafe-content.query.d.mts +2 -0
  218. package/dist/queries/llamaguard-unsafe-content.query.mjs +90 -0
  219. package/dist/queries/llamaguard-unsafe-content.query.mjs.map +1 -0
  220. package/dist/queries/readability-score.query.d.mts +2 -0
  221. package/dist/queries/{readability-score.query.js → readability-score.query.mjs} +11 -20
  222. package/dist/queries/readability-score.query.mjs.map +1 -0
  223. package/dist/servers/{crawl.server.d.ts → crawl.server.d.mts} +10 -10
  224. package/dist/servers/crawl.server.mjs +192 -0
  225. package/dist/servers/crawl.server.mjs.map +1 -0
  226. package/dist/servers/{express.server.d.ts → express.server.d.mts} +2 -3
  227. package/dist/servers/express.server.mjs +13 -0
  228. package/dist/servers/express.server.mjs.map +1 -0
  229. package/dist/servers/{maintenance.server.d.ts → maintenance.server.d.mts} +5 -5
  230. package/dist/servers/maintenance.server.mjs +97 -0
  231. package/dist/servers/maintenance.server.mjs.map +1 -0
  232. package/dist/servers/{rest.server.d.ts → rest.server.d.mts} +4 -4
  233. package/dist/servers/rest.server.mjs +62 -0
  234. package/dist/servers/rest.server.mjs.map +1 -0
  235. package/dist/servers/{socket-io.server.d.ts → socket-io.server.d.mts} +4 -4
  236. package/dist/servers/socket-io.server.mjs +22 -0
  237. package/dist/servers/socket-io.server.mjs.map +1 -0
  238. package/dist/services/{database.service.d.ts → database.service.d.mts} +12 -12
  239. package/dist/services/database.service.mjs +645 -0
  240. package/dist/services/database.service.mjs.map +1 -0
  241. package/dist/services/{ollama-rest.service.d.ts → ollama-rest.service.d.mts} +2 -2
  242. package/dist/services/ollama-rest.service.mjs +27 -0
  243. package/dist/services/ollama-rest.service.mjs.map +1 -0
  244. package/dist/services/{rig-llama-guard.service.d.ts → rig-llama-guard.service.d.mts} +2 -2
  245. package/dist/services/{rig-llama-guard.service.js → rig-llama-guard.service.mjs} +33 -43
  246. package/dist/services/rig-llama-guard.service.mjs.map +1 -0
  247. package/dist/services/{rig-queue-length.service.d.ts → rig-queue-length.service.d.mts} +1 -1
  248. package/dist/services/rig-queue-length.service.mjs +12 -0
  249. package/dist/services/rig-queue-length.service.mjs.map +1 -0
  250. package/dist/services/{rig-subjectivity-scale.service.d.ts → rig-subjectivity-scale.service.d.mts} +2 -2
  251. package/dist/services/rig-subjectivity-scale.service.mjs +82 -0
  252. package/dist/services/rig-subjectivity-scale.service.mjs.map +1 -0
  253. package/dist/services/{rig-ticket-rest.service.d.ts → rig-ticket-rest.service.d.mts} +3 -3
  254. package/dist/services/rig-ticket-rest.service.mjs +15 -0
  255. package/dist/services/rig-ticket-rest.service.mjs.map +1 -0
  256. package/dist/services/{rig-ticketed-promise.service.d.ts → rig-ticketed-promise.service.d.mts} +4 -6
  257. package/dist/services/{rig-ticketed-promise.service.js → rig-ticketed-promise.service.mjs} +7 -6
  258. package/dist/services/rig-ticketed-promise.service.mjs.map +1 -0
  259. package/dist/types/{tcrawl-config.js → tcrawl-config.mjs} +2 -2
  260. package/dist/types/tcrawl-config.mjs.map +1 -0
  261. package/dist/types/{thydra-config.js → thydra-config.mjs} +2 -2
  262. package/dist/types/thydra-config.mjs.map +1 -0
  263. package/dist/types/tparser-ctor.d.mts +7 -0
  264. package/dist/types/tparser-ctor.mjs +2 -0
  265. package/dist/types/tparser-ctor.mjs.map +1 -0
  266. package/dist/types/tquery.d.mts +7 -0
  267. package/dist/types/tquery.mjs +2 -0
  268. package/dist/types/tquery.mjs.map +1 -0
  269. package/dist/types/tqueue-length.mjs +2 -0
  270. package/dist/types/tqueue-length.mjs.map +1 -0
  271. package/dist/types/{trobots-config.js → trobots-config.mjs} +2 -2
  272. package/dist/types/trobots-config.mjs.map +1 -0
  273. package/package.json +37 -43
  274. package/dist/apis/autocomplete.api.js +0 -22
  275. package/dist/apis/autocomplete.api.js.map +0 -1
  276. package/dist/apis/bugs.api.js +0 -26
  277. package/dist/apis/bugs.api.js.map +0 -1
  278. package/dist/apis/crawl.api.js +0 -25
  279. package/dist/apis/crawl.api.js.map +0 -1
  280. package/dist/apis/domains.api.js.map +0 -1
  281. package/dist/apis/images.api.js +0 -27
  282. package/dist/apis/images.api.js.map +0 -1
  283. package/dist/apis/statistics.api.js +0 -41
  284. package/dist/apis/statistics.api.js.map +0 -1
  285. package/dist/apis/test.api.js +0 -19
  286. package/dist/apis/test.api.js.map +0 -1
  287. package/dist/apis/urls.api.js +0 -29
  288. package/dist/apis/urls.api.js.map +0 -1
  289. package/dist/apps/cleanup.app.js +0 -151
  290. package/dist/apps/cleanup.app.js.map +0 -1
  291. package/dist/apps/cross-populate-export.app.js +0 -75
  292. package/dist/apps/cross-populate-export.app.js.map +0 -1
  293. package/dist/apps/cross-populate-import.app.js +0 -100
  294. package/dist/apps/cross-populate-import.app.js.map +0 -1
  295. package/dist/apps/denylist.app.js +0 -132
  296. package/dist/apps/denylist.app.js.map +0 -1
  297. package/dist/apps/expire.app.js +0 -63
  298. package/dist/apps/expire.app.js.map +0 -1
  299. package/dist/apps/export-domain-urls.js +0 -99
  300. package/dist/apps/export-domain-urls.js.map +0 -1
  301. package/dist/apps/extract-text.app.js +0 -55
  302. package/dist/apps/extract-text.app.js.map +0 -1
  303. package/dist/apps/hydra.app.js +0 -218
  304. package/dist/apps/hydra.app.js.map +0 -1
  305. package/dist/apps/import.app.js +0 -57
  306. package/dist/apps/import.app.js.map +0 -1
  307. package/dist/apps/internal-hydra-common.app.js.map +0 -1
  308. package/dist/apps/move-to-archive.app.js +0 -46
  309. package/dist/apps/move-to-archive.app.js.map +0 -1
  310. package/dist/apps/prune-archive.app.js +0 -55
  311. package/dist/apps/prune-archive.app.js.map +0 -1
  312. package/dist/apps/query.app.js +0 -80
  313. package/dist/apps/query.app.js.map +0 -1
  314. package/dist/apps/reattempt.app.js +0 -83
  315. package/dist/apps/reattempt.app.js.map +0 -1
  316. package/dist/apps/requeue-domain.app.js +0 -64
  317. package/dist/apps/requeue-domain.app.js.map +0 -1
  318. package/dist/apps/seed.app.js +0 -69
  319. package/dist/apps/seed.app.js.map +0 -1
  320. package/dist/apps/startup.app.js +0 -64
  321. package/dist/apps/startup.app.js.map +0 -1
  322. package/dist/apps/unarchive-urls.app.js +0 -83
  323. package/dist/apps/unarchive-urls.app.js.map +0 -1
  324. package/dist/classes/cleaner.js +0 -266
  325. package/dist/classes/cleaner.js.map +0 -1
  326. package/dist/classes/crawler.js +0 -422
  327. package/dist/classes/crawler.js.map +0 -1
  328. package/dist/classes/dns.js.map +0 -1
  329. package/dist/classes/expirer.js +0 -121
  330. package/dist/classes/expirer.js.map +0 -1
  331. package/dist/classes/expiry.js.map +0 -1
  332. package/dist/classes/lists.js.map +0 -1
  333. package/dist/classes/robot.js +0 -82
  334. package/dist/classes/robot.js.map +0 -1
  335. package/dist/classes/tracker.js +0 -120
  336. package/dist/classes/tracker.js.map +0 -1
  337. package/dist/cli.js.map +0 -1
  338. package/dist/enums/eavailable-strategy.js.map +0 -1
  339. package/dist/enums/elist.js.map +0 -1
  340. package/dist/enums/eserver.js.map +0 -1
  341. package/dist/enums/ex-powered-by.js.map +0 -1
  342. package/dist/helpers/matcher.js.map +0 -1
  343. package/dist/helpers/random.js.map +0 -1
  344. package/dist/helpers/utf-decoder.js.map +0 -1
  345. package/dist/interfaces/iexpiry.js.map +0 -1
  346. package/dist/interfaces/imatch.js.map +0 -1
  347. package/dist/interfaces/iparser-config.js.map +0 -1
  348. package/dist/interfaces/iparser.js +0 -2
  349. package/dist/interfaces/iparser.js.map +0 -1
  350. package/dist/interfaces/irequest-outcome.js +0 -2
  351. package/dist/interfaces/irequest-outcome.js.map +0 -1
  352. package/dist/interfaces/iserver.js +0 -2
  353. package/dist/interfaces/iserver.js.map +0 -1
  354. package/dist/parsers/accessibility-metrics.parser.d.ts +0 -11
  355. package/dist/parsers/accessibility-metrics.parser.js +0 -40
  356. package/dist/parsers/accessibility-metrics.parser.js.map +0 -1
  357. package/dist/parsers/asp-error.parser.d.ts +0 -12
  358. package/dist/parsers/asp-error.parser.js +0 -48
  359. package/dist/parsers/asp-error.parser.js.map +0 -1
  360. package/dist/parsers/bad-words.parser.js +0 -27
  361. package/dist/parsers/bad-words.parser.js.map +0 -1
  362. package/dist/parsers/complex-english.parser.d.ts +0 -15
  363. package/dist/parsers/complex-english.parser.js +0 -61
  364. package/dist/parsers/complex-english.parser.js.map +0 -1
  365. package/dist/parsers/data.parser.d.ts +0 -14
  366. package/dist/parsers/data.parser.js.map +0 -1
  367. package/dist/parsers/dictionary.parser.js +0 -73
  368. package/dist/parsers/dictionary.parser.js.map +0 -1
  369. package/dist/parsers/html.parser.d.ts +0 -13
  370. package/dist/parsers/html.parser.js.map +0 -1
  371. package/dist/parsers/hyperlinks.parser.d.ts +0 -20
  372. package/dist/parsers/hyperlinks.parser.js +0 -115
  373. package/dist/parsers/hyperlinks.parser.js.map +0 -1
  374. package/dist/parsers/image-tags.parser.d.ts +0 -19
  375. package/dist/parsers/image-tags.parser.js +0 -52
  376. package/dist/parsers/image-tags.parser.js.map +0 -1
  377. package/dist/parsers/interest.parser.js +0 -69
  378. package/dist/parsers/interest.parser.js.map +0 -1
  379. package/dist/parsers/jpeg.parser.d.ts +0 -11
  380. package/dist/parsers/jpeg.parser.js +0 -39
  381. package/dist/parsers/jpeg.parser.js.map +0 -1
  382. package/dist/parsers/llama-guard.parser.js +0 -65
  383. package/dist/parsers/llama-guard.parser.js.map +0 -1
  384. package/dist/parsers/offence.parser.js +0 -69
  385. package/dist/parsers/offence.parser.js.map +0 -1
  386. package/dist/parsers/ollama.parser.js +0 -51
  387. package/dist/parsers/ollama.parser.js.map +0 -1
  388. package/dist/parsers/paragraphs.parser.js +0 -49
  389. package/dist/parsers/paragraphs.parser.js.map +0 -1
  390. package/dist/parsers/parser.js +0 -57
  391. package/dist/parsers/parser.js.map +0 -1
  392. package/dist/parsers/php-error.parser.d.ts +0 -12
  393. package/dist/parsers/php-error.parser.js +0 -52
  394. package/dist/parsers/php-error.parser.js.map +0 -1
  395. package/dist/parsers/phrase.parser.js +0 -26
  396. package/dist/parsers/phrase.parser.js.map +0 -1
  397. package/dist/parsers/regex.parser.js +0 -43
  398. package/dist/parsers/regex.parser.js.map +0 -1
  399. package/dist/parsers/server.parser.d.ts +0 -11
  400. package/dist/parsers/server.parser.js +0 -67
  401. package/dist/parsers/server.parser.js.map +0 -1
  402. package/dist/parsers/spelling.parser.js +0 -27
  403. package/dist/parsers/spelling.parser.js.map +0 -1
  404. package/dist/parsers/string.parser.d.ts +0 -8
  405. package/dist/parsers/string.parser.js.map +0 -1
  406. package/dist/parsers/text.parser.js +0 -41
  407. package/dist/parsers/text.parser.js.map +0 -1
  408. package/dist/parsers/words.parser.js +0 -40
  409. package/dist/parsers/words.parser.js.map +0 -1
  410. package/dist/queries/complex-english.query.d.ts +0 -2
  411. package/dist/queries/complex-english.query.js.map +0 -1
  412. package/dist/queries/flash-content.query.d.ts +0 -2
  413. package/dist/queries/flash-content.query.js +0 -82
  414. package/dist/queries/flash-content.query.js.map +0 -1
  415. package/dist/queries/linking-to-domains.query.d.ts +0 -2
  416. package/dist/queries/linking-to-domains.query.js +0 -130
  417. package/dist/queries/linking-to-domains.query.js.map +0 -1
  418. package/dist/queries/llamaguard-unsafe-content.query.d.ts +0 -2
  419. package/dist/queries/llamaguard-unsafe-content.query.js +0 -92
  420. package/dist/queries/llamaguard-unsafe-content.query.js.map +0 -1
  421. package/dist/queries/readability-score.query.d.ts +0 -2
  422. package/dist/queries/readability-score.query.js.map +0 -1
  423. package/dist/servers/crawl.server.js +0 -198
  424. package/dist/servers/crawl.server.js.map +0 -1
  425. package/dist/servers/express.server.js +0 -13
  426. package/dist/servers/express.server.js.map +0 -1
  427. package/dist/servers/maintenance.server.js +0 -103
  428. package/dist/servers/maintenance.server.js.map +0 -1
  429. package/dist/servers/rest.server.js +0 -46
  430. package/dist/servers/rest.server.js.map +0 -1
  431. package/dist/servers/socket-io.server.js +0 -61
  432. package/dist/servers/socket-io.server.js.map +0 -1
  433. package/dist/services/database.service.js +0 -795
  434. package/dist/services/database.service.js.map +0 -1
  435. package/dist/services/ollama-rest.service.js +0 -46
  436. package/dist/services/ollama-rest.service.js.map +0 -1
  437. package/dist/services/rig-llama-guard.service.js.map +0 -1
  438. package/dist/services/rig-queue-length.service.js +0 -22
  439. package/dist/services/rig-queue-length.service.js.map +0 -1
  440. package/dist/services/rig-subjectivity-scale.service.js +0 -96
  441. package/dist/services/rig-subjectivity-scale.service.js.map +0 -1
  442. package/dist/services/rig-ticket-rest.service.js +0 -29
  443. package/dist/services/rig-ticket-rest.service.js.map +0 -1
  444. package/dist/services/rig-ticketed-promise.service.js.map +0 -1
  445. package/dist/types/tcrawl-config.js.map +0 -1
  446. package/dist/types/thydra-config.js.map +0 -1
  447. package/dist/types/tparser-ctor.d.ts +0 -7
  448. package/dist/types/tparser-ctor.js +0 -2
  449. package/dist/types/tparser-ctor.js.map +0 -1
  450. package/dist/types/tquery.d.ts +0 -7
  451. package/dist/types/tquery.js +0 -2
  452. package/dist/types/tquery.js.map +0 -1
  453. package/dist/types/tqueue-length.js +0 -2
  454. package/dist/types/tqueue-length.js.map +0 -1
  455. package/dist/types/trobots-config.js.map +0 -1
  456. /package/dist/classes/{dns.d.ts → dns.d.mts} +0 -0
  457. /package/dist/{cli.d.ts → cli.d.mts} +0 -0
  458. /package/dist/enums/{eavailable-strategy.d.ts → eavailable-strategy.d.mts} +0 -0
  459. /package/dist/enums/{elist.d.ts → elist.d.mts} +0 -0
  460. /package/dist/enums/{eserver.d.ts → eserver.d.mts} +0 -0
  461. /package/dist/enums/{ex-powered-by.d.ts → ex-powered-by.d.mts} +0 -0
  462. /package/dist/interfaces/{imatch.d.ts → imatch.d.mts} +0 -0
  463. /package/dist/interfaces/{iparser-config.d.ts → iparser-config.d.mts} +0 -0
  464. /package/dist/interfaces/{iserver.d.ts → iserver.d.mts} +0 -0
  465. /package/dist/types/{tcrawl-config.d.ts → tcrawl-config.d.mts} +0 -0
  466. /package/dist/types/{thydra-config.d.ts → thydra-config.d.mts} +0 -0
  467. /package/dist/types/{tqueue-length.d.ts → tqueue-length.d.mts} +0 -0
  468. /package/dist/types/{trobots-config.d.ts → trobots-config.d.mts} +0 -0
@@ -0,0 +1,645 @@
1
+ import { URL } from 'url';
2
+ import mongodb from 'mongodb';
3
+ import { commonsObjectStripNulls, commonsStringRegexLike, commonsTypeAttemptNumber, commonsTypeHasPropertyNumber, commonsTypeHasPropertyString, commonsTypeHasPropertyTArray } from 'tscommons-esm-core';
4
+ import { isIUrl } from 'hydra-crawler-esm-assets';
5
+ import { isTPhpError } from 'hydra-crawler-esm-assets';
6
+ import { isTAspError } from 'hydra-crawler-esm-assets';
7
+ import { isTDomain } from 'hydra-crawler-esm-assets';
8
+ import { isTLink } from 'hydra-crawler-esm-assets';
9
+ import { EStatus, toEStatus } from 'hydra-crawler-esm-assets';
10
+ import { EComparator } from 'hydra-crawler-esm-assets';
11
+ import { commonsLogDebug, commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
12
+ import { CommonsMongodbService } from 'nodecommons-esm-database-mongodb';
13
+ import { EAvailableStrategy } from '../enums/eavailable-strategy.mjs';
14
+ export function isTMongoIdRow(test) {
15
+ if (!commonsTypeHasPropertyString(test, '_id'))
16
+ return false;
17
+ return true;
18
+ }
19
+ export function isTMongoIdTallyRow(test) {
20
+ if (!isTMongoIdRow(test))
21
+ return false;
22
+ if (!commonsTypeHasPropertyNumber(test, 'tally'))
23
+ return false;
24
+ return true;
25
+ }
26
+ export class DatabaseService extends CommonsMongodbService {
27
+ domains;
28
+ urls;
29
+ links;
30
+ // not automatically used, only by manual cli running
31
+ archiveds;
32
+ getDomains() {
33
+ if (!this.domains)
34
+ throw new Error('Domains collected has not been instantiated yet');
35
+ return this.domains;
36
+ }
37
+ getUrls() {
38
+ if (!this.urls)
39
+ throw new Error('Urls collected has not been instantiated yet');
40
+ return this.urls;
41
+ }
42
+ getLinks() {
43
+ if (!this.links)
44
+ throw new Error('Links collected has not been instantiated yet');
45
+ return this.links;
46
+ }
47
+ getArchiveds() {
48
+ if (!this.archiveds)
49
+ throw new Error('Archiveds collected has not been instantiated yet');
50
+ return this.archiveds;
51
+ }
52
+ async init() {
53
+ await super.init();
54
+ if (!this.database)
55
+ throw new Error('Database has not been instantiated yet');
56
+ this.domains = await this.ensureCollection('domains');
57
+ await this.domains.createIndex({ domain: 1 }, { unique: true });
58
+ await this.domains.createIndex({ ip: 1 }, { unique: false });
59
+ this.urls = await this.ensureCollection('urls');
60
+ await this.urls.createIndex({ url: 1 }, { unique: true });
61
+ await this.urls.createIndex({ domain: 1 }, { unique: false });
62
+ await this.urls.createIndex({ status: 1 }, { unique: false });
63
+ await this.urls.createIndex({ attempted: 1 }, { unique: false });
64
+ await this.urls.createIndex({ orphan: 1 }, { unique: false });
65
+ await this.urls.createIndex({ statusCode: 1 }, { unique: false });
66
+ await this.urls.createIndex({ 'headers.content-type': 1 }, { unique: false });
67
+ await this.urls.createIndex({ 'headers.content-length': 1 }, { unique: false });
68
+ this.links = await this.ensureCollection('links');
69
+ await this.links.createIndex({ url: 1 }, { unique: false });
70
+ await this.links.createIndex({ outgoing: 1 }, { unique: false });
71
+ await this.links.createIndex({ url: 1, outgoing: 1 }, { unique: true });
72
+ this.archiveds = await this.ensureCollection('archiveds');
73
+ // no indices for the archiveds
74
+ }
75
+ async initParser(ctor) {
76
+ const parser = new ctor();
77
+ await parser.init(this);
78
+ }
79
+ getRawDatabase() {
80
+ return super.getRawDatabase();
81
+ }
82
+ async wipe() {
83
+ await this.getLinks().deleteMany({});
84
+ await this.getUrls().deleteMany({});
85
+ await this.getDomains().deleteMany({});
86
+ }
87
+ async resetActive() {
88
+ await this.getUrls().updateMany({ status: EStatus.ACTIVE }, { $set: { status: EStatus.QUEUED } });
89
+ }
90
+ async domain(domain, ip) {
91
+ try {
92
+ await this.getDomains().updateOne({ domain: domain }, { $set: { ip: ip } }, { upsert: true });
93
+ return true;
94
+ }
95
+ catch (ex) {
96
+ commonsLogDebug('debug position 7');
97
+ console.log(ex);
98
+ return false;
99
+ }
100
+ }
101
+ async queue(url, isDeny) {
102
+ const whatwg = new URL(url);
103
+ if (!whatwg.protocol.match(/^http(s?):$/))
104
+ return false;
105
+ try {
106
+ const status = isDeny ? EStatus.DENY : EStatus.QUEUED;
107
+ // un-archive if currently archived
108
+ await this.getUrls().deleteOne({ url: url, status: EStatus.ARCHIVED });
109
+ await this.getUrls().insertOne({
110
+ url: url,
111
+ domain: whatwg.hostname,
112
+ status: status
113
+ });
114
+ return true;
115
+ }
116
+ catch (_ex) {
117
+ return false;
118
+ }
119
+ }
120
+ async available(strategy, threshold, limit, existing, restrictTo) {
121
+ if (limit === 0)
122
+ return [];
123
+ const comparator = strategy === EAvailableStrategy.LARGEST ? EComparator.GTE : EComparator.LT;
124
+ const thresholdMatch = { tally: {} };
125
+ switch (comparator) {
126
+ case EComparator.GTE:
127
+ thresholdMatch.tally = { $gte: threshold };
128
+ break;
129
+ case EComparator.LT:
130
+ thresholdMatch.tally = { $lt: threshold };
131
+ break;
132
+ }
133
+ const sortOrder = strategy === EAvailableStrategy.SMALLEST ? 1 : -1;
134
+ const domainMatch = { $in: existing };
135
+ if (restrictTo.length > 0)
136
+ domainMatch['$in'] = restrictTo;
137
+ const results = this.getUrls().aggregate([
138
+ { $match: { status: EStatus.QUEUED, domain: domainMatch } },
139
+ { $group: { _id: '$domain', tally: { $sum: 1 } } },
140
+ { $match: thresholdMatch },
141
+ { $sort: { tally: sortOrder } },
142
+ { $limit: limit }
143
+ ], { allowDiskUse: true });
144
+ const entries = await this.listQueryResults(results, isTMongoIdTallyRow);
145
+ return entries
146
+ // eslint-disable-next-line no-underscore-dangle
147
+ .map((entry) => entry._id);
148
+ }
149
+ async next(domain) {
150
+ const next = await this.getUrls().findOne({
151
+ status: EStatus.QUEUED,
152
+ domain: domain
153
+ });
154
+ if (next === null)
155
+ return undefined;
156
+ return next.url;
157
+ }
158
+ async setStatus(url, status) {
159
+ const updates = { status: status };
160
+ if (![EStatus.QUEUED, EStatus.ACTIVE].includes(status))
161
+ updates['attempted'] = new Date();
162
+ if (status === EStatus.DONE)
163
+ updates['done'] = new Date();
164
+ try {
165
+ await this.getUrls().updateOne({ url: url }, { $set: updates });
166
+ return true;
167
+ }
168
+ catch (ex) {
169
+ console.error(ex);
170
+ return false;
171
+ }
172
+ }
173
+ async setStatusCode(url, code) {
174
+ try {
175
+ await this.getUrls().updateOne({ url: url }, { $set: { statusCode: code } });
176
+ return true;
177
+ }
178
+ catch (ex) {
179
+ console.error(ex);
180
+ return false;
181
+ }
182
+ }
183
+ async setHeaders(url, headers) {
184
+ try {
185
+ await this.getUrls().updateOne({ url: url }, { $set: { headers: headers } });
186
+ return true;
187
+ }
188
+ catch (ex) {
189
+ console.error(ex);
190
+ return false;
191
+ }
192
+ }
193
+ async setData(url, context, data) {
194
+ try {
195
+ const update = {};
196
+ update[context] = data;
197
+ await this.getUrls().updateOne({ url: url }, { $set: update });
198
+ return true;
199
+ }
200
+ catch (ex) {
201
+ console.error(ex);
202
+ return false;
203
+ }
204
+ }
205
+ async unsetData(url, context) {
206
+ try {
207
+ const update = {};
208
+ update[context] = true;
209
+ await this.getUrls().updateOne({ url: url }, { $unset: update });
210
+ return true;
211
+ }
212
+ catch (ex) {
213
+ console.error(ex);
214
+ return false;
215
+ }
216
+ }
217
+ async getTtl(url) {
218
+ const row = await this.getUrls().findOne({ url: url });
219
+ if (row === null)
220
+ return undefined;
221
+ if (!commonsTypeHasPropertyNumber(row, 'ttl'))
222
+ return undefined;
223
+ return row['ttl'];
224
+ }
225
+ async setTtl(url, ttl) {
226
+ try {
227
+ await this.getUrls().updateOne({ url: url }, { $set: { ttl: ttl } });
228
+ return true;
229
+ }
230
+ catch (ex) {
231
+ console.error(ex);
232
+ return false;
233
+ }
234
+ }
235
+ async unsetTtl(url) {
236
+ try {
237
+ await this.getUrls().updateOne({ url: url }, { $unset: { ttl: true } });
238
+ return true;
239
+ }
240
+ catch (ex) {
241
+ console.error(ex);
242
+ return false;
243
+ }
244
+ }
245
+ async getHash(url) {
246
+ const row = await this.getUrls().findOne({ url: url });
247
+ if (row === null)
248
+ return undefined;
249
+ if (!commonsTypeHasPropertyString(row, 'hash'))
250
+ return undefined;
251
+ return row['hash'];
252
+ }
253
+ async setHash(url, hash) {
254
+ try {
255
+ await this.getUrls().updateOne({ url: url }, { $set: {
256
+ hash: hash,
257
+ hashSet: new Date()
258
+ } });
259
+ return true;
260
+ }
261
+ catch (ex) {
262
+ console.error(ex);
263
+ return false;
264
+ }
265
+ }
266
+ async unsetHash(url) {
267
+ try {
268
+ await this.getUrls().updateOne({ url: url }, { $unset: {
269
+ hash: true,
270
+ hashSet: true
271
+ } });
272
+ return true;
273
+ }
274
+ catch (ex) {
275
+ console.error(ex);
276
+ return false;
277
+ }
278
+ }
279
+ async setFailReason(url, reason) {
280
+ try {
281
+ await this.getUrls().updateOne({ url: url }, { $set: {
282
+ reason: reason
283
+ } });
284
+ return true;
285
+ }
286
+ catch (ex) {
287
+ console.error(ex);
288
+ return false;
289
+ }
290
+ }
291
+ async unsetFailReason(url) {
292
+ try {
293
+ await this.getUrls().updateOne({ url: url }, { $unset: {
294
+ reason: true
295
+ } });
296
+ return true;
297
+ }
298
+ catch (ex) {
299
+ console.error(ex);
300
+ return false;
301
+ }
302
+ }
303
+ async link(url, links) {
304
+ // more efficient to only remove removed and only add new
305
+ // rather than just wiping all existing and re-adding
306
+ const find = this.getLinks().find({
307
+ url: url
308
+ }, {});
309
+ const existing = (await this.listQueryResults(find, isTLink))
310
+ .map((link) => link.outgoing);
311
+ const removed = [];
312
+ const added = [];
313
+ for (const link of links) {
314
+ if (!existing.includes(link) && !added.includes(link))
315
+ added.push(link);
316
+ }
317
+ for (const link of existing) {
318
+ if (!links.includes(link))
319
+ removed.push(link);
320
+ }
321
+ for (const outgoing of removed) {
322
+ try {
323
+ await this.getLinks().deleteMany({
324
+ url: url,
325
+ outgoing: outgoing
326
+ });
327
+ }
328
+ catch (_ex) {
329
+ /* do nothing */
330
+ }
331
+ }
332
+ for (const outgoing of added) {
333
+ try {
334
+ await this.getLinks().insertOne({
335
+ url: url,
336
+ outgoing: outgoing
337
+ });
338
+ }
339
+ catch (ex) {
340
+ switch (ex.code || -1) {
341
+ case 11000:
342
+ // ignore duplicates
343
+ commonsLogError(`DUPLICATE: ${url}, ${outgoing}`);
344
+ break;
345
+ case 17280:
346
+ case 17282:
347
+ commonsLogError(`INDEX TOO LARGE: ${url}, ${outgoing}`);
348
+ // ignore index too large
349
+ break;
350
+ default:
351
+ commonsLogDebug('debug position 8');
352
+ console.log(ex);
353
+ throw ex;
354
+ }
355
+ }
356
+ }
357
+ return true;
358
+ }
359
+ async markDead(domain) {
360
+ try {
361
+ await this.getUrls().updateMany({ domain: domain, status: { $in: [EStatus.QUEUED, EStatus.ACTIVE] } }, { $set: { status: EStatus.DEAD, attempted: new Date() } });
362
+ return true;
363
+ }
364
+ catch (ex) {
365
+ console.error(ex);
366
+ return false;
367
+ }
368
+ }
369
+ async listStatusTallies() {
370
+ const results = this.getUrls().aggregate([
371
+ { $match: { status: { $ne: EStatus.ARCHIVED } } },
372
+ { $group: {
373
+ _id: '$status',
374
+ tally: { $sum: 1 }
375
+ } }
376
+ ]);
377
+ const rows = await this.listQueryResults(results, isTMongoIdTallyRow);
378
+ const map = new Map();
379
+ for (const row of rows) {
380
+ // eslint-disable-next-line no-underscore-dangle
381
+ const status = toEStatus(row._id);
382
+ if (status)
383
+ map.set(status, row.tally);
384
+ }
385
+ return map;
386
+ }
387
+ async getLinkTalliesCount() {
388
+ return await this.getLinks().estimatedDocumentCount();
389
+ }
390
+ async getDomainTalliesCount() {
391
+ return await this.getDomains().estimatedDocumentCount();
392
+ }
393
+ async listDomainQueuedTallies() {
394
+ const results = this.getUrls().aggregate([
395
+ { $match: {
396
+ status: EStatus.QUEUED
397
+ } },
398
+ { $group: {
399
+ _id: '$domain',
400
+ tally: { $sum: 1 }
401
+ } }
402
+ ]);
403
+ const rows = await this.listQueryResults(results, isTMongoIdTallyRow);
404
+ const map = new Map();
405
+ for (const row of rows) {
406
+ // eslint-disable-next-line no-underscore-dangle
407
+ map.set(row._id, row.tally);
408
+ }
409
+ return map;
410
+ }
411
+ async listPhpErrors() {
412
+ const results = this.getUrls().find({
413
+ status: { $ne: EStatus.ARCHIVED },
414
+ phpErrors: { $exists: true }
415
+ }, {});
416
+ return (await this.listQueryResults(results, isIUrl))
417
+ .map((url) => {
418
+ if (!commonsTypeHasPropertyTArray(url, 'phpErrors', isTPhpError))
419
+ throw new Error('Invalid PHP error object');
420
+ return {
421
+ url: url.url,
422
+ errors: url.phpErrors
423
+ };
424
+ });
425
+ }
426
+ async listAspErrors() {
427
+ const results = this.getUrls().find({
428
+ status: { $ne: EStatus.ARCHIVED },
429
+ aspErrors: { $exists: true }
430
+ }, {});
431
+ return (await this.listQueryResults(results, isIUrl))
432
+ .map((url) => {
433
+ if (!commonsTypeHasPropertyTArray(url, 'aspErrors', isTAspError))
434
+ throw new Error('Invalid PHP error object');
435
+ return {
436
+ url: url.url,
437
+ errors: url.aspErrors
438
+ };
439
+ });
440
+ }
441
+ async listDone200DomainUrls(domain) {
442
+ const results = this.getUrls().find({ $and: [
443
+ { domain: domain },
444
+ { status: EStatus.DONE },
445
+ { statusCode: { $gte: 200 } },
446
+ { statusCode: { $lt: 300 } }
447
+ ] }, {});
448
+ return (await this.listQueryResults(results, isIUrl))
449
+ .map((url) => url.url);
450
+ }
451
+ async listDomains() {
452
+ const results = this.getDomains().find(
453
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
454
+ { ip: { $exists: true, $ne: null } }, // this is ok, despite the type objection to null
455
+ {});
456
+ // since we're doing $ne: null above, we don't need to strip nulls, as there won't be any
457
+ return await this.listQueryResults(results, isTDomain);
458
+ }
459
+ async listDomainsByLike(term) {
460
+ const results = this.getDomains().find({
461
+ domain: new RegExp(commonsStringRegexLike(`%${term}%`), 'i')
462
+ }, {});
463
+ return (await this.listQueryResults(results, isTDomain))
464
+ .map((encoded) => commonsObjectStripNulls(encoded));
465
+ }
466
+ async listInboundLinks(url) {
467
+ const results = this.getLinks().find({ outgoing: url }, {});
468
+ return (await this.listQueryResults(results, isTLink))
469
+ .map((link) => link.url);
470
+ }
471
+ async listOutboundLinks(url) {
472
+ const results = this.getLinks().find({ url: url }, {});
473
+ return (await this.listQueryResults(results, isTLink))
474
+ .map((link) => link.outgoing);
475
+ }
476
+ async listImagesBySizeThreshold(size, comparator) {
477
+ const queries = [
478
+ { status: EStatus.DONE },
479
+ { 'headers.content-type': /^image\/(jpeg)/ },
480
+ { 'headers.content-length': { $exists: true } }
481
+ ];
482
+ switch (comparator) {
483
+ case EComparator.GT:
484
+ queries.push({ 'headers.content-length': { $gt: size } });
485
+ break;
486
+ case EComparator.LT:
487
+ queries.push({ 'headers.content-length': { $lt: size } });
488
+ break;
489
+ case EComparator.GTE:
490
+ queries.push({ 'headers.content-length': { $gte: size } });
491
+ break;
492
+ case EComparator.LTE:
493
+ queries.push({ 'headers.content-length': { $lte: size } });
494
+ break;
495
+ }
496
+ const results = this.getUrls().find({ $and: queries }, {});
497
+ return (await this.listQueryResults(results, isIUrl))
498
+ .map((row) => ({
499
+ url: row.url,
500
+ size: commonsTypeAttemptNumber(row['headers']['content-length']) || -1
501
+ }));
502
+ }
503
+ async insertArchiveBatch(documents) {
504
+ const bulkInsert = this.getArchiveds().initializeUnorderedBulkOp();
505
+ const insertIds = [];
506
+ for await (const document of documents) {
507
+ const typecast = document;
508
+ // eslint-disable-next-line no-underscore-dangle
509
+ const id = typecast._id;
510
+ // Insert without raising an error for duplicates
511
+ bulkInsert
512
+ .find({ _id: id })
513
+ .upsert()
514
+ .replaceOne(document);
515
+ insertIds.push(id);
516
+ }
517
+ if (insertIds.length > 0)
518
+ await bulkInsert.execute();
519
+ return insertIds;
520
+ }
521
+ async deleteUrlsBatch(documents) {
522
+ // NB, this presumes that the links to and from this have already been removed by the cleaner
523
+ const bulkDelete = this.getUrls().initializeUnorderedBulkOp();
524
+ let tally = 0;
525
+ for await (const document of documents) {
526
+ const typecast = document;
527
+ // eslint-disable-next-line no-underscore-dangle
528
+ const id = typecast._id;
529
+ bulkDelete
530
+ .find({ _id: id })
531
+ .deleteOne();
532
+ tally++;
533
+ }
534
+ if (tally > 0)
535
+ await bulkDelete.execute();
536
+ }
537
+ async moveArchivedUrlsToArchive(batchSize, limit) {
538
+ let total = 0;
539
+ while (true) {
540
+ const log1 = commonsLogDoing('database', `Inserting up to ${batchSize} urls entries into archiveds`);
541
+ const existings = this.getUrls().find({ status: EStatus.ARCHIVED }).limit(batchSize);
542
+ const done = (await this.insertArchiveBatch(existings))
543
+ .map((id) => new mongodb.ObjectId(id));
544
+ await existings.close();
545
+ total += done.length;
546
+ log1.result(`${done.length}, ${total} total`);
547
+ if (done.length === 0)
548
+ break;
549
+ const log2 = commonsLogDoing('database', 'Deleting previously moved urls');
550
+ const toDelete = this.getUrls().find({ _id: { $in: done } }).limit(batchSize);
551
+ await this.deleteUrlsBatch(toDelete);
552
+ await toDelete.close();
553
+ if (limit !== undefined)
554
+ limit -= done.length;
555
+ log2.success();
556
+ if (limit !== undefined && limit <= 0)
557
+ break;
558
+ }
559
+ }
560
+ async listDuplicateArchivedUrls() {
561
+ const log1 = commonsLogDoing('database', 'Building cursor for all archiveds');
562
+ // we have to use this strange sharded increasing array of sets, as the maximum values a Set (or Map) can hold is 16777216 (2^24)
563
+ const urls = [];
564
+ const duplicates = new Set();
565
+ let tally = 0;
566
+ while (true) {
567
+ let subTally = 0;
568
+ const currentUrls = new Set();
569
+ try {
570
+ const archiveds = this.getArchiveds().find({}, {
571
+ sort: { _id: 1 },
572
+ skip: tally,
573
+ limit: 1000000
574
+ });
575
+ log1.success();
576
+ const log2 = commonsLogDoing('database', 'Searching for duplicates');
577
+ await archiveds.forEach((archived) => {
578
+ try {
579
+ let match = false;
580
+ for (const s of [...urls, currentUrls]) {
581
+ if (s.has(archived.url))
582
+ match = true;
583
+ }
584
+ if (match) {
585
+ duplicates.add(archived.url);
586
+ }
587
+ else {
588
+ currentUrls.add(archived.url);
589
+ }
590
+ tally++;
591
+ subTally++;
592
+ if ((tally % 10000) === 0) {
593
+ let total = 0;
594
+ for (const s of [...urls, currentUrls])
595
+ total += s.size;
596
+ log2.progress(`${tally}, ${total} unique urls, ${duplicates.size} duplicates`);
597
+ }
598
+ }
599
+ catch (e) {
600
+ console.log(e);
601
+ process.exit(1);
602
+ }
603
+ });
604
+ let batchTotal = 0;
605
+ for (const s of [...urls, currentUrls])
606
+ batchTotal += s.size;
607
+ log2.result(`${tally}, ${batchTotal} unique urls, ${duplicates.size} duplicates`);
608
+ await archiveds.close();
609
+ }
610
+ catch (e2) {
611
+ console.log(e2);
612
+ process.exit(1);
613
+ }
614
+ urls.push(currentUrls);
615
+ if (subTally === 0)
616
+ break;
617
+ }
618
+ return Array.from(duplicates.values());
619
+ }
620
+ async purgeArchiveDuplicates(url) {
621
+ const archiveds = this.getArchiveds().find({
622
+ url: url
623
+ }, {});
624
+ const bulkDelete = this.getArchiveds().initializeUnorderedBulkOp();
625
+ let first = true;
626
+ let tally = 0;
627
+ for await (const document of archiveds) {
628
+ if (first) {
629
+ first = false;
630
+ continue;
631
+ }
632
+ const typecast = document;
633
+ // eslint-disable-next-line no-underscore-dangle
634
+ const id = typecast._id;
635
+ bulkDelete
636
+ .find({ _id: id })
637
+ .deleteOne();
638
+ tally++;
639
+ }
640
+ if (tally > 0)
641
+ await bulkDelete.execute();
642
+ await archiveds.close();
643
+ }
644
+ }
645
+ //# sourceMappingURL=database.service.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.service.mjs","sourceRoot":"","sources":["../../src/services/database.service.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,4BAA4B,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAI1N,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAa,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAa,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAW,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAS,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAMzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AAKtE,MAAM,UAAU,aAAa,CAAC,IAAa;IAC1C,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAMD,MAAM,UAAU,kBAAkB,CAAC,IAAa;IAC/C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,eAAgB,SAAQ,qBAAqB;IACjD,OAAO,CAAwC;IAC/C,IAAI,CAAqC;IACzC,KAAK,CAAsC;IAEnD,qDAAqD;IAC7C,SAAS,CAAqC;IAE/C,UAAU;QAChB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAEtF,OAAO,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAEM,OAAO;QACb,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAEhF,OAAO,IAAI,CAAC,IAAI,CAAC;IAClB,CAAC;IAEM,QAAQ;QACd,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAElF,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IAEM,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAE1F,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAE9E,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAU,SAAS,CAAC,CAAC;QAC/D,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7D,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAO,MAAM,CAAC,CAAC;QACtD,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAClE,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9E,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,wBAAwB,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAEhF,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAQ,OAAO,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAExE,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAO,WAAW,CAAC,CAAC;QAChE,+BAA+B;IAChC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,IAAiB;QACxC,MAAM,MAAM,GAAY,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAEe,cAAc;QAC7B,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;IAC/B,CAAC;IAEM,KAAK,CAAC,IAAI;QAChB,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,WAAW;QACvB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAC1B,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CACrC,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,EAAU;QAC7C,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,SAAS,CAC/B,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EACpB,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;YAEF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,MAAgB;QAC/C,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;YAAE,OAAO,KAAK,CAAC;QAExD,IAAI,CAAC;YACJ,MAAM,MAAM,GAAY,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAE/D,mCAAmC;YACnC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEvE,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC;gBAC7B,GAAG,EAAE,GAAG;gBACR,MAAM,EAAE,MAAM,CAAC,QAAQ;gBACvB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,SAAS,CACpB,QAA4B,EAC5B,SAAiB,EACjB,KAAa,EACb,QAAkB,EAClB,UAAoB;QAErB,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE3B,MAAM,UAAU,GAAgB,QAAQ,KAAK,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;QAE3G,MAAM,cAAc,GAA2C,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAC7E,QAAQ,UAAU,EAAE,CAAC;YACpB,KAAK,WAAW,CAAC,GAAG;gBACnB,cAAc,CAAC,KAAK,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;gBAC3C,MAAM;YACP,KAAK,WAAW,CAAC,EAAE;gBAClB,cAAc,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;gBAC1C,MAAM;QACR,CAAC;QAED,MAAM,SAAS,GAAW,QAAQ,KAAK,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5E,MAAM,WAAW,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC;QACtC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;YAAE,WAAW,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;QAE3D,MAAM,OAAO,GAAgD,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAmB;YACtG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE;YAC3D,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;YAClD,EAAE,MAAM,EAAE,cAAc,EAAE;YAC1B,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAG,EAAE;YAChC,EAAE,MAAM,EAAE,KAAK,EAAE;SAClB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3B,MAAM,OAAO,GAAuB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAE7F,OAAO,OAAO;YACZ,gDAAgD;aAC/C,GAAG,CAAC,CAAC,KAAuB,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACxD,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,MAAc;QAC/B,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAO;YACzD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QACH,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,SAAS,CAAC;QAEpC,OAAO,IAAI,CAAC,GAAG,CAAC;IACjB,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,MAAe;QAClD,MAAM,OAAO,GAIT,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAEvB,IAAI,CAAC,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,WAAW,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QAE5F,IAAI,MAAM,KAAK,OAAO,CAAC,IAAI;YAAE,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1D,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE,OAAO,EAAE,CAClB,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,GAAW,EAAE,IAAY;QACnD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAC/B,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,OAAgD;QACpF,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAC/B,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,OAAe,EAAE,IAAa;QAC/D,IAAI,CAAC;YACJ,MAAM,MAAM,GAAiC,EAAE,CAAC;YAChD,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;YAEvB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE,MAAM,EAAE,CACjB,CAAC;YAEF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,OAAe;QAClD,IAAI,CAAC;YACJ,MAAM,MAAM,GAA8B,EAAE,CAAC;YAC7C,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;YAEvB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,MAAM,EAAE,MAAM,EAAE,CACnB,CAAC;YAEF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,GAAW;QAC9B,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACxE,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,SAAS,CAAC;QAEnC,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAChE,OAAQ,GAAkC,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,GAAW;QAC3C,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CACvB,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,GAAW;QAChC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAC1B,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,GAAW;QAC/B,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACxE,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,SAAS,CAAC;QAEnC,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC;YAAE,OAAO,SAAS,CAAC;QACjE,OAAQ,GAAmC,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,IAAY;QAC7C,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE;oBACN,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI,IAAI,EAAE;iBACpB,EAAE,CACJ,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,MAAM,EAAE;oBACR,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;iBACd,EAAE,CACJ,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,GAAW,EAAE,MAAc;QACrD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,IAAI,EAAE;oBACN,MAAM,EAAE,MAAM;iBACf,EAAE,CACJ,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,GAAW;QACvC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAC5B,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,IAAI;iBACb,EAAE,CACJ,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,KAAe;QAC7C,yDAAyD;QACzD,qDAAqD;QAErD,MAAM,IAAI,GAA8B,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ;YAClE,GAAG,EAAE,GAAG;SACT,EAAE,EAAE,CAAC,CAAC;QAEP,MAAM,QAAQ,GAAa,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;aACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC;oBAC/B,GAAG,EAAE,GAAG;oBACR,QAAQ,EAAE,QAAQ;iBACnB,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,gBAAgB;YACjB,CAAC;QACF,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC;oBAC9B,GAAG,EAAE,GAAG;oBACR,QAAQ,EAAE,QAAQ;iBACnB,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,QAAS,EAAuB,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC;oBAC7C,KAAK,KAAK;wBACT,oBAAoB;wBACpB,eAAe,CAAC,cAAc,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;wBAClD,MAAM;oBACP,KAAK,KAAK,CAAC;oBACX,KAAK,KAAK;wBACT,eAAe,CAAC,oBAAoB,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;wBACxD,yBAAyB;wBACzB,MAAM;oBACP;wBACC,eAAe,CAAC,kBAAkB,CAAC,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAChB,MAAM,EAAE,CAAC;gBACX,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,QAAQ,CAAC,MAAc;QACnC,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7B,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EACvE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,CAC1D,CAAC;YAEF,OAAO,IAAI,CAAC;QACb,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,MAAM,OAAO,GAAgD,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAmB;YACtG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE;YACjD,EAAE,MAAM,EAAE;oBACR,GAAG,EAAE,SAAS;oBACd,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;iBACnB,EAAE;SACJ,CAAC,CAAC;QAEH,MAAM,IAAI,GAAuB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAE1F,MAAM,GAAG,GAAyB,IAAI,GAAG,EAAmB,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,gDAAgD;YAChD,MAAM,MAAM,GAAsB,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,MAAM;gBAAE,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAEM,KAAK,CAAC,mBAAmB;QAC/B,OAAO,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,sBAAsB,EAAE,CAAC;IACvD,CAAC;IAEM,KAAK,CAAC,qBAAqB;QACjC,OAAO,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,sBAAsB,EAAE,CAAC;IACzD,CAAC;IAEM,KAAK,CAAC,uBAAuB;QACnC,MAAM,OAAO,GAAgD,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAmB;YACtG,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,OAAO,CAAC,MAAM;iBACvB,EAAE;YACH,EAAE,MAAM,EAAE;oBACR,GAAG,EAAE,SAAS;oBACd,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;iBACnB,EAAE;SACJ,CAAC,CAAC;QAEH,MAAM,IAAI,GAAuB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAE1F,MAAM,GAAG,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC3D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,gDAAgD;YAChD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAEM,KAAK,CAAC,aAAa;QACzB,MAAM,OAAO,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3D;YACE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;YACjC,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SAC7B,EACD,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;aAClD,GAAG,CAAC,CAAC,GAAS,EAAyB,EAAE;YACzC,IAAI,CAAC,4BAA4B,CAAY,GAAG,EAAE,WAAW,EAAE,WAAW,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAEzH,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAG,GAA6C,CAAC,SAAS;aACjE,CAAC;QACH,CAAC,CAAC,CAAC;IACN,CAAC;IAEM,KAAK,CAAC,aAAa;QACzB,MAAM,OAAO,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3D;YACE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;YACjC,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;SAC7B,EACD,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;aAClD,GAAG,CAAC,CAAC,GAAS,EAAyB,EAAE;YACzC,IAAI,CAAC,4BAA4B,CAAY,GAAG,EAAE,WAAW,EAAE,WAAW,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAEzH,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAG,GAA6C,CAAC,SAAS;aACjE,CAAC;QACH,CAAC,CAAC,CAAC;IACN,CAAC;IAEM,KAAK,CAAC,qBAAqB,CAAC,MAAc;QAChD,MAAM,OAAO,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3D,EAAE,IAAI,EAAE;gBACN,EAAE,MAAM,EAAE,MAAM,EAAE;gBAClB,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;gBACxB,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;gBAC7B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;aAC7B,EAAC,EACF,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;aAClD,GAAG,CAAC,CAAC,GAAS,EAAU,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,WAAW;QACvB,MAAM,OAAO,GAAgC,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI;QACjE,mEAAmE;QACnE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAW,EAAE,EAAE,EAAE,iDAAiD;QAC9F,EAAE,CACH,CAAC;QAEF,yFAAyF;QACzF,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACxD,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAC1C,MAAM,OAAO,GAAgC,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU;YAC3E,MAAM,EAAE,IAAI,MAAM,CAAC,sBAAsB,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC;SAC7D,EAAE,EAAE,CAAC,CAAC;QAEP,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;aACrD,GAAG,CAAC,CAAC,OAAgB,EAAW,EAAE,CAAC,uBAAuB,CAAC,OAAO,CAAY,CAAC,CAAC;IACpF,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACxC,MAAM,OAAO,GAA8B,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAC7D,EAAE,QAAQ,EAAE,GAAG,EAAE,EACjB,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aACnD,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,iBAAiB,CAAC,GAAW;QACzC,MAAM,OAAO,GAA8B,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAC7D,EAAE,GAAG,EAAE,GAAG,EAAE,EACZ,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aACnD,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAEM,KAAK,CAAC,yBAAyB,CAAC,IAAY,EAAE,UAAuB;QAC3E,MAAM,OAAO,GAAuE;YAClF,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;YACxB,EAAE,sBAAsB,EAAE,gBAAgB,EAAE;YAC5C,EAAE,wBAAwB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;SAChD,CAAC;QAEF,QAAQ,UAAU,EAAE,CAAC;YACpB,KAAK,WAAW,CAAC,EAAE;gBAClB,OAAO,CAAC,IAAI,CAAC,EAAE,wBAAwB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM;YACP,KAAK,WAAW,CAAC,EAAE;gBAClB,OAAO,CAAC,IAAI,CAAC,EAAE,wBAAwB,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM;YACP,KAAK,WAAW,CAAC,GAAG;gBACnB,OAAO,CAAC,IAAI,CAAC,EAAE,wBAAwB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACP,KAAK,WAAW,CAAC,GAAG;gBACnB,OAAO,CAAC,IAAI,CAAC,EAAE,wBAAwB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC3D,MAAM;QACR,CAAC;QAED,MAAM,OAAO,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3D,EAAE,IAAI,EAAE,OAAO,EAAE,EACjB,EAAE,CACH,CAAC;QAEF,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;aAClD,GAAG,CAAC,CAAC,GAAS,EAAU,EAAE,CAAC,CAAC;YAC3B,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,IAAI,EAAE,wBAAwB,CAAG,GAAuB,CAAC,SAAS,CAAkC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC;SAC9H,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,SAAmC;QACnE,MAAM,UAAU,GAAmC,IAAI,CAAC,YAAY,EAAE,CAAC,yBAAyB,EAAE,CAAC;QAEnG,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACxC,MAAM,QAAQ,GAA2B,QAAkC,CAAC;YAE5E,gDAAgD;YAChD,MAAM,EAAE,GAAW,QAAQ,CAAC,GAAG,CAAC;YAEhC,iDAAiD;YACjD,UAAU;iBACP,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;iBACjB,MAAM,EAAE;iBACR,UAAU,CAAC,QAAQ,CAAC,CAAC;YAExB,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;QAErD,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,SAAmC;QAChE,6FAA6F;QAE7F,MAAM,UAAU,GAAmC,IAAI,CAAC,OAAO,EAAE,CAAC,yBAAyB,EAAE,CAAC;QAE9F,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACxC,MAAM,QAAQ,GAA2B,QAAkC,CAAC;YAE5E,gDAAgD;YAChD,MAAM,EAAE,GAAY,QAAQ,CAAC,GAAG,CAAC;YAEjC,UAAU;iBACP,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;iBACjB,SAAS,EAAE,CAAC;YAEf,KAAK,EAAE,CAAC;QACT,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,yBAAyB,CACpC,SAAiB,EACjB,KAAc;QAEf,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAAuB,eAAe,CAAC,UAAU,EAAE,mBAAmB,SAAS,8BAA8B,CAAC,CAAC;YAEzH,MAAM,SAAS,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC/G,MAAM,IAAI,GAAuB,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;iBACxE,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YAEnE,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YAExB,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;YAErB,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,KAAK,KAAK,QAAQ,CAAC,CAAC;YAC9C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAE7B,MAAM,IAAI,GAAuB,eAAe,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAC;YAE/F,MAAM,QAAQ,GAA6B,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACxG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAErC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YAEvB,IAAI,KAAK,KAAK,SAAS;gBAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;YAE9C,IAAI,CAAC,OAAO,EAAE,CAAC;YAEf,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,CAAC;gBAAE,MAAM;QAC9C,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,yBAAyB;QACrC,MAAM,IAAI,GAAuB,eAAe,CAAC,UAAU,EAAE,mCAAmC,CAAC,CAAC;QAElG,iIAAiI;QAEjI,MAAM,IAAI,GAAkB,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAgB,IAAI,GAAG,EAAU,CAAC;QAClD,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,QAAQ,GAAW,CAAC,CAAC;YAEzB,MAAM,WAAW,GAAgB,IAAI,GAAG,EAAU,CAAC;YAEnD,IAAI,CAAC;gBACJ,MAAM,SAAS,GAA6B,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CAClE,EAAE,EACF;oBACE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;oBAChB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,OAAO;iBACf,CACF,CAAC;gBACF,IAAI,CAAC,OAAO,EAAE,CAAC;gBAEf,MAAM,IAAI,GAAuB,eAAe,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAC;gBAEzF,MAAM,SAAS,CAAC,OAAO,CACrB,CAAC,QAAc,EAAQ,EAAE;oBACxB,IAAI,CAAC;wBACJ,IAAI,KAAK,GAAY,KAAK,CAAC;wBAC3B,KAAK,MAAM,CAAC,IAAI,CAAE,GAAG,IAAI,EAAE,WAAW,CAAE,EAAE,CAAC;4BAC1C,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;gCAAE,KAAK,GAAG,IAAI,CAAC;wBACvC,CAAC;wBAED,IAAI,KAAK,EAAE,CAAC;4BACX,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAC9B,CAAC;6BAAM,CAAC;4BACP,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAC/B,CAAC;wBAED,KAAK,EAAE,CAAC;wBACR,QAAQ,EAAE,CAAC;wBACX,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;4BAC3B,IAAI,KAAK,GAAW,CAAC,CAAC;4BACtB,KAAK,MAAM,CAAC,IAAI,CAAE,GAAG,IAAI,EAAE,WAAW,CAAE;gCAAE,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC;4BAC1D,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,iBAAiB,UAAU,CAAC,IAAI,aAAa,CAAC,CAAC;wBAChF,CAAC;oBACF,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;wBACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBACjB,CAAC;gBACF,CAAC,CACF,CAAC;gBAEF,IAAI,UAAU,GAAW,CAAC,CAAC;gBAC3B,KAAK,MAAM,CAAC,IAAI,CAAE,GAAG,IAAI,EAAE,WAAW,CAAE;oBAAE,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC;gBAC/D,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,KAAK,UAAU,iBAAiB,UAAU,CAAC,IAAI,aAAa,CAAC,CAAC;gBAElF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACzB,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvB,IAAI,QAAQ,KAAK,CAAC;gBAAE,MAAM;QAC3B,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,sBAAsB,CAAC,GAAW;QAC9C,MAAM,SAAS,GAA6B,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CAClE;YACE,GAAG,EAAE,GAAG;SACT,EACD,EAAE,CACH,CAAC;QAEF,MAAM,UAAU,GAAmC,IAAI,CAAC,YAAY,EAAE,CAAC,yBAAyB,EAAE,CAAC;QAEnG,IAAI,KAAK,GAAY,IAAI,CAAC;QAC1B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,KAAK,CAAC;gBACd,SAAS;YACV,CAAC;YAED,MAAM,QAAQ,GAA2B,QAAkC,CAAC;YAE5E,gDAAgD;YAChD,MAAM,EAAE,GAAY,QAAQ,CAAC,GAAG,CAAC;YAEjC,UAAU;iBACP,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;iBACjB,SAAS,EAAE,CAAC;YAEf,KAAK,EAAE,CAAC;QACT,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;QAE1C,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACD"}