hydra-crawler 2.8.4 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/{autocomplete.api.d.ts → autocomplete.api.d.mts} +4 -4
- package/dist/apis/autocomplete.api.mjs +14 -0
- package/dist/apis/autocomplete.api.mjs.map +1 -0
- package/dist/apis/{bugs.api.d.ts → bugs.api.d.mts} +4 -4
- package/dist/apis/bugs.api.mjs +18 -0
- package/dist/apis/bugs.api.mjs.map +1 -0
- package/dist/apis/{crawl.api.d.ts → crawl.api.d.mts} +4 -4
- package/dist/apis/crawl.api.mjs +17 -0
- package/dist/apis/crawl.api.mjs.map +1 -0
- package/dist/apis/{domains.api.d.ts → domains.api.d.mts} +4 -4
- package/dist/apis/{domains.api.js → domains.api.mjs} +10 -18
- package/dist/apis/domains.api.mjs.map +1 -0
- package/dist/apis/{images.api.d.ts → images.api.d.mts} +4 -4
- package/dist/apis/images.api.mjs +19 -0
- package/dist/apis/images.api.mjs.map +1 -0
- package/dist/apis/{statistics.api.d.ts → statistics.api.d.mts} +4 -4
- package/dist/apis/statistics.api.mjs +33 -0
- package/dist/apis/statistics.api.mjs.map +1 -0
- package/dist/apis/{test.api.d.ts → test.api.d.mts} +3 -3
- package/dist/apis/test.api.mjs +10 -0
- package/dist/apis/test.api.mjs.map +1 -0
- package/dist/apis/{urls.api.d.ts → urls.api.d.mts} +4 -4
- package/dist/apis/urls.api.mjs +21 -0
- package/dist/apis/urls.api.mjs.map +1 -0
- package/dist/apps/{cleanup.app.d.ts → cleanup.app.d.mts} +5 -5
- package/dist/apps/cleanup.app.mjs +129 -0
- package/dist/apps/cleanup.app.mjs.map +1 -0
- package/dist/apps/{cross-populate-export.app.d.ts → cross-populate-export.app.d.mts} +3 -3
- package/dist/apps/cross-populate-export.app.mjs +61 -0
- package/dist/apps/cross-populate-export.app.mjs.map +1 -0
- package/dist/apps/{cross-populate-import.app.d.ts → cross-populate-import.app.d.mts} +3 -3
- package/dist/apps/cross-populate-import.app.mjs +61 -0
- package/dist/apps/cross-populate-import.app.mjs.map +1 -0
- package/dist/apps/{denylist.app.d.ts → denylist.app.d.mts} +5 -5
- package/dist/apps/denylist.app.mjs +122 -0
- package/dist/apps/denylist.app.mjs.map +1 -0
- package/dist/apps/{expire.app.d.ts → expire.app.d.mts} +8 -7
- package/dist/apps/expire.app.mjs +51 -0
- package/dist/apps/expire.app.mjs.map +1 -0
- package/dist/apps/{export-domain-urls.d.ts → export-domain-urls.d.mts} +3 -3
- package/dist/apps/export-domain-urls.mjs +85 -0
- package/dist/apps/export-domain-urls.mjs.map +1 -0
- package/dist/apps/{extract-text.app.d.ts → extract-text.app.d.mts} +2 -2
- package/dist/apps/extract-text.app.mjs +43 -0
- package/dist/apps/extract-text.app.mjs.map +1 -0
- package/dist/apps/{hydra.app.d.ts → hydra.app.d.mts} +10 -10
- package/dist/apps/hydra.app.mjs +222 -0
- package/dist/apps/hydra.app.mjs.map +1 -0
- package/dist/apps/{import.app.d.ts → import.app.d.mts} +3 -3
- package/dist/apps/import.app.mjs +48 -0
- package/dist/apps/import.app.mjs.map +1 -0
- package/dist/apps/{internal-hydra-common.app.d.ts → internal-hydra-common.app.d.mts} +6 -6
- package/dist/apps/{internal-hydra-common.app.js → internal-hydra-common.app.mjs} +1 -1
- package/dist/apps/internal-hydra-common.app.mjs.map +1 -0
- package/dist/apps/{move-to-archive.app.d.ts → move-to-archive.app.d.mts} +2 -2
- package/dist/apps/move-to-archive.app.mjs +31 -0
- package/dist/apps/move-to-archive.app.mjs.map +1 -0
- package/dist/apps/{prune-archive.app.d.ts → prune-archive.app.d.mts} +2 -2
- package/dist/apps/prune-archive.app.mjs +40 -0
- package/dist/apps/prune-archive.app.mjs.map +1 -0
- package/dist/apps/{query.app.d.ts → query.app.d.mts} +6 -6
- package/dist/apps/query.app.mjs +69 -0
- package/dist/apps/query.app.mjs.map +1 -0
- package/dist/apps/{reattempt.app.d.ts → reattempt.app.d.mts} +6 -6
- package/dist/apps/reattempt.app.mjs +74 -0
- package/dist/apps/reattempt.app.mjs.map +1 -0
- package/dist/apps/{requeue-domain.app.d.ts → requeue-domain.app.d.mts} +3 -3
- package/dist/apps/requeue-domain.app.mjs +52 -0
- package/dist/apps/requeue-domain.app.mjs.map +1 -0
- package/dist/apps/{seed.app.d.ts → seed.app.d.mts} +5 -5
- package/dist/apps/seed.app.mjs +59 -0
- package/dist/apps/seed.app.mjs.map +1 -0
- package/dist/apps/{startup.app.d.ts → startup.app.d.mts} +3 -3
- package/dist/apps/startup.app.mjs +55 -0
- package/dist/apps/startup.app.mjs.map +1 -0
- package/dist/apps/{unarchive-urls.app.d.ts → unarchive-urls.app.d.mts} +5 -5
- package/dist/apps/unarchive-urls.app.mjs +71 -0
- package/dist/apps/unarchive-urls.app.mjs.map +1 -0
- package/dist/classes/{cleaner.d.ts → cleaner.d.mts} +2 -2
- package/dist/classes/cleaner.mjs +257 -0
- package/dist/classes/cleaner.mjs.map +1 -0
- package/dist/classes/{crawler.d.ts → crawler.d.mts} +10 -12
- package/dist/classes/crawler.mjs +418 -0
- package/dist/classes/crawler.mjs.map +1 -0
- package/dist/classes/{dns.js → dns.mjs} +4 -4
- package/dist/classes/dns.mjs.map +1 -0
- package/dist/classes/{expirer.d.ts → expirer.d.mts} +2 -2
- package/dist/classes/expirer.mjs +117 -0
- package/dist/classes/expirer.mjs.map +1 -0
- package/dist/classes/{expiry.d.ts → expiry.d.mts} +1 -1
- package/dist/classes/{expiry.js → expiry.mjs} +7 -9
- package/dist/classes/expiry.mjs.map +1 -0
- package/dist/classes/{lists.d.ts → lists.d.mts} +2 -2
- package/dist/classes/{lists.js → lists.mjs} +4 -4
- package/dist/classes/lists.mjs.map +1 -0
- package/dist/classes/{robot.d.ts → robot.d.mts} +3 -3
- package/dist/classes/robot.mjs +74 -0
- package/dist/classes/robot.mjs.map +1 -0
- package/dist/classes/{tracker.d.ts → tracker.d.mts} +3 -3
- package/dist/classes/tracker.mjs +101 -0
- package/dist/classes/tracker.mjs.map +1 -0
- package/dist/{cli.js → cli.mjs} +46 -58
- package/dist/cli.mjs.map +1 -0
- package/dist/enums/{eavailable-strategy.js → eavailable-strategy.mjs} +1 -1
- package/dist/enums/eavailable-strategy.mjs.map +1 -0
- package/dist/enums/{elist.js → elist.mjs} +1 -1
- package/dist/enums/elist.mjs.map +1 -0
- package/dist/enums/{eserver.js → eserver.mjs} +1 -1
- package/dist/enums/eserver.mjs.map +1 -0
- package/dist/enums/{ex-powered-by.js → ex-powered-by.mjs} +1 -1
- package/dist/enums/ex-powered-by.mjs.map +1 -0
- package/dist/helpers/{matcher.d.ts → matcher.d.mts} +1 -1
- package/dist/helpers/{matcher.js → matcher.mjs} +1 -1
- package/dist/helpers/matcher.mjs.map +1 -0
- package/dist/helpers/{random.d.ts → random.d.mts} +1 -1
- package/dist/helpers/{random.js → random.mjs} +1 -1
- package/dist/helpers/random.mjs.map +1 -0
- package/dist/helpers/{utf-decoder.d.ts → utf-decoder.d.mts} +0 -1
- package/dist/helpers/{utf-decoder.js → utf-decoder.mjs} +3 -3
- package/dist/helpers/utf-decoder.mjs.map +1 -0
- package/dist/interfaces/{iexpiry.d.ts → iexpiry.d.mts} +1 -1
- package/dist/interfaces/{iexpiry.js → iexpiry.mjs} +3 -3
- package/dist/interfaces/iexpiry.mjs.map +1 -0
- package/dist/interfaces/{imatch.js → imatch.mjs} +2 -2
- package/dist/interfaces/imatch.mjs.map +1 -0
- package/dist/interfaces/{iparser-config.js → iparser-config.mjs} +2 -2
- package/dist/interfaces/iparser-config.mjs.map +1 -0
- package/dist/interfaces/{iparser.d.ts → iparser.d.mts} +1 -1
- package/dist/interfaces/iparser.mjs +2 -0
- package/dist/interfaces/iparser.mjs.map +1 -0
- package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts} +0 -2
- package/dist/interfaces/irequest-outcome.mjs +2 -0
- package/dist/interfaces/irequest-outcome.mjs.map +1 -0
- package/dist/interfaces/iserver.mjs +2 -0
- package/dist/interfaces/iserver.mjs.map +1 -0
- package/dist/parsers/accessibility-metrics.parser.d.mts +11 -0
- package/dist/parsers/accessibility-metrics.parser.mjs +30 -0
- package/dist/parsers/accessibility-metrics.parser.mjs.map +1 -0
- package/dist/parsers/asp-error.parser.d.mts +12 -0
- package/dist/parsers/asp-error.parser.mjs +38 -0
- package/dist/parsers/asp-error.parser.mjs.map +1 -0
- package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts} +6 -6
- package/dist/parsers/bad-words.parser.mjs +17 -0
- package/dist/parsers/bad-words.parser.mjs.map +1 -0
- package/dist/parsers/complex-english.parser.d.mts +15 -0
- package/dist/parsers/complex-english.parser.mjs +52 -0
- package/dist/parsers/complex-english.parser.mjs.map +1 -0
- package/dist/parsers/data.parser.d.mts +13 -0
- package/dist/parsers/{data.parser.js → data.parser.mjs} +8 -7
- package/dist/parsers/data.parser.mjs.map +1 -0
- package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts} +6 -6
- package/dist/parsers/dictionary.parser.mjs +63 -0
- package/dist/parsers/dictionary.parser.mjs.map +1 -0
- package/dist/parsers/html.parser.d.mts +13 -0
- package/dist/parsers/{html.parser.js → html.parser.mjs} +4 -3
- package/dist/parsers/html.parser.mjs.map +1 -0
- package/dist/parsers/hyperlinks.parser.d.mts +20 -0
- package/dist/parsers/hyperlinks.parser.mjs +104 -0
- package/dist/parsers/hyperlinks.parser.mjs.map +1 -0
- package/dist/parsers/image-tags.parser.d.mts +19 -0
- package/dist/parsers/image-tags.parser.mjs +42 -0
- package/dist/parsers/image-tags.parser.mjs.map +1 -0
- package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts} +7 -7
- package/dist/parsers/interest.parser.mjs +60 -0
- package/dist/parsers/interest.parser.mjs.map +1 -0
- package/dist/parsers/jpeg.parser.d.mts +11 -0
- package/dist/parsers/jpeg.parser.mjs +29 -0
- package/dist/parsers/jpeg.parser.mjs.map +1 -0
- package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts} +7 -7
- package/dist/parsers/llama-guard.parser.mjs +56 -0
- package/dist/parsers/llama-guard.parser.mjs.map +1 -0
- package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts} +7 -7
- package/dist/parsers/offence.parser.mjs +60 -0
- package/dist/parsers/offence.parser.mjs.map +1 -0
- package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts} +6 -6
- package/dist/parsers/ollama.parser.mjs +43 -0
- package/dist/parsers/ollama.parser.mjs.map +1 -0
- package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts} +5 -5
- package/dist/parsers/paragraphs.parser.mjs +38 -0
- package/dist/parsers/paragraphs.parser.mjs.map +1 -0
- package/dist/parsers/{parser.d.ts → parser.d.mts} +6 -6
- package/dist/parsers/parser.mjs +45 -0
- package/dist/parsers/parser.mjs.map +1 -0
- package/dist/parsers/php-error.parser.d.mts +12 -0
- package/dist/parsers/php-error.parser.mjs +42 -0
- package/dist/parsers/php-error.parser.mjs.map +1 -0
- package/dist/parsers/{phrase.parser.d.ts → phrase.parser.d.mts} +3 -3
- package/dist/parsers/phrase.parser.mjs +15 -0
- package/dist/parsers/phrase.parser.mjs.map +1 -0
- package/dist/parsers/{regex.parser.d.ts → regex.parser.d.mts} +3 -3
- package/dist/parsers/regex.parser.mjs +29 -0
- package/dist/parsers/regex.parser.mjs.map +1 -0
- package/dist/parsers/server.parser.d.mts +11 -0
- package/dist/parsers/server.parser.mjs +57 -0
- package/dist/parsers/server.parser.mjs.map +1 -0
- package/dist/parsers/{spelling.parser.d.ts → spelling.parser.d.mts} +6 -6
- package/dist/parsers/spelling.parser.mjs +17 -0
- package/dist/parsers/spelling.parser.mjs.map +1 -0
- package/dist/parsers/string.parser.d.mts +8 -0
- package/dist/parsers/{string.parser.js → string.parser.mjs} +5 -4
- package/dist/parsers/string.parser.mjs.map +1 -0
- package/dist/parsers/{text.parser.d.ts → text.parser.d.mts} +3 -3
- package/dist/parsers/text.parser.mjs +30 -0
- package/dist/parsers/text.parser.mjs.map +1 -0
- package/dist/parsers/{words.parser.d.ts → words.parser.d.mts} +3 -3
- package/dist/parsers/words.parser.mjs +29 -0
- package/dist/parsers/words.parser.mjs.map +1 -0
- package/dist/queries/complex-english.query.d.mts +2 -0
- package/dist/queries/{complex-english.query.js → complex-english.query.mjs} +18 -27
- package/dist/queries/complex-english.query.mjs.map +1 -0
- package/dist/queries/flash-content.query.d.mts +2 -0
- package/dist/queries/flash-content.query.mjs +80 -0
- package/dist/queries/flash-content.query.mjs.map +1 -0
- package/dist/queries/linking-to-domains.query.d.mts +2 -0
- package/dist/queries/linking-to-domains.query.mjs +128 -0
- package/dist/queries/linking-to-domains.query.mjs.map +1 -0
- package/dist/queries/llamaguard-unsafe-content.query.d.mts +2 -0
- package/dist/queries/llamaguard-unsafe-content.query.mjs +90 -0
- package/dist/queries/llamaguard-unsafe-content.query.mjs.map +1 -0
- package/dist/queries/readability-score.query.d.mts +2 -0
- package/dist/queries/{readability-score.query.js → readability-score.query.mjs} +11 -20
- package/dist/queries/readability-score.query.mjs.map +1 -0
- package/dist/servers/{crawl.server.d.ts → crawl.server.d.mts} +10 -10
- package/dist/servers/crawl.server.mjs +192 -0
- package/dist/servers/crawl.server.mjs.map +1 -0
- package/dist/servers/{express.server.d.ts → express.server.d.mts} +2 -3
- package/dist/servers/express.server.mjs +13 -0
- package/dist/servers/express.server.mjs.map +1 -0
- package/dist/servers/{maintenance.server.d.ts → maintenance.server.d.mts} +5 -5
- package/dist/servers/maintenance.server.mjs +97 -0
- package/dist/servers/maintenance.server.mjs.map +1 -0
- package/dist/servers/{rest.server.d.ts → rest.server.d.mts} +4 -4
- package/dist/servers/rest.server.mjs +62 -0
- package/dist/servers/rest.server.mjs.map +1 -0
- package/dist/servers/{socket-io.server.d.ts → socket-io.server.d.mts} +4 -4
- package/dist/servers/socket-io.server.mjs +22 -0
- package/dist/servers/socket-io.server.mjs.map +1 -0
- package/dist/services/{database.service.d.ts → database.service.d.mts} +12 -12
- package/dist/services/database.service.mjs +645 -0
- package/dist/services/database.service.mjs.map +1 -0
- package/dist/services/{ollama-rest.service.d.ts → ollama-rest.service.d.mts} +2 -2
- package/dist/services/ollama-rest.service.mjs +27 -0
- package/dist/services/ollama-rest.service.mjs.map +1 -0
- package/dist/services/{rig-llama-guard.service.d.ts → rig-llama-guard.service.d.mts} +2 -2
- package/dist/services/{rig-llama-guard.service.js → rig-llama-guard.service.mjs} +33 -43
- package/dist/services/rig-llama-guard.service.mjs.map +1 -0
- package/dist/services/{rig-queue-length.service.d.ts → rig-queue-length.service.d.mts} +1 -1
- package/dist/services/rig-queue-length.service.mjs +12 -0
- package/dist/services/rig-queue-length.service.mjs.map +1 -0
- package/dist/services/{rig-subjectivity-scale.service.d.ts → rig-subjectivity-scale.service.d.mts} +2 -2
- package/dist/services/rig-subjectivity-scale.service.mjs +82 -0
- package/dist/services/rig-subjectivity-scale.service.mjs.map +1 -0
- package/dist/services/{rig-ticket-rest.service.d.ts → rig-ticket-rest.service.d.mts} +3 -3
- package/dist/services/rig-ticket-rest.service.mjs +15 -0
- package/dist/services/rig-ticket-rest.service.mjs.map +1 -0
- package/dist/services/{rig-ticketed-promise.service.d.ts → rig-ticketed-promise.service.d.mts} +4 -6
- package/dist/services/{rig-ticketed-promise.service.js → rig-ticketed-promise.service.mjs} +7 -6
- package/dist/services/rig-ticketed-promise.service.mjs.map +1 -0
- package/dist/types/{tcrawl-config.js → tcrawl-config.mjs} +2 -2
- package/dist/types/tcrawl-config.mjs.map +1 -0
- package/dist/types/{thydra-config.js → thydra-config.mjs} +2 -2
- package/dist/types/thydra-config.mjs.map +1 -0
- package/dist/types/tparser-ctor.d.mts +7 -0
- package/dist/types/tparser-ctor.mjs +2 -0
- package/dist/types/tparser-ctor.mjs.map +1 -0
- package/dist/types/tquery.d.mts +7 -0
- package/dist/types/tquery.mjs +2 -0
- package/dist/types/tquery.mjs.map +1 -0
- package/dist/types/tqueue-length.mjs +2 -0
- package/dist/types/tqueue-length.mjs.map +1 -0
- package/dist/types/{trobots-config.js → trobots-config.mjs} +2 -2
- package/dist/types/trobots-config.mjs.map +1 -0
- package/package.json +37 -43
- package/dist/apis/autocomplete.api.js +0 -22
- package/dist/apis/autocomplete.api.js.map +0 -1
- package/dist/apis/bugs.api.js +0 -26
- package/dist/apis/bugs.api.js.map +0 -1
- package/dist/apis/crawl.api.js +0 -25
- package/dist/apis/crawl.api.js.map +0 -1
- package/dist/apis/domains.api.js.map +0 -1
- package/dist/apis/images.api.js +0 -27
- package/dist/apis/images.api.js.map +0 -1
- package/dist/apis/statistics.api.js +0 -41
- package/dist/apis/statistics.api.js.map +0 -1
- package/dist/apis/test.api.js +0 -19
- package/dist/apis/test.api.js.map +0 -1
- package/dist/apis/urls.api.js +0 -29
- package/dist/apis/urls.api.js.map +0 -1
- package/dist/apps/cleanup.app.js +0 -151
- package/dist/apps/cleanup.app.js.map +0 -1
- package/dist/apps/cross-populate-export.app.js +0 -75
- package/dist/apps/cross-populate-export.app.js.map +0 -1
- package/dist/apps/cross-populate-import.app.js +0 -100
- package/dist/apps/cross-populate-import.app.js.map +0 -1
- package/dist/apps/denylist.app.js +0 -132
- package/dist/apps/denylist.app.js.map +0 -1
- package/dist/apps/expire.app.js +0 -63
- package/dist/apps/expire.app.js.map +0 -1
- package/dist/apps/export-domain-urls.js +0 -99
- package/dist/apps/export-domain-urls.js.map +0 -1
- package/dist/apps/extract-text.app.js +0 -55
- package/dist/apps/extract-text.app.js.map +0 -1
- package/dist/apps/hydra.app.js +0 -218
- package/dist/apps/hydra.app.js.map +0 -1
- package/dist/apps/import.app.js +0 -57
- package/dist/apps/import.app.js.map +0 -1
- package/dist/apps/internal-hydra-common.app.js.map +0 -1
- package/dist/apps/move-to-archive.app.js +0 -46
- package/dist/apps/move-to-archive.app.js.map +0 -1
- package/dist/apps/prune-archive.app.js +0 -55
- package/dist/apps/prune-archive.app.js.map +0 -1
- package/dist/apps/query.app.js +0 -80
- package/dist/apps/query.app.js.map +0 -1
- package/dist/apps/reattempt.app.js +0 -83
- package/dist/apps/reattempt.app.js.map +0 -1
- package/dist/apps/requeue-domain.app.js +0 -64
- package/dist/apps/requeue-domain.app.js.map +0 -1
- package/dist/apps/seed.app.js +0 -69
- package/dist/apps/seed.app.js.map +0 -1
- package/dist/apps/startup.app.js +0 -64
- package/dist/apps/startup.app.js.map +0 -1
- package/dist/apps/unarchive-urls.app.js +0 -83
- package/dist/apps/unarchive-urls.app.js.map +0 -1
- package/dist/classes/cleaner.js +0 -266
- package/dist/classes/cleaner.js.map +0 -1
- package/dist/classes/crawler.js +0 -422
- package/dist/classes/crawler.js.map +0 -1
- package/dist/classes/dns.js.map +0 -1
- package/dist/classes/expirer.js +0 -121
- package/dist/classes/expirer.js.map +0 -1
- package/dist/classes/expiry.js.map +0 -1
- package/dist/classes/lists.js.map +0 -1
- package/dist/classes/robot.js +0 -82
- package/dist/classes/robot.js.map +0 -1
- package/dist/classes/tracker.js +0 -120
- package/dist/classes/tracker.js.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/enums/eavailable-strategy.js.map +0 -1
- package/dist/enums/elist.js.map +0 -1
- package/dist/enums/eserver.js.map +0 -1
- package/dist/enums/ex-powered-by.js.map +0 -1
- package/dist/helpers/matcher.js.map +0 -1
- package/dist/helpers/random.js.map +0 -1
- package/dist/helpers/utf-decoder.js.map +0 -1
- package/dist/interfaces/iexpiry.js.map +0 -1
- package/dist/interfaces/imatch.js.map +0 -1
- package/dist/interfaces/iparser-config.js.map +0 -1
- package/dist/interfaces/iparser.js +0 -2
- package/dist/interfaces/iparser.js.map +0 -1
- package/dist/interfaces/irequest-outcome.js +0 -2
- package/dist/interfaces/irequest-outcome.js.map +0 -1
- package/dist/interfaces/iserver.js +0 -2
- package/dist/interfaces/iserver.js.map +0 -1
- package/dist/parsers/accessibility-metrics.parser.d.ts +0 -11
- package/dist/parsers/accessibility-metrics.parser.js +0 -40
- package/dist/parsers/accessibility-metrics.parser.js.map +0 -1
- package/dist/parsers/asp-error.parser.d.ts +0 -12
- package/dist/parsers/asp-error.parser.js +0 -48
- package/dist/parsers/asp-error.parser.js.map +0 -1
- package/dist/parsers/bad-words.parser.js +0 -27
- package/dist/parsers/bad-words.parser.js.map +0 -1
- package/dist/parsers/complex-english.parser.d.ts +0 -15
- package/dist/parsers/complex-english.parser.js +0 -61
- package/dist/parsers/complex-english.parser.js.map +0 -1
- package/dist/parsers/data.parser.d.ts +0 -14
- package/dist/parsers/data.parser.js.map +0 -1
- package/dist/parsers/dictionary.parser.js +0 -73
- package/dist/parsers/dictionary.parser.js.map +0 -1
- package/dist/parsers/html.parser.d.ts +0 -13
- package/dist/parsers/html.parser.js.map +0 -1
- package/dist/parsers/hyperlinks.parser.d.ts +0 -20
- package/dist/parsers/hyperlinks.parser.js +0 -115
- package/dist/parsers/hyperlinks.parser.js.map +0 -1
- package/dist/parsers/image-tags.parser.d.ts +0 -19
- package/dist/parsers/image-tags.parser.js +0 -52
- package/dist/parsers/image-tags.parser.js.map +0 -1
- package/dist/parsers/interest.parser.js +0 -69
- package/dist/parsers/interest.parser.js.map +0 -1
- package/dist/parsers/jpeg.parser.d.ts +0 -11
- package/dist/parsers/jpeg.parser.js +0 -39
- package/dist/parsers/jpeg.parser.js.map +0 -1
- package/dist/parsers/llama-guard.parser.js +0 -65
- package/dist/parsers/llama-guard.parser.js.map +0 -1
- package/dist/parsers/offence.parser.js +0 -69
- package/dist/parsers/offence.parser.js.map +0 -1
- package/dist/parsers/ollama.parser.js +0 -51
- package/dist/parsers/ollama.parser.js.map +0 -1
- package/dist/parsers/paragraphs.parser.js +0 -49
- package/dist/parsers/paragraphs.parser.js.map +0 -1
- package/dist/parsers/parser.js +0 -57
- package/dist/parsers/parser.js.map +0 -1
- package/dist/parsers/php-error.parser.d.ts +0 -12
- package/dist/parsers/php-error.parser.js +0 -52
- package/dist/parsers/php-error.parser.js.map +0 -1
- package/dist/parsers/phrase.parser.js +0 -26
- package/dist/parsers/phrase.parser.js.map +0 -1
- package/dist/parsers/regex.parser.js +0 -43
- package/dist/parsers/regex.parser.js.map +0 -1
- package/dist/parsers/server.parser.d.ts +0 -11
- package/dist/parsers/server.parser.js +0 -67
- package/dist/parsers/server.parser.js.map +0 -1
- package/dist/parsers/spelling.parser.js +0 -27
- package/dist/parsers/spelling.parser.js.map +0 -1
- package/dist/parsers/string.parser.d.ts +0 -8
- package/dist/parsers/string.parser.js.map +0 -1
- package/dist/parsers/text.parser.js +0 -41
- package/dist/parsers/text.parser.js.map +0 -1
- package/dist/parsers/words.parser.js +0 -40
- package/dist/parsers/words.parser.js.map +0 -1
- package/dist/queries/complex-english.query.d.ts +0 -2
- package/dist/queries/complex-english.query.js.map +0 -1
- package/dist/queries/flash-content.query.d.ts +0 -2
- package/dist/queries/flash-content.query.js +0 -82
- package/dist/queries/flash-content.query.js.map +0 -1
- package/dist/queries/linking-to-domains.query.d.ts +0 -2
- package/dist/queries/linking-to-domains.query.js +0 -130
- package/dist/queries/linking-to-domains.query.js.map +0 -1
- package/dist/queries/llamaguard-unsafe-content.query.d.ts +0 -2
- package/dist/queries/llamaguard-unsafe-content.query.js +0 -92
- package/dist/queries/llamaguard-unsafe-content.query.js.map +0 -1
- package/dist/queries/readability-score.query.d.ts +0 -2
- package/dist/queries/readability-score.query.js.map +0 -1
- package/dist/servers/crawl.server.js +0 -198
- package/dist/servers/crawl.server.js.map +0 -1
- package/dist/servers/express.server.js +0 -13
- package/dist/servers/express.server.js.map +0 -1
- package/dist/servers/maintenance.server.js +0 -103
- package/dist/servers/maintenance.server.js.map +0 -1
- package/dist/servers/rest.server.js +0 -46
- package/dist/servers/rest.server.js.map +0 -1
- package/dist/servers/socket-io.server.js +0 -61
- package/dist/servers/socket-io.server.js.map +0 -1
- package/dist/services/database.service.js +0 -795
- package/dist/services/database.service.js.map +0 -1
- package/dist/services/ollama-rest.service.js +0 -46
- package/dist/services/ollama-rest.service.js.map +0 -1
- package/dist/services/rig-llama-guard.service.js.map +0 -1
- package/dist/services/rig-queue-length.service.js +0 -22
- package/dist/services/rig-queue-length.service.js.map +0 -1
- package/dist/services/rig-subjectivity-scale.service.js +0 -96
- package/dist/services/rig-subjectivity-scale.service.js.map +0 -1
- package/dist/services/rig-ticket-rest.service.js +0 -29
- package/dist/services/rig-ticket-rest.service.js.map +0 -1
- package/dist/services/rig-ticketed-promise.service.js.map +0 -1
- package/dist/types/tcrawl-config.js.map +0 -1
- package/dist/types/thydra-config.js.map +0 -1
- package/dist/types/tparser-ctor.d.ts +0 -7
- package/dist/types/tparser-ctor.js +0 -2
- package/dist/types/tparser-ctor.js.map +0 -1
- package/dist/types/tquery.d.ts +0 -7
- package/dist/types/tquery.js +0 -2
- package/dist/types/tquery.js.map +0 -1
- package/dist/types/tqueue-length.js +0 -2
- package/dist/types/tqueue-length.js.map +0 -1
- package/dist/types/trobots-config.js.map +0 -1
- /package/dist/classes/{dns.d.ts → dns.d.mts} +0 -0
- /package/dist/{cli.d.ts → cli.d.mts} +0 -0
- /package/dist/enums/{eavailable-strategy.d.ts → eavailable-strategy.d.mts} +0 -0
- /package/dist/enums/{elist.d.ts → elist.d.mts} +0 -0
- /package/dist/enums/{eserver.d.ts → eserver.d.mts} +0 -0
- /package/dist/enums/{ex-powered-by.d.ts → ex-powered-by.d.mts} +0 -0
- /package/dist/interfaces/{imatch.d.ts → imatch.d.mts} +0 -0
- /package/dist/interfaces/{iparser-config.d.ts → iparser-config.d.mts} +0 -0
- /package/dist/interfaces/{iserver.d.ts → iserver.d.mts} +0 -0
- /package/dist/types/{tcrawl-config.d.ts → tcrawl-config.d.mts} +0 -0
- /package/dist/types/{thydra-config.d.ts → thydra-config.d.mts} +0 -0
- /package/dist/types/{tqueue-length.d.ts → tqueue-length.d.mts} +0 -0
- /package/dist/types/{trobots-config.d.ts → trobots-config.d.mts} +0 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { commonsAsyncTimeout, CommonsSchedule } from 'tscommons-esm-async';
|
|
2
|
+
import { commonsLogAlert, commonsLogDebug, commonsLogError } from 'nodecommons-esm-log';
|
|
3
|
+
import { commonsGracefulAbortAddCallback } from 'nodecommons-esm-process';
|
|
4
|
+
import { Expirer } from '../classes/expirer.mjs';
|
|
5
|
+
import { Cleaner } from '../classes/cleaner.mjs';
|
|
6
|
+
var EAction;
|
|
7
|
+
(function (EAction) {
|
|
8
|
+
EAction["EXPIRE"] = "expire";
|
|
9
|
+
EAction["PURGE_EMPTY_DOMAINS"] = "purge-empty-domains";
|
|
10
|
+
EAction["PURGE_ORPHAN_URLS"] = "purge-orphan-urls";
|
|
11
|
+
})(EAction || (EAction = {}));
|
|
12
|
+
export function toEAction(value) {
|
|
13
|
+
switch (value) {
|
|
14
|
+
case EAction.EXPIRE.toString():
|
|
15
|
+
return EAction.EXPIRE;
|
|
16
|
+
case EAction.PURGE_EMPTY_DOMAINS.toString():
|
|
17
|
+
return EAction.PURGE_EMPTY_DOMAINS;
|
|
18
|
+
case EAction.PURGE_ORPHAN_URLS.toString():
|
|
19
|
+
return EAction.PURGE_ORPHAN_URLS;
|
|
20
|
+
}
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
export class MaintenanceServer {
|
|
24
|
+
crawl;
|
|
25
|
+
hardLimit;
|
|
26
|
+
schedule;
|
|
27
|
+
expirer;
|
|
28
|
+
cleaner;
|
|
29
|
+
isPaused = false;
|
|
30
|
+
isRunning = false;
|
|
31
|
+
constructor(times, expiry, lists, database, crawl, hardLimit) {
|
|
32
|
+
this.crawl = crawl;
|
|
33
|
+
this.hardLimit = hardLimit;
|
|
34
|
+
this.schedule = new CommonsSchedule('hydra-maintenance');
|
|
35
|
+
this.schedule.parse(times, (action) => toEAction(action), async (action) => {
|
|
36
|
+
// prevent duplicate runs
|
|
37
|
+
if (this.isRunning) {
|
|
38
|
+
commonsLogDebug('Maintenance is already running. Not re-running');
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
this.isRunning = true;
|
|
42
|
+
try {
|
|
43
|
+
let claimedPause = false;
|
|
44
|
+
if (!this.isPaused) {
|
|
45
|
+
this.isPaused = true;
|
|
46
|
+
claimedPause = true;
|
|
47
|
+
this.crawl.pause();
|
|
48
|
+
for (let i = 30; i-- > 0;) {
|
|
49
|
+
commonsLogAlert(`Going down for maintenance ... ${i}`);
|
|
50
|
+
try {
|
|
51
|
+
await commonsAsyncTimeout(1000);
|
|
52
|
+
}
|
|
53
|
+
catch (_ex) {
|
|
54
|
+
/* do nothing */
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
await this.perform(action);
|
|
59
|
+
if (claimedPause) {
|
|
60
|
+
commonsLogAlert('Resuming from maintenance');
|
|
61
|
+
this.isPaused = false;
|
|
62
|
+
this.crawl.resume();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
console.log(e);
|
|
67
|
+
commonsLogError(e.message);
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
this.isRunning = false;
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
this.expirer = new Expirer(expiry, database);
|
|
74
|
+
this.cleaner = new Cleaner(lists, database);
|
|
75
|
+
commonsGracefulAbortAddCallback(() => {
|
|
76
|
+
commonsLogAlert('SIGINT abort flag is set. Aborting maintenance server.');
|
|
77
|
+
this.schedule.stop();
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
async perform(action) {
|
|
81
|
+
switch (action) {
|
|
82
|
+
case EAction.EXPIRE:
|
|
83
|
+
await this.expirer.expire();
|
|
84
|
+
break;
|
|
85
|
+
case EAction.PURGE_EMPTY_DOMAINS:
|
|
86
|
+
await this.cleaner.purgeEmptyDomains();
|
|
87
|
+
break;
|
|
88
|
+
case EAction.PURGE_ORPHAN_URLS:
|
|
89
|
+
await this.cleaner.purgeOrphanUrls(this.hardLimit);
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
start() {
|
|
94
|
+
this.schedule.start();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=maintenance.server.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"maintenance.server.mjs","sourceRoot":"","sources":["../../src/servers/maintenance.server.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,qBAAqB,CAAC;AAEjG,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAC;AAI1E,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAMjD,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;IACnC,CAAC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAanB;IACA;IAbF,QAAQ,CAAkB;IAC1B,OAAO,CAAU;IACjB,OAAO,CAAU;IAEjB,QAAQ,GAAY,KAAK,CAAC;IAC1B,SAAS,GAAY,KAAK,CAAC;IAEnC,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB,EAClB,SAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAS;QAE3B,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,KAAK,EAAE,MAAe,EAAiB,EAAE;YACxC,yBAAyB;YACzB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,eAAe,CAAC,gDAAgD,CAAC,CAAC;gBAClE,OAAO;YACR,CAAC;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YAEtB,IAAI,CAAC;gBACJ,IAAI,YAAY,GAAY,KAAK,CAAC;gBAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACrB,YAAY,GAAG,IAAI,CAAC;oBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAEnB,KAAK,IAAI,CAAC,GAAW,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC;wBACnC,eAAe,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;wBAEvD,IAAI,CAAC;4BACJ,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBACjC,CAAC;wBAAC,OAAO,GAAG,EAAE,CAAC;4BACd,gBAAgB;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE3B,IAAI,YAAY,EAAE,CAAC;oBAClB,eAAe,CAAC,2BAA2B,CAAC,CAAC;oBAE7C,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;oBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,CAAC;YACF,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,eAAe,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;YACvC,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACxB,CAAC;QACF,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,eAAe,CAAC,wDAAwD,CAAC,CAAC;YAC1E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,MAAe;QACpC,QAAQ,MAAM,EAAE,CAAC;YAChB,KAAK,OAAO,CAAC,MAAM;gBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM;YACP,KAAK,OAAO,CAAC,mBAAmB;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;gBACvC,MAAM;YACP,KAAK,OAAO,CAAC,iBAAiB;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACnD,MAAM;QACR,CAAC;IACF,CAAC;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import * as express from 'express';
|
|
2
|
-
import { CommonsConfig } from 'tscommons-
|
|
3
|
-
import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-
|
|
4
|
-
import { CommonsRestServer } from 'nodecommons-
|
|
5
|
-
import { DatabaseService } from '../services/database.service';
|
|
2
|
+
import { CommonsConfig } from 'tscommons-esm-config';
|
|
3
|
+
import { CommonsStrictExpressServer, ICommonsRequestWithStrictParams } from 'nodecommons-esm-express';
|
|
4
|
+
import { CommonsRestServer } from 'nodecommons-esm-rest';
|
|
5
|
+
import { DatabaseService } from '../services/database.service.mjs';
|
|
6
6
|
export declare class RestServer extends CommonsRestServer<ICommonsRequestWithStrictParams, express.Response> {
|
|
7
7
|
constructor(expressServer: CommonsStrictExpressServer, database: DatabaseService, config: CommonsConfig);
|
|
8
8
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { isICommonsExpressConfig } from 'nodecommons-esm-express';
|
|
2
|
+
import { CommonsRestServer } from 'nodecommons-esm-rest';
|
|
3
|
+
import { commonsLogDoing } from 'nodecommons-esm-log';
|
|
4
|
+
import { commonsNginxProxyPath } from 'nodecommons-esm-http';
|
|
5
|
+
import { TestApi } from '../apis/test.api.mjs';
|
|
6
|
+
import { StatisticsApi } from '../apis/statistics.api.mjs';
|
|
7
|
+
import { CrawlApi } from '../apis/crawl.api.mjs';
|
|
8
|
+
import { BugsApi } from '../apis/bugs.api.mjs';
|
|
9
|
+
import { DomainsApi } from '../apis/domains.api.mjs';
|
|
10
|
+
import { UrlsApi } from '../apis/urls.api.mjs';
|
|
11
|
+
import { ImagesApi } from '../apis/images.api.mjs';
|
|
12
|
+
import { AutocompleteApi } from '../apis/autocomplete.api.mjs';
|
|
13
|
+
export class RestServer extends CommonsRestServer {
|
|
14
|
+
constructor(expressServer, database, config) {
|
|
15
|
+
super(expressServer);
|
|
16
|
+
const expressConfig = config.getObject('express');
|
|
17
|
+
if (!isICommonsExpressConfig(expressConfig))
|
|
18
|
+
throw new Error('Express config is not valid');
|
|
19
|
+
const path = commonsNginxProxyPath(expressConfig.path);
|
|
20
|
+
{ // scope
|
|
21
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Test API');
|
|
22
|
+
new TestApi(this, path);
|
|
23
|
+
log.success();
|
|
24
|
+
}
|
|
25
|
+
{ // scope
|
|
26
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Statistics API');
|
|
27
|
+
new StatisticsApi(this, database, path);
|
|
28
|
+
log.success();
|
|
29
|
+
}
|
|
30
|
+
{ // scope
|
|
31
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Crawl API');
|
|
32
|
+
new CrawlApi(this, database, path);
|
|
33
|
+
log.success();
|
|
34
|
+
}
|
|
35
|
+
{ // scope
|
|
36
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Bugs API');
|
|
37
|
+
new BugsApi(this, database, path);
|
|
38
|
+
log.success();
|
|
39
|
+
}
|
|
40
|
+
{ // scope
|
|
41
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Domains API');
|
|
42
|
+
new DomainsApi(this, database, path);
|
|
43
|
+
log.success();
|
|
44
|
+
}
|
|
45
|
+
{ // scope
|
|
46
|
+
const log = commonsLogDoing('rest-server', 'Installing REST URLs API');
|
|
47
|
+
new UrlsApi(this, database, path);
|
|
48
|
+
log.success();
|
|
49
|
+
}
|
|
50
|
+
{ // scope
|
|
51
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Images API');
|
|
52
|
+
new ImagesApi(this, database, path);
|
|
53
|
+
log.success();
|
|
54
|
+
}
|
|
55
|
+
{ // scope
|
|
56
|
+
const log = commonsLogDoing('rest-server', 'Installing REST Autocomplete API');
|
|
57
|
+
new AutocompleteApi(this, database, path);
|
|
58
|
+
log.success();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=rest.server.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rest.server.mjs","sourceRoot":"","sources":["../../src/servers/rest.server.mts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI/D,MAAM,OAAO,UAAW,SAAQ,iBAAoE;IACnG,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE/D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACxB,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,gCAAgC,CAAC,CAAC;YACjG,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACxC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,2BAA2B,CAAC,CAAC;YAC5F,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACnC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,6BAA6B,CAAC,CAAC;YAC9F,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACrC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;YAC7F,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YACpC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,aAAa,EAAE,kCAAkC,CAAC,CAAC;YACnG,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1C,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { TStatusTallies } from 'hydra-crawler-
|
|
2
|
-
import { TFetch } from 'hydra-crawler-
|
|
3
|
-
import { TOutcome } from 'hydra-crawler-
|
|
4
|
-
import { CommonsAppSocketIoServer } from 'nodecommons-
|
|
1
|
+
import { TStatusTallies } from 'hydra-crawler-esm-assets';
|
|
2
|
+
import { TFetch } from 'hydra-crawler-esm-assets';
|
|
3
|
+
import { TOutcome } from 'hydra-crawler-esm-assets';
|
|
4
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-esm-app-socket-io';
|
|
5
5
|
export declare class SocketIoServer extends CommonsAppSocketIoServer {
|
|
6
6
|
fetching(fetch: TFetch): Promise<void>;
|
|
7
7
|
outcome(outcome: TOutcome): Promise<void>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { CommonsAppSocketIoServer } from 'nodecommons-esm-app-socket-io';
|
|
2
|
+
export class SocketIoServer extends CommonsAppSocketIoServer {
|
|
3
|
+
async fetching(fetch) {
|
|
4
|
+
await super.broadcast('fetching', fetch);
|
|
5
|
+
}
|
|
6
|
+
async outcome(outcome) {
|
|
7
|
+
await super.broadcast('outcome', outcome);
|
|
8
|
+
}
|
|
9
|
+
async statusTallies(tallies) {
|
|
10
|
+
await super.broadcast('statusTallies', tallies);
|
|
11
|
+
}
|
|
12
|
+
async linkTallies(tally) {
|
|
13
|
+
await super.broadcast('linkTallies', tally);
|
|
14
|
+
}
|
|
15
|
+
async domainTallies(tally) {
|
|
16
|
+
await super.broadcast('domainTallies', tally);
|
|
17
|
+
}
|
|
18
|
+
async bandwidth(bps) {
|
|
19
|
+
await super.broadcast('bandwidth', bps);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=socket-io.server.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"socket-io.server.mjs","sourceRoot":"","sources":["../../src/servers/socket-io.server.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,OAAO,cAAe,SAAQ,wBAAwB;IACpD,KAAK,CAAC,QAAQ,CAAC,KAAa;QAClC,MAAM,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;IAEM,KAAK,CAAC,OAAO,CAAC,OAAiB;QACrC,MAAM,KAAK,CAAC,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,OAAuB;QACjD,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,KAAa;QACrC,MAAM,KAAK,CAAC,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAEM,KAAK,CAAC,aAAa,CAAC,KAAa;QACvC,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,GAAW;QACjC,MAAM,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;CACD"}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import mongodb from 'mongodb';
|
|
2
|
-
import { IUrlErrors } from 'hydra-crawler-
|
|
3
|
-
import { IImage } from 'hydra-crawler-
|
|
4
|
-
import { IUrl } from 'hydra-crawler-
|
|
5
|
-
import { TPhpError } from 'hydra-crawler-
|
|
6
|
-
import { TAspError } from 'hydra-crawler-
|
|
7
|
-
import { TDomain } from 'hydra-crawler-
|
|
8
|
-
import { TLink } from 'hydra-crawler-
|
|
9
|
-
import { EStatus } from 'hydra-crawler-
|
|
10
|
-
import { EComparator } from 'hydra-crawler-
|
|
11
|
-
import { CommonsMongodbService } from 'nodecommons-
|
|
12
|
-
import { TParserCtor } from '../types/tparser-ctor';
|
|
13
|
-
import { EAvailableStrategy } from '../enums/eavailable-strategy';
|
|
2
|
+
import { IUrlErrors } from 'hydra-crawler-esm-assets';
|
|
3
|
+
import { IImage } from 'hydra-crawler-esm-assets';
|
|
4
|
+
import { IUrl } from 'hydra-crawler-esm-assets';
|
|
5
|
+
import { TPhpError } from 'hydra-crawler-esm-assets';
|
|
6
|
+
import { TAspError } from 'hydra-crawler-esm-assets';
|
|
7
|
+
import { TDomain } from 'hydra-crawler-esm-assets';
|
|
8
|
+
import { TLink } from 'hydra-crawler-esm-assets';
|
|
9
|
+
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
10
|
+
import { EComparator } from 'hydra-crawler-esm-assets';
|
|
11
|
+
import { CommonsMongodbService } from 'nodecommons-esm-database-mongodb';
|
|
12
|
+
import { TParserCtor } from '../types/tparser-ctor.mjs';
|
|
13
|
+
import { EAvailableStrategy } from '../enums/eavailable-strategy.mjs';
|
|
14
14
|
export type TMongoIdRow = {
|
|
15
15
|
_id: string;
|
|
16
16
|
};
|