hydra-crawler 2.8.4 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apis/{autocomplete.api.d.ts → autocomplete.api.d.mts} +4 -4
- package/dist/apis/autocomplete.api.mjs +14 -0
- package/dist/apis/autocomplete.api.mjs.map +1 -0
- package/dist/apis/{bugs.api.d.ts → bugs.api.d.mts} +4 -4
- package/dist/apis/bugs.api.mjs +18 -0
- package/dist/apis/bugs.api.mjs.map +1 -0
- package/dist/apis/{crawl.api.d.ts → crawl.api.d.mts} +4 -4
- package/dist/apis/crawl.api.mjs +17 -0
- package/dist/apis/crawl.api.mjs.map +1 -0
- package/dist/apis/{domains.api.d.ts → domains.api.d.mts} +4 -4
- package/dist/apis/{domains.api.js → domains.api.mjs} +10 -18
- package/dist/apis/domains.api.mjs.map +1 -0
- package/dist/apis/{images.api.d.ts → images.api.d.mts} +4 -4
- package/dist/apis/images.api.mjs +19 -0
- package/dist/apis/images.api.mjs.map +1 -0
- package/dist/apis/{statistics.api.d.ts → statistics.api.d.mts} +4 -4
- package/dist/apis/statistics.api.mjs +33 -0
- package/dist/apis/statistics.api.mjs.map +1 -0
- package/dist/apis/{test.api.d.ts → test.api.d.mts} +3 -3
- package/dist/apis/test.api.mjs +10 -0
- package/dist/apis/test.api.mjs.map +1 -0
- package/dist/apis/{urls.api.d.ts → urls.api.d.mts} +4 -4
- package/dist/apis/urls.api.mjs +21 -0
- package/dist/apis/urls.api.mjs.map +1 -0
- package/dist/apps/{cleanup.app.d.ts → cleanup.app.d.mts} +5 -5
- package/dist/apps/cleanup.app.mjs +129 -0
- package/dist/apps/cleanup.app.mjs.map +1 -0
- package/dist/apps/{cross-populate-export.app.d.ts → cross-populate-export.app.d.mts} +3 -3
- package/dist/apps/cross-populate-export.app.mjs +61 -0
- package/dist/apps/cross-populate-export.app.mjs.map +1 -0
- package/dist/apps/{cross-populate-import.app.d.ts → cross-populate-import.app.d.mts} +3 -3
- package/dist/apps/cross-populate-import.app.mjs +61 -0
- package/dist/apps/cross-populate-import.app.mjs.map +1 -0
- package/dist/apps/{denylist.app.d.ts → denylist.app.d.mts} +5 -5
- package/dist/apps/denylist.app.mjs +135 -0
- package/dist/apps/denylist.app.mjs.map +1 -0
- package/dist/apps/{expire.app.d.ts → expire.app.d.mts} +8 -7
- package/dist/apps/expire.app.mjs +51 -0
- package/dist/apps/expire.app.mjs.map +1 -0
- package/dist/apps/{export-domain-urls.d.ts → export-domain-urls.d.mts} +3 -3
- package/dist/apps/export-domain-urls.mjs +85 -0
- package/dist/apps/export-domain-urls.mjs.map +1 -0
- package/dist/apps/{extract-text.app.d.ts → extract-text.app.d.mts} +2 -2
- package/dist/apps/extract-text.app.mjs +43 -0
- package/dist/apps/extract-text.app.mjs.map +1 -0
- package/dist/apps/{hydra.app.d.ts → hydra.app.d.mts} +10 -10
- package/dist/apps/hydra.app.mjs +222 -0
- package/dist/apps/hydra.app.mjs.map +1 -0
- package/dist/apps/{import.app.d.ts → import.app.d.mts} +3 -3
- package/dist/apps/import.app.mjs +48 -0
- package/dist/apps/import.app.mjs.map +1 -0
- package/dist/apps/{internal-hydra-common.app.d.ts → internal-hydra-common.app.d.mts} +6 -6
- package/dist/apps/{internal-hydra-common.app.js → internal-hydra-common.app.mjs} +1 -1
- package/dist/apps/internal-hydra-common.app.mjs.map +1 -0
- package/dist/apps/{move-to-archive.app.d.ts → move-to-archive.app.d.mts} +2 -2
- package/dist/apps/move-to-archive.app.mjs +31 -0
- package/dist/apps/move-to-archive.app.mjs.map +1 -0
- package/dist/apps/{prune-archive.app.d.ts → prune-archive.app.d.mts} +2 -2
- package/dist/apps/prune-archive.app.mjs +40 -0
- package/dist/apps/prune-archive.app.mjs.map +1 -0
- package/dist/apps/{query.app.d.ts → query.app.d.mts} +6 -6
- package/dist/apps/query.app.mjs +69 -0
- package/dist/apps/query.app.mjs.map +1 -0
- package/dist/apps/{reattempt.app.d.ts → reattempt.app.d.mts} +6 -6
- package/dist/apps/reattempt.app.mjs +81 -0
- package/dist/apps/reattempt.app.mjs.map +1 -0
- package/dist/apps/{requeue-domain.app.d.ts → requeue-domain.app.d.mts} +3 -3
- package/dist/apps/requeue-domain.app.mjs +52 -0
- package/dist/apps/requeue-domain.app.mjs.map +1 -0
- package/dist/apps/{seed.app.d.ts → seed.app.d.mts} +5 -5
- package/dist/apps/seed.app.mjs +59 -0
- package/dist/apps/seed.app.mjs.map +1 -0
- package/dist/apps/{startup.app.d.ts → startup.app.d.mts} +3 -3
- package/dist/apps/startup.app.mjs +55 -0
- package/dist/apps/startup.app.mjs.map +1 -0
- package/dist/apps/{unarchive-urls.app.d.ts → unarchive-urls.app.d.mts} +5 -5
- package/dist/apps/unarchive-urls.app.mjs +78 -0
- package/dist/apps/unarchive-urls.app.mjs.map +1 -0
- package/dist/classes/{cleaner.d.ts → cleaner.d.mts} +2 -2
- package/dist/classes/cleaner.mjs +264 -0
- package/dist/classes/cleaner.mjs.map +1 -0
- package/dist/classes/{crawler.d.ts → crawler.d.mts} +10 -12
- package/dist/classes/{crawler.js → crawler.mjs} +216 -184
- package/dist/classes/crawler.mjs.map +1 -0
- package/dist/classes/{dns.js → dns.mjs} +4 -4
- package/dist/classes/dns.mjs.map +1 -0
- package/dist/classes/{expirer.d.ts → expirer.d.mts} +2 -2
- package/dist/classes/expirer.mjs +117 -0
- package/dist/classes/expirer.mjs.map +1 -0
- package/dist/classes/{expiry.d.ts → expiry.d.mts} +1 -1
- package/dist/classes/{expiry.js → expiry.mjs} +7 -9
- package/dist/classes/expiry.mjs.map +1 -0
- package/dist/classes/{lists.d.ts → lists.d.mts} +2 -2
- package/dist/classes/{lists.js → lists.mjs} +4 -4
- package/dist/classes/lists.mjs.map +1 -0
- package/dist/classes/{robot.d.ts → robot.d.mts} +3 -3
- package/dist/classes/robot.mjs +74 -0
- package/dist/classes/robot.mjs.map +1 -0
- package/dist/classes/{tracker.d.ts → tracker.d.mts} +3 -3
- package/dist/classes/tracker.mjs +101 -0
- package/dist/classes/tracker.mjs.map +1 -0
- package/dist/{cli.js → cli.mjs} +46 -58
- package/dist/cli.mjs.map +1 -0
- package/dist/enums/{eavailable-strategy.js → eavailable-strategy.mjs} +1 -1
- package/dist/enums/eavailable-strategy.mjs.map +1 -0
- package/dist/enums/{elist.js → elist.mjs} +1 -1
- package/dist/enums/elist.mjs.map +1 -0
- package/dist/enums/{eserver.js → eserver.mjs} +1 -1
- package/dist/enums/eserver.mjs.map +1 -0
- package/dist/enums/{ex-powered-by.js → ex-powered-by.mjs} +1 -1
- package/dist/enums/ex-powered-by.mjs.map +1 -0
- package/dist/helpers/{matcher.d.ts → matcher.d.mts} +1 -1
- package/dist/helpers/{matcher.js → matcher.mjs} +1 -1
- package/dist/helpers/matcher.mjs.map +1 -0
- package/dist/helpers/{random.d.ts → random.d.mts} +1 -1
- package/dist/helpers/{random.js → random.mjs} +1 -1
- package/dist/helpers/random.mjs.map +1 -0
- package/dist/helpers/{utf-decoder.d.ts → utf-decoder.d.mts} +0 -1
- package/dist/helpers/{utf-decoder.js → utf-decoder.mjs} +3 -3
- package/dist/helpers/utf-decoder.mjs.map +1 -0
- package/dist/interfaces/{iexpiry.d.ts → iexpiry.d.mts} +1 -1
- package/dist/interfaces/{iexpiry.js → iexpiry.mjs} +3 -3
- package/dist/interfaces/iexpiry.mjs.map +1 -0
- package/dist/interfaces/{imatch.js → imatch.mjs} +2 -2
- package/dist/interfaces/imatch.mjs.map +1 -0
- package/dist/interfaces/{iparser-config.js → iparser-config.mjs} +2 -2
- package/dist/interfaces/iparser-config.mjs.map +1 -0
- package/dist/interfaces/{iparser.d.ts → iparser.d.mts} +1 -1
- package/dist/interfaces/iparser.mjs +2 -0
- package/dist/interfaces/iparser.mjs.map +1 -0
- package/dist/interfaces/{irequest-outcome.d.ts → irequest-outcome.d.mts} +0 -2
- package/dist/interfaces/irequest-outcome.mjs +2 -0
- package/dist/interfaces/irequest-outcome.mjs.map +1 -0
- package/dist/interfaces/iserver.mjs +2 -0
- package/dist/interfaces/iserver.mjs.map +1 -0
- package/dist/parsers/accessibility-metrics.parser.d.mts +11 -0
- package/dist/parsers/accessibility-metrics.parser.mjs +30 -0
- package/dist/parsers/accessibility-metrics.parser.mjs.map +1 -0
- package/dist/parsers/asp-error.parser.d.mts +12 -0
- package/dist/parsers/asp-error.parser.mjs +38 -0
- package/dist/parsers/asp-error.parser.mjs.map +1 -0
- package/dist/parsers/{bad-words.parser.d.ts → bad-words.parser.d.mts} +6 -6
- package/dist/parsers/bad-words.parser.mjs +17 -0
- package/dist/parsers/bad-words.parser.mjs.map +1 -0
- package/dist/parsers/complex-english.parser.d.mts +15 -0
- package/dist/parsers/complex-english.parser.mjs +52 -0
- package/dist/parsers/complex-english.parser.mjs.map +1 -0
- package/dist/parsers/data.parser.d.mts +13 -0
- package/dist/parsers/{data.parser.js → data.parser.mjs} +8 -7
- package/dist/parsers/data.parser.mjs.map +1 -0
- package/dist/parsers/{dictionary.parser.d.ts → dictionary.parser.d.mts} +6 -6
- package/dist/parsers/dictionary.parser.mjs +63 -0
- package/dist/parsers/dictionary.parser.mjs.map +1 -0
- package/dist/parsers/html.parser.d.mts +13 -0
- package/dist/parsers/{html.parser.js → html.parser.mjs} +4 -3
- package/dist/parsers/html.parser.mjs.map +1 -0
- package/dist/parsers/hyperlinks.parser.d.mts +20 -0
- package/dist/parsers/hyperlinks.parser.mjs +104 -0
- package/dist/parsers/hyperlinks.parser.mjs.map +1 -0
- package/dist/parsers/image-tags.parser.d.mts +19 -0
- package/dist/parsers/image-tags.parser.mjs +42 -0
- package/dist/parsers/image-tags.parser.mjs.map +1 -0
- package/dist/parsers/{interest.parser.d.ts → interest.parser.d.mts} +7 -7
- package/dist/parsers/interest.parser.mjs +60 -0
- package/dist/parsers/interest.parser.mjs.map +1 -0
- package/dist/parsers/jpeg.parser.d.mts +11 -0
- package/dist/parsers/jpeg.parser.mjs +29 -0
- package/dist/parsers/jpeg.parser.mjs.map +1 -0
- package/dist/parsers/{llama-guard.parser.d.ts → llama-guard.parser.d.mts} +7 -7
- package/dist/parsers/llama-guard.parser.mjs +56 -0
- package/dist/parsers/llama-guard.parser.mjs.map +1 -0
- package/dist/parsers/{offence.parser.d.ts → offence.parser.d.mts} +7 -7
- package/dist/parsers/offence.parser.mjs +60 -0
- package/dist/parsers/offence.parser.mjs.map +1 -0
- package/dist/parsers/{ollama.parser.d.ts → ollama.parser.d.mts} +6 -6
- package/dist/parsers/ollama.parser.mjs +43 -0
- package/dist/parsers/ollama.parser.mjs.map +1 -0
- package/dist/parsers/{paragraphs.parser.d.ts → paragraphs.parser.d.mts} +5 -5
- package/dist/parsers/paragraphs.parser.mjs +38 -0
- package/dist/parsers/paragraphs.parser.mjs.map +1 -0
- package/dist/parsers/{parser.d.ts → parser.d.mts} +6 -6
- package/dist/parsers/parser.mjs +45 -0
- package/dist/parsers/parser.mjs.map +1 -0
- package/dist/parsers/php-error.parser.d.mts +12 -0
- package/dist/parsers/php-error.parser.mjs +42 -0
- package/dist/parsers/php-error.parser.mjs.map +1 -0
- package/dist/parsers/{phrase.parser.d.ts → phrase.parser.d.mts} +3 -3
- package/dist/parsers/phrase.parser.mjs +15 -0
- package/dist/parsers/phrase.parser.mjs.map +1 -0
- package/dist/parsers/{regex.parser.d.ts → regex.parser.d.mts} +3 -3
- package/dist/parsers/regex.parser.mjs +29 -0
- package/dist/parsers/regex.parser.mjs.map +1 -0
- package/dist/parsers/server.parser.d.mts +11 -0
- package/dist/parsers/server.parser.mjs +57 -0
- package/dist/parsers/server.parser.mjs.map +1 -0
- package/dist/parsers/{spelling.parser.d.ts → spelling.parser.d.mts} +6 -6
- package/dist/parsers/spelling.parser.mjs +17 -0
- package/dist/parsers/spelling.parser.mjs.map +1 -0
- package/dist/parsers/string.parser.d.mts +8 -0
- package/dist/parsers/{string.parser.js → string.parser.mjs} +5 -4
- package/dist/parsers/string.parser.mjs.map +1 -0
- package/dist/parsers/{text.parser.d.ts → text.parser.d.mts} +3 -3
- package/dist/parsers/text.parser.mjs +30 -0
- package/dist/parsers/text.parser.mjs.map +1 -0
- package/dist/parsers/{words.parser.d.ts → words.parser.d.mts} +3 -3
- package/dist/parsers/words.parser.mjs +29 -0
- package/dist/parsers/words.parser.mjs.map +1 -0
- package/dist/queries/complex-english.query.d.mts +2 -0
- package/dist/queries/{complex-english.query.js → complex-english.query.mjs} +26 -28
- package/dist/queries/complex-english.query.mjs.map +1 -0
- package/dist/queries/flash-content.query.d.mts +2 -0
- package/dist/queries/flash-content.query.mjs +80 -0
- package/dist/queries/flash-content.query.mjs.map +1 -0
- package/dist/queries/linking-to-domains.query.d.mts +2 -0
- package/dist/queries/linking-to-domains.query.mjs +128 -0
- package/dist/queries/linking-to-domains.query.mjs.map +1 -0
- package/dist/queries/llamaguard-unsafe-content.query.d.mts +2 -0
- package/dist/queries/llamaguard-unsafe-content.query.mjs +90 -0
- package/dist/queries/llamaguard-unsafe-content.query.mjs.map +1 -0
- package/dist/queries/readability-score.query.d.mts +2 -0
- package/dist/queries/{readability-score.query.js → readability-score.query.mjs} +11 -20
- package/dist/queries/readability-score.query.mjs.map +1 -0
- package/dist/servers/{crawl.server.d.ts → crawl.server.d.mts} +10 -10
- package/dist/servers/crawl.server.mjs +192 -0
- package/dist/servers/crawl.server.mjs.map +1 -0
- package/dist/servers/{express.server.d.ts → express.server.d.mts} +2 -3
- package/dist/servers/express.server.mjs +13 -0
- package/dist/servers/express.server.mjs.map +1 -0
- package/dist/servers/{maintenance.server.d.ts → maintenance.server.d.mts} +5 -5
- package/dist/servers/maintenance.server.mjs +97 -0
- package/dist/servers/maintenance.server.mjs.map +1 -0
- package/dist/servers/{rest.server.d.ts → rest.server.d.mts} +4 -4
- package/dist/servers/rest.server.mjs +62 -0
- package/dist/servers/rest.server.mjs.map +1 -0
- package/dist/servers/{socket-io.server.d.ts → socket-io.server.d.mts} +4 -4
- package/dist/servers/socket-io.server.mjs +22 -0
- package/dist/servers/socket-io.server.mjs.map +1 -0
- package/dist/services/{database.service.d.ts → database.service.d.mts} +12 -12
- package/dist/services/database.service.mjs +645 -0
- package/dist/services/database.service.mjs.map +1 -0
- package/dist/services/{ollama-rest.service.d.ts → ollama-rest.service.d.mts} +2 -2
- package/dist/services/ollama-rest.service.mjs +27 -0
- package/dist/services/ollama-rest.service.mjs.map +1 -0
- package/dist/services/{rig-llama-guard.service.d.ts → rig-llama-guard.service.d.mts} +2 -2
- package/dist/services/{rig-llama-guard.service.js → rig-llama-guard.service.mjs} +33 -43
- package/dist/services/rig-llama-guard.service.mjs.map +1 -0
- package/dist/services/{rig-queue-length.service.d.ts → rig-queue-length.service.d.mts} +1 -1
- package/dist/services/rig-queue-length.service.mjs +12 -0
- package/dist/services/rig-queue-length.service.mjs.map +1 -0
- package/dist/services/{rig-subjectivity-scale.service.d.ts → rig-subjectivity-scale.service.d.mts} +2 -2
- package/dist/services/rig-subjectivity-scale.service.mjs +82 -0
- package/dist/services/rig-subjectivity-scale.service.mjs.map +1 -0
- package/dist/services/{rig-ticket-rest.service.d.ts → rig-ticket-rest.service.d.mts} +3 -3
- package/dist/services/rig-ticket-rest.service.mjs +15 -0
- package/dist/services/rig-ticket-rest.service.mjs.map +1 -0
- package/dist/services/{rig-ticketed-promise.service.d.ts → rig-ticketed-promise.service.d.mts} +4 -6
- package/dist/services/{rig-ticketed-promise.service.js → rig-ticketed-promise.service.mjs} +7 -6
- package/dist/services/rig-ticketed-promise.service.mjs.map +1 -0
- package/dist/types/{tcrawl-config.js → tcrawl-config.mjs} +2 -2
- package/dist/types/tcrawl-config.mjs.map +1 -0
- package/dist/types/{thydra-config.js → thydra-config.mjs} +2 -2
- package/dist/types/thydra-config.mjs.map +1 -0
- package/dist/types/tparser-ctor.d.mts +7 -0
- package/dist/types/tparser-ctor.mjs +2 -0
- package/dist/types/tparser-ctor.mjs.map +1 -0
- package/dist/types/tquery.d.mts +7 -0
- package/dist/types/tquery.mjs +2 -0
- package/dist/types/tquery.mjs.map +1 -0
- package/dist/types/tqueue-length.mjs +2 -0
- package/dist/types/tqueue-length.mjs.map +1 -0
- package/dist/types/{trobots-config.js → trobots-config.mjs} +2 -2
- package/dist/types/trobots-config.mjs.map +1 -0
- package/package.json +37 -43
- package/dist/apis/autocomplete.api.js +0 -22
- package/dist/apis/autocomplete.api.js.map +0 -1
- package/dist/apis/bugs.api.js +0 -26
- package/dist/apis/bugs.api.js.map +0 -1
- package/dist/apis/crawl.api.js +0 -25
- package/dist/apis/crawl.api.js.map +0 -1
- package/dist/apis/domains.api.js.map +0 -1
- package/dist/apis/images.api.js +0 -27
- package/dist/apis/images.api.js.map +0 -1
- package/dist/apis/statistics.api.js +0 -41
- package/dist/apis/statistics.api.js.map +0 -1
- package/dist/apis/test.api.js +0 -19
- package/dist/apis/test.api.js.map +0 -1
- package/dist/apis/urls.api.js +0 -29
- package/dist/apis/urls.api.js.map +0 -1
- package/dist/apps/cleanup.app.js +0 -151
- package/dist/apps/cleanup.app.js.map +0 -1
- package/dist/apps/cross-populate-export.app.js +0 -75
- package/dist/apps/cross-populate-export.app.js.map +0 -1
- package/dist/apps/cross-populate-import.app.js +0 -100
- package/dist/apps/cross-populate-import.app.js.map +0 -1
- package/dist/apps/denylist.app.js +0 -132
- package/dist/apps/denylist.app.js.map +0 -1
- package/dist/apps/expire.app.js +0 -63
- package/dist/apps/expire.app.js.map +0 -1
- package/dist/apps/export-domain-urls.js +0 -99
- package/dist/apps/export-domain-urls.js.map +0 -1
- package/dist/apps/extract-text.app.js +0 -55
- package/dist/apps/extract-text.app.js.map +0 -1
- package/dist/apps/hydra.app.js +0 -218
- package/dist/apps/hydra.app.js.map +0 -1
- package/dist/apps/import.app.js +0 -57
- package/dist/apps/import.app.js.map +0 -1
- package/dist/apps/internal-hydra-common.app.js.map +0 -1
- package/dist/apps/move-to-archive.app.js +0 -46
- package/dist/apps/move-to-archive.app.js.map +0 -1
- package/dist/apps/prune-archive.app.js +0 -55
- package/dist/apps/prune-archive.app.js.map +0 -1
- package/dist/apps/query.app.js +0 -80
- package/dist/apps/query.app.js.map +0 -1
- package/dist/apps/reattempt.app.js +0 -83
- package/dist/apps/reattempt.app.js.map +0 -1
- package/dist/apps/requeue-domain.app.js +0 -64
- package/dist/apps/requeue-domain.app.js.map +0 -1
- package/dist/apps/seed.app.js +0 -69
- package/dist/apps/seed.app.js.map +0 -1
- package/dist/apps/startup.app.js +0 -64
- package/dist/apps/startup.app.js.map +0 -1
- package/dist/apps/unarchive-urls.app.js +0 -83
- package/dist/apps/unarchive-urls.app.js.map +0 -1
- package/dist/classes/cleaner.js +0 -266
- package/dist/classes/cleaner.js.map +0 -1
- package/dist/classes/crawler.js.map +0 -1
- package/dist/classes/dns.js.map +0 -1
- package/dist/classes/expirer.js +0 -121
- package/dist/classes/expirer.js.map +0 -1
- package/dist/classes/expiry.js.map +0 -1
- package/dist/classes/lists.js.map +0 -1
- package/dist/classes/robot.js +0 -82
- package/dist/classes/robot.js.map +0 -1
- package/dist/classes/tracker.js +0 -120
- package/dist/classes/tracker.js.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/enums/eavailable-strategy.js.map +0 -1
- package/dist/enums/elist.js.map +0 -1
- package/dist/enums/eserver.js.map +0 -1
- package/dist/enums/ex-powered-by.js.map +0 -1
- package/dist/helpers/matcher.js.map +0 -1
- package/dist/helpers/random.js.map +0 -1
- package/dist/helpers/utf-decoder.js.map +0 -1
- package/dist/interfaces/iexpiry.js.map +0 -1
- package/dist/interfaces/imatch.js.map +0 -1
- package/dist/interfaces/iparser-config.js.map +0 -1
- package/dist/interfaces/iparser.js +0 -2
- package/dist/interfaces/iparser.js.map +0 -1
- package/dist/interfaces/irequest-outcome.js +0 -2
- package/dist/interfaces/irequest-outcome.js.map +0 -1
- package/dist/interfaces/iserver.js +0 -2
- package/dist/interfaces/iserver.js.map +0 -1
- package/dist/parsers/accessibility-metrics.parser.d.ts +0 -11
- package/dist/parsers/accessibility-metrics.parser.js +0 -40
- package/dist/parsers/accessibility-metrics.parser.js.map +0 -1
- package/dist/parsers/asp-error.parser.d.ts +0 -12
- package/dist/parsers/asp-error.parser.js +0 -48
- package/dist/parsers/asp-error.parser.js.map +0 -1
- package/dist/parsers/bad-words.parser.js +0 -27
- package/dist/parsers/bad-words.parser.js.map +0 -1
- package/dist/parsers/complex-english.parser.d.ts +0 -15
- package/dist/parsers/complex-english.parser.js +0 -61
- package/dist/parsers/complex-english.parser.js.map +0 -1
- package/dist/parsers/data.parser.d.ts +0 -14
- package/dist/parsers/data.parser.js.map +0 -1
- package/dist/parsers/dictionary.parser.js +0 -73
- package/dist/parsers/dictionary.parser.js.map +0 -1
- package/dist/parsers/html.parser.d.ts +0 -13
- package/dist/parsers/html.parser.js.map +0 -1
- package/dist/parsers/hyperlinks.parser.d.ts +0 -20
- package/dist/parsers/hyperlinks.parser.js +0 -115
- package/dist/parsers/hyperlinks.parser.js.map +0 -1
- package/dist/parsers/image-tags.parser.d.ts +0 -19
- package/dist/parsers/image-tags.parser.js +0 -52
- package/dist/parsers/image-tags.parser.js.map +0 -1
- package/dist/parsers/interest.parser.js +0 -69
- package/dist/parsers/interest.parser.js.map +0 -1
- package/dist/parsers/jpeg.parser.d.ts +0 -11
- package/dist/parsers/jpeg.parser.js +0 -39
- package/dist/parsers/jpeg.parser.js.map +0 -1
- package/dist/parsers/llama-guard.parser.js +0 -65
- package/dist/parsers/llama-guard.parser.js.map +0 -1
- package/dist/parsers/offence.parser.js +0 -69
- package/dist/parsers/offence.parser.js.map +0 -1
- package/dist/parsers/ollama.parser.js +0 -51
- package/dist/parsers/ollama.parser.js.map +0 -1
- package/dist/parsers/paragraphs.parser.js +0 -49
- package/dist/parsers/paragraphs.parser.js.map +0 -1
- package/dist/parsers/parser.js +0 -57
- package/dist/parsers/parser.js.map +0 -1
- package/dist/parsers/php-error.parser.d.ts +0 -12
- package/dist/parsers/php-error.parser.js +0 -52
- package/dist/parsers/php-error.parser.js.map +0 -1
- package/dist/parsers/phrase.parser.js +0 -26
- package/dist/parsers/phrase.parser.js.map +0 -1
- package/dist/parsers/regex.parser.js +0 -43
- package/dist/parsers/regex.parser.js.map +0 -1
- package/dist/parsers/server.parser.d.ts +0 -11
- package/dist/parsers/server.parser.js +0 -67
- package/dist/parsers/server.parser.js.map +0 -1
- package/dist/parsers/spelling.parser.js +0 -27
- package/dist/parsers/spelling.parser.js.map +0 -1
- package/dist/parsers/string.parser.d.ts +0 -8
- package/dist/parsers/string.parser.js.map +0 -1
- package/dist/parsers/text.parser.js +0 -41
- package/dist/parsers/text.parser.js.map +0 -1
- package/dist/parsers/words.parser.js +0 -40
- package/dist/parsers/words.parser.js.map +0 -1
- package/dist/queries/complex-english.query.d.ts +0 -2
- package/dist/queries/complex-english.query.js.map +0 -1
- package/dist/queries/flash-content.query.d.ts +0 -2
- package/dist/queries/flash-content.query.js +0 -82
- package/dist/queries/flash-content.query.js.map +0 -1
- package/dist/queries/linking-to-domains.query.d.ts +0 -2
- package/dist/queries/linking-to-domains.query.js +0 -130
- package/dist/queries/linking-to-domains.query.js.map +0 -1
- package/dist/queries/llamaguard-unsafe-content.query.d.ts +0 -2
- package/dist/queries/llamaguard-unsafe-content.query.js +0 -92
- package/dist/queries/llamaguard-unsafe-content.query.js.map +0 -1
- package/dist/queries/readability-score.query.d.ts +0 -2
- package/dist/queries/readability-score.query.js.map +0 -1
- package/dist/servers/crawl.server.js +0 -198
- package/dist/servers/crawl.server.js.map +0 -1
- package/dist/servers/express.server.js +0 -13
- package/dist/servers/express.server.js.map +0 -1
- package/dist/servers/maintenance.server.js +0 -103
- package/dist/servers/maintenance.server.js.map +0 -1
- package/dist/servers/rest.server.js +0 -46
- package/dist/servers/rest.server.js.map +0 -1
- package/dist/servers/socket-io.server.js +0 -61
- package/dist/servers/socket-io.server.js.map +0 -1
- package/dist/services/database.service.js +0 -795
- package/dist/services/database.service.js.map +0 -1
- package/dist/services/ollama-rest.service.js +0 -46
- package/dist/services/ollama-rest.service.js.map +0 -1
- package/dist/services/rig-llama-guard.service.js.map +0 -1
- package/dist/services/rig-queue-length.service.js +0 -22
- package/dist/services/rig-queue-length.service.js.map +0 -1
- package/dist/services/rig-subjectivity-scale.service.js +0 -96
- package/dist/services/rig-subjectivity-scale.service.js.map +0 -1
- package/dist/services/rig-ticket-rest.service.js +0 -29
- package/dist/services/rig-ticket-rest.service.js.map +0 -1
- package/dist/services/rig-ticketed-promise.service.js.map +0 -1
- package/dist/types/tcrawl-config.js.map +0 -1
- package/dist/types/thydra-config.js.map +0 -1
- package/dist/types/tparser-ctor.d.ts +0 -7
- package/dist/types/tparser-ctor.js +0 -2
- package/dist/types/tparser-ctor.js.map +0 -1
- package/dist/types/tquery.d.ts +0 -7
- package/dist/types/tquery.js +0 -2
- package/dist/types/tquery.js.map +0 -1
- package/dist/types/tqueue-length.js +0 -2
- package/dist/types/tqueue-length.js.map +0 -1
- package/dist/types/trobots-config.js.map +0 -1
- /package/dist/classes/{dns.d.ts → dns.d.mts} +0 -0
- /package/dist/{cli.d.ts → cli.d.mts} +0 -0
- /package/dist/enums/{eavailable-strategy.d.ts → eavailable-strategy.d.mts} +0 -0
- /package/dist/enums/{elist.d.ts → elist.d.mts} +0 -0
- /package/dist/enums/{eserver.d.ts → eserver.d.mts} +0 -0
- /package/dist/enums/{ex-powered-by.d.ts → ex-powered-by.d.mts} +0 -0
- /package/dist/interfaces/{imatch.d.ts → imatch.d.mts} +0 -0
- /package/dist/interfaces/{iparser-config.d.ts → iparser-config.d.mts} +0 -0
- /package/dist/interfaces/{iserver.d.ts → iserver.d.mts} +0 -0
- /package/dist/types/{tcrawl-config.d.ts → tcrawl-config.d.mts} +0 -0
- /package/dist/types/{thydra-config.d.ts → thydra-config.d.mts} +0 -0
- /package/dist/types/{tqueue-length.d.ts → tqueue-length.d.mts} +0 -0
- /package/dist/types/{trobots-config.d.ts → trobots-config.d.mts} +0 -0
|
@@ -1,26 +1,25 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
1
|
import { URL } from 'url';
|
|
11
2
|
import * as http from 'http';
|
|
12
3
|
import * as https from 'https';
|
|
13
|
-
import { commonsMapToObject, commonsTypeHasPropertyString } from 'tscommons-
|
|
14
|
-
import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-
|
|
15
|
-
import { EStatus } from 'hydra-crawler-
|
|
16
|
-
import {
|
|
17
|
-
import { commonsGracefulAbortAddCallback, commonsGracefulAbortRemoveCallback } from 'nodecommons-
|
|
18
|
-
import { commonsHashMd5 } from 'nodecommons-
|
|
19
|
-
import { Robot } from '../classes/robot';
|
|
20
|
-
import { Random } from '../helpers/random';
|
|
21
|
-
import { EList } from '../enums/elist';
|
|
22
|
-
import { Dns } from './dns';
|
|
4
|
+
import { commonsMapToObject, commonsTypeHasPropertyString } from 'tscommons-esm-core';
|
|
5
|
+
import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-esm-async';
|
|
6
|
+
import { EStatus } from 'hydra-crawler-esm-assets';
|
|
7
|
+
import { commonsLogAlert, commonsLogDebug, commonsLogError, commonsLogInfo } from 'nodecommons-esm-log';
|
|
8
|
+
import { commonsGracefulAbortAddCallback, commonsGracefulAbortRemoveCallback } from 'nodecommons-esm-process';
|
|
9
|
+
import { commonsHashMd5 } from 'nodecommons-esm-security';
|
|
10
|
+
import { Robot } from '../classes/robot.mjs';
|
|
11
|
+
import { Random } from '../helpers/random.mjs';
|
|
12
|
+
import { EList } from '../enums/elist.mjs';
|
|
13
|
+
import { Dns } from './dns.mjs';
|
|
23
14
|
export class Crawler {
|
|
15
|
+
domain;
|
|
16
|
+
database;
|
|
17
|
+
crawlConfig;
|
|
18
|
+
parsersConfig;
|
|
19
|
+
robotsConfig;
|
|
20
|
+
parsers;
|
|
21
|
+
lists;
|
|
22
|
+
tracker;
|
|
24
23
|
static applyMasqueradeHeaders(request) {
|
|
25
24
|
request.setHeader('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8');
|
|
26
25
|
request.setHeader('Accept-Encoding', 'identity');
|
|
@@ -39,7 +38,7 @@ export class Crawler {
|
|
|
39
38
|
.then(() => {
|
|
40
39
|
if (timedOut || completed)
|
|
41
40
|
return;
|
|
42
|
-
|
|
41
|
+
commonsLogDebug(`Secondary timeout for ${url}`);
|
|
43
42
|
timedOut = true;
|
|
44
43
|
reject(new Error(timedOutMessage));
|
|
45
44
|
return;
|
|
@@ -47,7 +46,7 @@ export class Crawler {
|
|
|
47
46
|
.catch((e) => {
|
|
48
47
|
if (e.message === 'abortTimeout called')
|
|
49
48
|
return;
|
|
50
|
-
|
|
49
|
+
commonsLogDebug('debug position 10');
|
|
51
50
|
console.log(e);
|
|
52
51
|
throw e;
|
|
53
52
|
});
|
|
@@ -65,7 +64,7 @@ export class Crawler {
|
|
|
65
64
|
if (!exceeded) {
|
|
66
65
|
exceeded = true;
|
|
67
66
|
res.destroy();
|
|
68
|
-
|
|
67
|
+
commonsLogAlert(`Exceeded maxFileSize for URL: ${url}`);
|
|
69
68
|
}
|
|
70
69
|
return;
|
|
71
70
|
}
|
|
@@ -158,6 +157,8 @@ export class Crawler {
|
|
|
158
157
|
}
|
|
159
158
|
return map;
|
|
160
159
|
}
|
|
160
|
+
isPaused = false;
|
|
161
|
+
isAborted = false;
|
|
161
162
|
constructor(domain, database, crawlConfig, parsersConfig, robotsConfig, parsers, lists, tracker) {
|
|
162
163
|
this.domain = domain;
|
|
163
164
|
this.database = database;
|
|
@@ -167,55 +168,54 @@ export class Crawler {
|
|
|
167
168
|
this.parsers = parsers;
|
|
168
169
|
this.lists = lists;
|
|
169
170
|
this.tracker = tracker;
|
|
170
|
-
this.isPaused = false;
|
|
171
|
-
this.isAborted = false;
|
|
172
171
|
}
|
|
173
172
|
abort() {
|
|
174
173
|
this.isAborted = true;
|
|
175
174
|
commonsAsyncAbortTimeout(`crawl_${this.domain}`);
|
|
176
175
|
}
|
|
177
176
|
pause() {
|
|
178
|
-
|
|
177
|
+
commonsLogAlert(`Pausing crawler for ${this.domain}`);
|
|
179
178
|
this.isPaused = true;
|
|
180
179
|
}
|
|
181
180
|
resume() {
|
|
182
|
-
|
|
181
|
+
commonsLogInfo(`Resuming crawler for ${this.domain}`);
|
|
183
182
|
this.isPaused = false;
|
|
184
183
|
}
|
|
185
|
-
fetch(url, setDomainIp) {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
184
|
+
async fetch(url, setDomainIp) {
|
|
185
|
+
commonsLogInfo(`Fetching ${url}`);
|
|
186
|
+
const whatwg = new URL(url);
|
|
187
|
+
let handler;
|
|
188
|
+
switch (whatwg.protocol) {
|
|
189
|
+
case 'http:':
|
|
190
|
+
handler = http;
|
|
191
|
+
break;
|
|
192
|
+
case 'https:':
|
|
193
|
+
handler = https;
|
|
194
|
+
break;
|
|
195
|
+
default:
|
|
196
|
+
throw new Error(`unable to handle protocol ${whatwg.protocol}`);
|
|
197
|
+
}
|
|
198
|
+
let outcome;
|
|
199
|
+
// eslint-disable-next-line no-useless-catch
|
|
200
|
+
try {
|
|
201
|
+
outcome = await Crawler.request(handler, url, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
|
|
202
|
+
}
|
|
203
|
+
catch (ex) {
|
|
204
|
+
//if (!/^Timeout on crawl of/.test(ex.message)) console.error(ex);
|
|
205
|
+
throw ex;
|
|
206
|
+
}
|
|
207
|
+
if (setDomainIp) {
|
|
208
|
+
await this.database.domain(whatwg.hostname, outcome.ip);
|
|
209
|
+
if (this.tracker)
|
|
210
|
+
this.tracker.domain();
|
|
211
|
+
}
|
|
212
|
+
await this.database.setStatusCode(url, outcome.statusCode);
|
|
213
|
+
const headers = Crawler.pruneHeaders(outcome.headers, this.crawlConfig.keepHeaders);
|
|
214
|
+
if (!headers.has('content-length') && outcome.data !== undefined)
|
|
215
|
+
headers.set('content-length', outcome.data.length);
|
|
216
|
+
await this.database.setHeaders(url, commonsMapToObject(headers));
|
|
217
|
+
const outgoing = [];
|
|
218
|
+
try {
|
|
219
219
|
if (outcome.statusCode >= 300
|
|
220
220
|
&& outcome.statusCode < 400
|
|
221
221
|
&& headers.has('location')
|
|
@@ -224,22 +224,38 @@ export class Crawler {
|
|
|
224
224
|
// redirect codes
|
|
225
225
|
const location = new URL(headers.get('location'), url);
|
|
226
226
|
if (location.protocol.match(/^http(s?):$/)) {
|
|
227
|
-
const added =
|
|
227
|
+
const added = await this.database.queue(location.toString(), this.lists.match(EList.DENY, location.toString()));
|
|
228
228
|
if (added && this.tracker)
|
|
229
229
|
this.tracker.delta(this.lists.match(EList.DENY, location.toString()) ? EStatus.DENY : EStatus.QUEUED, 1);
|
|
230
230
|
if (location.toString() !== url)
|
|
231
231
|
outgoing.push(location.toString());
|
|
232
232
|
}
|
|
233
233
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
234
|
+
}
|
|
235
|
+
catch (_e) {
|
|
236
|
+
// probably an invalid URL
|
|
237
|
+
// so ignore
|
|
238
|
+
}
|
|
239
|
+
if (outcome.data) {
|
|
240
|
+
const hash = commonsHashMd5(outcome.data);
|
|
241
|
+
const existing = await this.database.getHash(url);
|
|
242
|
+
if (hash !== existing)
|
|
243
|
+
await this.database.setHash(url, hash);
|
|
244
|
+
}
|
|
245
|
+
if (outcome.data && outcome.data.length && headers.has('content-type')) {
|
|
246
|
+
const contentType = headers.get('content-type');
|
|
247
|
+
let isAllow;
|
|
248
|
+
let isValid = true;
|
|
249
|
+
try {
|
|
250
|
+
isAllow = this.lists.match(EList.ALLOW, url);
|
|
251
|
+
}
|
|
252
|
+
catch (_e) {
|
|
253
|
+
// probably an invalid URL
|
|
254
|
+
// so ignore
|
|
255
|
+
isValid = false;
|
|
256
|
+
isAllow = false; // NA
|
|
239
257
|
}
|
|
240
|
-
if (
|
|
241
|
-
const contentType = headers.get('content-type');
|
|
242
|
-
const isAllow = this.lists.match(EList.ALLOW, url);
|
|
258
|
+
if (isValid) {
|
|
243
259
|
for (const ctor of this.parsers) {
|
|
244
260
|
const parser = new ctor(url, outcome, this.parsersConfig);
|
|
245
261
|
if (!parser.isEnabled())
|
|
@@ -247,15 +263,15 @@ export class Crawler {
|
|
|
247
263
|
if (!parser.supports(contentType, isAllow))
|
|
248
264
|
continue;
|
|
249
265
|
try {
|
|
250
|
-
|
|
266
|
+
await parser.parse(this.database);
|
|
251
267
|
}
|
|
252
268
|
catch (ex) {
|
|
253
|
-
|
|
269
|
+
commonsLogDebug('debug position 3');
|
|
254
270
|
console.log(ex);
|
|
255
271
|
}
|
|
256
272
|
if (isAllow) {
|
|
257
273
|
try {
|
|
258
|
-
const links =
|
|
274
|
+
const links = await parser.links();
|
|
259
275
|
if (links.length === 0)
|
|
260
276
|
continue;
|
|
261
277
|
for (const link of links) {
|
|
@@ -264,159 +280,175 @@ export class Crawler {
|
|
|
264
280
|
try {
|
|
265
281
|
outgoing.push(link);
|
|
266
282
|
}
|
|
267
|
-
catch (
|
|
283
|
+
catch (_ex) { /* do nothing */ }
|
|
268
284
|
}
|
|
269
285
|
}
|
|
270
286
|
catch (ex) {
|
|
271
|
-
|
|
287
|
+
commonsLogDebug('debug position 2');
|
|
272
288
|
console.log(ex);
|
|
273
289
|
}
|
|
274
290
|
}
|
|
275
291
|
}
|
|
276
292
|
}
|
|
277
|
-
|
|
278
|
-
|
|
293
|
+
}
|
|
294
|
+
for (const link of outgoing) {
|
|
295
|
+
try {
|
|
296
|
+
const added = await this.database.queue(link, this.lists.match(EList.DENY, link));
|
|
279
297
|
if (added && this.tracker)
|
|
280
298
|
this.tracker.delta(this.lists.match(EList.DENY, link) ? EStatus.DENY : EStatus.QUEUED, 1);
|
|
281
299
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
300
|
+
catch (_e) {
|
|
301
|
+
// probably an invalid URL
|
|
302
|
+
// so ignore
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
await this.database.link(url, outgoing);
|
|
306
|
+
if (this.tracker)
|
|
307
|
+
this.tracker.link(outgoing.length);
|
|
308
|
+
return outcome.statusCode;
|
|
287
309
|
}
|
|
288
|
-
crawl() {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
310
|
+
async crawl() {
|
|
311
|
+
const gracefulAbortId = commonsGracefulAbortAddCallback(() => {
|
|
312
|
+
commonsLogAlert(`SIGINT abort flag is set. Aborting crawler ${this.domain}.`);
|
|
313
|
+
this.abort();
|
|
314
|
+
});
|
|
315
|
+
try {
|
|
294
316
|
try {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
commonsOutputError(`DNS timed out for ${this.domain}. Will allow reattempts`);
|
|
301
|
-
return;
|
|
302
|
-
}
|
|
303
|
-
commonsOutputError(`Domain ${this.domain} is dead.`);
|
|
304
|
-
yield this.database.markDead(this.domain);
|
|
317
|
+
await Dns.resolve(this.domain, this.crawlConfig.connectTimeout);
|
|
318
|
+
}
|
|
319
|
+
catch (ex) {
|
|
320
|
+
if (ex.message === 'DNS timeout' && !this.crawlConfig.treatTimeoutDnsAsDead) {
|
|
321
|
+
commonsLogError(`DNS timed out for ${this.domain}. Will allow reattempts`);
|
|
305
322
|
return;
|
|
306
323
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
324
|
+
commonsLogError(`Domain ${this.domain} is dead.`);
|
|
325
|
+
await this.database.markDead(this.domain);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
const robots = new Robot(this.domain, this.crawlConfig, this.robotsConfig, this.tracker);
|
|
329
|
+
await robots.load();
|
|
330
|
+
let tally = 0;
|
|
331
|
+
while (true) {
|
|
332
|
+
if (this.isAborted)
|
|
333
|
+
break;
|
|
334
|
+
if (this.isPaused) {
|
|
335
|
+
await commonsAsyncTimeout(1000);
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
const url = await this.database.next(this.domain);
|
|
339
|
+
if (!url) {
|
|
340
|
+
commonsLogInfo(`Crawl for ${this.domain} completed.`);
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
await this.database.unsetFailReason(url);
|
|
344
|
+
if (this.tracker)
|
|
345
|
+
this.tracker.delta(EStatus.QUEUED, -1);
|
|
346
|
+
try {
|
|
325
347
|
if (this.lists.match(EList.DENY, url)) {
|
|
326
|
-
|
|
327
|
-
|
|
348
|
+
await this.database.setStatus(url, EStatus.DENY);
|
|
349
|
+
await this.database.unsetTtl(url);
|
|
328
350
|
if (this.tracker)
|
|
329
351
|
this.tracker.delta(EStatus.DENY, 1);
|
|
330
352
|
// To get here, a url was queued but is now denylisted.
|
|
331
353
|
// It may have had outgoings, so remove any that were there.
|
|
332
|
-
|
|
354
|
+
await this.database.link(url, []);
|
|
333
355
|
continue;
|
|
334
356
|
}
|
|
357
|
+
}
|
|
358
|
+
catch (_e) {
|
|
359
|
+
// probably an invalid URL
|
|
360
|
+
// so ignore
|
|
361
|
+
}
|
|
362
|
+
try {
|
|
335
363
|
if (robots.isDisallowed(url)) {
|
|
336
|
-
|
|
337
|
-
|
|
364
|
+
await this.database.setStatus(url, EStatus.DISALLOWED);
|
|
365
|
+
await this.database.unsetTtl(url);
|
|
338
366
|
if (this.tracker)
|
|
339
367
|
this.tracker.delta(EStatus.DISALLOWED, 1);
|
|
340
368
|
// To get here, a url was queued but is now disallowed.
|
|
341
369
|
// It may have had outgoings, so remove any that were there.
|
|
342
|
-
|
|
370
|
+
await this.database.link(url, []);
|
|
343
371
|
continue;
|
|
344
372
|
}
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
373
|
+
}
|
|
374
|
+
catch (_e) {
|
|
375
|
+
// probably an invalid URL
|
|
376
|
+
// so ignore
|
|
377
|
+
}
|
|
378
|
+
if (!await this.database.setStatus(url, EStatus.ACTIVE)) {
|
|
379
|
+
commonsLogError(`Unable to set status to active for ${url}. Aborting crawler.`);
|
|
380
|
+
break;
|
|
381
|
+
}
|
|
382
|
+
if (this.tracker)
|
|
383
|
+
this.tracker.delta(EStatus.ACTIVE, 1);
|
|
384
|
+
if (tally > 0) {
|
|
385
|
+
try {
|
|
386
|
+
const timeout = Random.randomise(this.crawlConfig.betweenFetchDelay, this.crawlConfig);
|
|
387
|
+
await commonsAsyncTimeout(timeout, `crawl_${this.domain}`);
|
|
388
|
+
while (this.isPaused && !this.isAborted) {
|
|
389
|
+
await commonsAsyncTimeout(1000);
|
|
358
390
|
}
|
|
359
|
-
catch (ex) { /* ignore */ }
|
|
360
391
|
}
|
|
361
|
-
|
|
362
|
-
|
|
392
|
+
catch (_ex) { /* ignore */ }
|
|
393
|
+
}
|
|
394
|
+
if (this.isAborted)
|
|
395
|
+
break;
|
|
396
|
+
if (this.tracker)
|
|
397
|
+
await this.tracker.fetching(url);
|
|
398
|
+
try {
|
|
399
|
+
const statusCode = await this.fetch(url, tally === 0);
|
|
363
400
|
if (this.tracker)
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
401
|
+
await this.tracker.done(url, statusCode);
|
|
402
|
+
await this.database.setStatus(url, EStatus.DONE);
|
|
403
|
+
await this.database.unsetTtl(url);
|
|
404
|
+
if (this.tracker)
|
|
405
|
+
this.tracker.delta(EStatus.DONE, 1);
|
|
406
|
+
}
|
|
407
|
+
catch (ex) {
|
|
408
|
+
if (this.tracker)
|
|
409
|
+
await this.tracker.failed(url);
|
|
410
|
+
const ttl = await this.database.getTtl(url) || 0;
|
|
411
|
+
if (ttl < this.crawlConfig.maxFailedTtl) {
|
|
412
|
+
await this.database.setTtl(url, ttl + 1);
|
|
413
|
+
await this.database.setStatus(url, EStatus.QUEUED);
|
|
371
414
|
if (this.tracker)
|
|
372
|
-
this.tracker.delta(EStatus.
|
|
415
|
+
this.tracker.delta(EStatus.QUEUED, 1);
|
|
373
416
|
}
|
|
374
|
-
|
|
417
|
+
else {
|
|
418
|
+
await this.database.setStatus(url, EStatus.FAILED);
|
|
375
419
|
if (this.tracker)
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
yield this.database.setTtl(url, ttl + 1);
|
|
380
|
-
yield this.database.setStatus(url, EStatus.QUEUED);
|
|
381
|
-
if (this.tracker)
|
|
382
|
-
this.tracker.delta(EStatus.QUEUED, 1);
|
|
420
|
+
this.tracker.delta(EStatus.FAILED, 1);
|
|
421
|
+
if (commonsTypeHasPropertyString(ex, 'code')) {
|
|
422
|
+
await this.database.setFailReason(url, ex.code);
|
|
383
423
|
}
|
|
384
|
-
else {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
this.tracker.delta(EStatus.FAILED, 1);
|
|
388
|
-
if (commonsTypeHasPropertyString(ex, 'code')) {
|
|
389
|
-
yield this.database.setFailReason(url, ex.code);
|
|
424
|
+
else if (commonsTypeHasPropertyString(ex, 'message')) {
|
|
425
|
+
if (/^Timeout on crawl/.test(ex.message)) {
|
|
426
|
+
await this.database.setFailReason(url, 'TIMEOUT');
|
|
390
427
|
}
|
|
391
|
-
else
|
|
392
|
-
|
|
393
|
-
yield this.database.setFailReason(url, 'TIMEOUT');
|
|
394
|
-
}
|
|
395
|
-
else {
|
|
396
|
-
yield this.database.setFailReason(url, ex.message);
|
|
397
|
-
}
|
|
428
|
+
else {
|
|
429
|
+
await this.database.setFailReason(url, ex.message);
|
|
398
430
|
}
|
|
399
431
|
}
|
|
400
432
|
}
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
433
|
+
}
|
|
434
|
+
finally {
|
|
435
|
+
if (this.tracker)
|
|
436
|
+
this.tracker.delta(EStatus.ACTIVE, -1);
|
|
437
|
+
}
|
|
438
|
+
tally++;
|
|
439
|
+
if (tally >= this.crawlConfig.maxFetchesPerCrawl) {
|
|
440
|
+
commonsLogAlert(`Crawl for ${this.domain} reached the maximum permitted fetches for this session.`);
|
|
441
|
+
break;
|
|
410
442
|
}
|
|
411
443
|
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
}
|
|
444
|
+
}
|
|
445
|
+
catch (e) {
|
|
446
|
+
commonsLogDebug('debug position 1');
|
|
447
|
+
console.log(e);
|
|
448
|
+
}
|
|
449
|
+
finally {
|
|
450
|
+
commonsGracefulAbortRemoveCallback(gracefulAbortId);
|
|
451
|
+
}
|
|
420
452
|
}
|
|
421
453
|
}
|
|
422
|
-
//# sourceMappingURL=crawler.
|
|
454
|
+
//# sourceMappingURL=crawler.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawler.mjs","sourceRoot":"","sources":["../../src/classes/crawler.mts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAEtF,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACxG,OAAO,EAAE,+BAA+B,EAAE,kCAAkC,EAAE,MAAM,yBAAyB,CAAC;AAC9G,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAK7C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAU/C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAEhC,MAAM,OAAO,OAAO;IAsMT;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IA5MH,MAAM,CAAC,sBAAsB,CAAC,OAA2B;QAC/D,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,uFAAuF,CAAC,CAAC;QACrH,OAAO,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,qJAAqJ,CAAC,CAAC;IACxL,CAAC;IAEM,MAAM,CAAC,OAAO,CACnB,OAAiC,EACjC,GAAW,EACX,cAAsB,EACtB,WAAmB,EACnB,OAAiB;QAElB,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAqC,EAAE,MAA0B,EAAQ,EAAE;YAC/G,MAAM,KAAK,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,GAAY,KAAK,CAAC;YAE9B,IAAI,SAAS,GAAY,KAAK,CAAC;YAC/B,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,eAAe,GAAW,uBAAuB,GAAG,EAAE,CAAC;YAE7D,MAAM,kBAAkB,GAAW,WAAW,GAAG,oBAAoB,CAAC;YACtE,mBAAmB,CACjB,cAAc,GAAG,KAAK,EACtB,kBAAkB,CACnB;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,eAAe,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBAEhD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACnC,OAAO;YACR,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,eAAe,CAAC,mBAAmB,CAAC,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC;YACT,CAAC,CAAC,CAAC;YAEL,MAAM,GAAG,GAAuB,OAAO,CAAC,OAAO,CAC7C,GAAG,EACH,CAAC,GAAyB,EAAQ,EAAE;gBACnC,4EAA4E;gBAC5E,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAW,GAAG,GAAG,KAAK,CAAC;gBAEpC,gDAAgD;gBAEhD,IAAI,IAAI,GAAW,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,GAAG,CAAC,EAAE,CACJ,MAAM,EACN,CAAC,KAAa,EAAQ,EAAE;oBACvB,IAAI,IAAI,GAAG,WAAW,EAAE,CAAC;wBACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,QAAQ,GAAG,IAAI,CAAC;4BAChB,GAAG,CAAC,OAAO,EAAE,CAAC;4BAEd,eAAe,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;wBACzD,CAAC;wBAED,OAAO;oBACR,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,OAAO;wBAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CACF,CAAC;gBACF,GAAG,CAAC,EAAE,CACJ,KAAK,EACL,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,SAAS,GAAG,IAAI,CAAC;oBAEjB,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,OAAO,CAAC;wBACN,OAAO,EAAE,OAAO;wBAChB,mEAAmE;wBACnE,EAAE,EAAG,GAAuB,CAAC,UAAU,CAAC;wBACxC,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG;wBACjC,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACJ,CAAC,CACF,CAAC;gBAEF,wEAAwE;gBACxE,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;oBACpB,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAEpC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAS,EAAE;gBAC5B,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,QAAQ,GAAG,IAAI,CAAC;gBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CACJ,UAAU,EACV,CAAC,GAAyB,EAAQ,EAAE;gBAClC,GAAuB,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACrE,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;gBACpB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;YACb,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,QAAQ,EACR,CAAC,MAAc,EAAQ,EAAE;gBACxB,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CACP,SAAS,EACT,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,QAAQ,GAAG,IAAI,CAAC;oBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACpC,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAA4B,EAAE,WAAqB;QAC9E,MAAM,GAAG,GAAoC,IAAI,GAAG,EAA8B,CAAC;QAEnF,MAAM,IAAI,GAAoC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACtD,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE9D,MAAM,KAAK,GAAW,OAAO,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;YACjD,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;gBAClB,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM;gBACzD,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBAAC,MAAM;gBAC5D,KAAK,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAC,MAAM;gBACtD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACrD,CAAC;QACF,CAAC;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,QAAQ,GAAY,KAAK,CAAC;IAC1B,SAAS,GAAY,KAAK,CAAC;IAEnC,YACU,MAAc,EACd,QAAyB,EACzB,WAAyB,EACzB,aAA6C,EAC7C,YAA2B,EAC3B,OAAsB,EACtB,KAAY,EACZ,OAAiB;QAPjB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAAgC;QAC7C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAe;QACtB,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAU;IACxB,CAAC;IAEI,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK;QACX,eAAe,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAEM,MAAM;QACZ,cAAc,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,WAAoB;QACnD,cAAc,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;QAElC,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,OAA2C,CAAC;QAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,KAAK,OAAO;gBAAE,OAAO,GAAG,IAAI,CAAC;gBAAC,MAAM;YACpC,KAAK,QAAQ;gBAAE,OAAO,GAAG,KAAK,CAAC;gBAAC,MAAM;YACtC;gBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,OAAwB,CAAC;QAC7B,4CAA4C;QAC5C,IAAI,CAAC;YACJ,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,OAAO,EACP,GAAG,EACH,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;QACH,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,kEAAkE;YAClE,MAAM,EAAE,CAAC;QACV,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAoC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QACrH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrH,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IAAI,CAAC;YACJ,IACC,OAAO,CAAC,UAAU,IAAI,GAAG;mBACtB,OAAO,CAAC,UAAU,GAAG,GAAG;mBACxB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;mBACvB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,0CAA0C;cAC/E,CAAC;gBACF,iBAAiB;gBACjB,MAAM,QAAQ,GAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAY,EAAE,GAAG,CAAC,CAAC;gBAEvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;oBAC5C,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBACzH,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAEpI,IAAI,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG;wBAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACrE,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,0BAA0B;YAC1B,YAAY;QACb,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,GAAW,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpE,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACxE,MAAM,WAAW,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAY,CAAC;YAEnE,IAAI,OAAgB,CAAC;YACrB,IAAI,OAAO,GAAY,IAAI,CAAC;YAC5B,IAAI,CAAC;gBACJ,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC9C,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,0BAA0B;gBAC1B,YAAY;gBACZ,OAAO,GAAG,KAAK,CAAC;gBAChB,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK;YACvB,CAAC;YAED,IAAI,OAAO,EAAE,CAAC;gBACb,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,MAAM,GAAY,IAAI,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBACnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;wBAAE,SAAS;oBAElC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;wBAAE,SAAS;oBAErD,IAAI,CAAC;wBACJ,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACnC,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC;wBACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjB,CAAC;oBAED,IAAI,OAAO,EAAE,CAAC;wBACb,IAAI,CAAC;4BACJ,MAAM,KAAK,GAAa,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;4BAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gCAAE,SAAS;4BAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gCAC1B,IAAI,IAAI,KAAK,GAAG;oCAAE,SAAS;gCAC3B,IAAI,CAAC;oCACJ,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gCACrB,CAAC;gCAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;4BACnC,CAAC;wBACF,CAAC;wBAAC,OAAO,EAAE,EAAE,CAAC;4BACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;4BACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACJ,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3F,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtH,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,0BAA0B;gBAC1B,YAAY;YACb,CAAC;QACF,CAAC;QACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAErD,OAAO,OAAO,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEM,KAAK,CAAC,KAAK;QACjB,MAAM,eAAe,GAAW,+BAA+B,CAAC,GAAS,EAAE;YAC1E,eAAe,CAAC,8CAA8C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9E,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACJ,IAAI,CAAC;gBACJ,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,IAAK,EAAY,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC;oBACxF,eAAe,CAAC,qBAAqB,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;oBAC3E,OAAO;gBACR,CAAC;gBAED,eAAe,CAAC,UAAU,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1C,OAAO;YACR,CAAC;YAED,MAAM,MAAM,GAAU,IAAI,KAAK,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CACb,CAAC;YACF,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAEpB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS;oBAAE,MAAM;gBAE1B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnB,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;oBAChC,SAAS;gBACV,CAAC;gBAED,MAAM,GAAG,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpE,IAAI,CAAC,GAAG,EAAE,CAAC;oBACV,cAAc,CAAC,aAAa,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;oBACtD,MAAM;gBACP,CAAC;gBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAEzC,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;gBAEzD,IAAI,CAAC;oBACJ,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;wBACvC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;wBAEtD,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;oBACV,CAAC;gBACF,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,0BAA0B;oBAC1B,YAAY;gBACb,CAAC;gBAED,IAAI,CAAC;oBACJ,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;wBAE5D,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;oBACV,CAAC;gBACF,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,0BAA0B;oBAC1B,YAAY;gBACb,CAAC;gBAED,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzD,eAAe,CAAC,sCAAsC,GAAG,qBAAqB,CAAC,CAAC;oBAChF,MAAM;gBACP,CAAC;gBACD,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACf,IAAI,CAAC;wBACJ,MAAM,OAAO,GAAW,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;wBAE/F,MAAM,mBAAmB,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;wBAE3D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;4BACzC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBACjC,CAAC;oBACF,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;gBAC/B,CAAC;gBACD,IAAI,IAAI,CAAC,SAAS;oBAAE,MAAM;gBAE1B,IAAI,IAAI,CAAC,OAAO;oBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACnD,IAAI,CAAC;oBACJ,MAAM,UAAU,GAAW,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;oBAE9D,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAC3D,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;oBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAEjD,MAAM,GAAG,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;wBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;wBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;wBACnD,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBACzD,CAAC;yBAAM,CAAC;wBACP,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;wBACnD,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;wBAExD,IAAI,4BAA4B,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;4BAC9C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAAuB,CAAC,IAAI,CAAC,CAAC;wBACvE,CAAC;6BAAM,IAAI,4BAA4B,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC;4BACxD,IAAI,mBAAmB,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,EAAE,CAAC;gCACnE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;4BACnD,CAAC;iCAAM,CAAC;gCACP,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAA0B,CAAC,OAAO,CAAC,CAAC;4BAC7E,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;wBAAS,CAAC;oBACV,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBAED,KAAK,EAAE,CAAC;gBACR,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;oBAClD,eAAe,CAAC,aAAa,IAAI,CAAC,MAAM,0DAA0D,CAAC,CAAC;oBACpG,MAAM;gBACP,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,eAAe,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;gBAAS,CAAC;YACV,kCAAkC,CAAC,eAAe,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as dns from 'dns';
|
|
2
|
-
import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-
|
|
3
|
-
import {
|
|
2
|
+
import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-esm-async';
|
|
3
|
+
import { commonsLogDebug } from 'nodecommons-esm-log';
|
|
4
4
|
export class Dns {
|
|
5
5
|
static resolve(domain, connectTimeout) {
|
|
6
6
|
// try IP4 first, then IP6
|
|
@@ -15,7 +15,7 @@ export class Dns {
|
|
|
15
15
|
.catch((e) => {
|
|
16
16
|
if (e.message === 'abortTimeout called')
|
|
17
17
|
return;
|
|
18
|
-
|
|
18
|
+
commonsLogDebug('debug position 4');
|
|
19
19
|
console.log(e);
|
|
20
20
|
reject(e);
|
|
21
21
|
});
|
|
@@ -44,4 +44,4 @@ export class Dns {
|
|
|
44
44
|
});
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
|
-
//# sourceMappingURL=dns.
|
|
47
|
+
//# sourceMappingURL=dns.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dns.mjs","sourceRoot":"","sources":["../../src/classes/dns.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAE3B,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,OAAgB,GAAG;IACjB,MAAM,CAAC,OAAO,CACnB,MAAc,EACd,cAAsB;QAEvB,0BAA0B;QAE1B,OAAO,IAAI,OAAO,CAAW,CAAC,OAA8B,EAAE,MAA0B,EAAQ,EAAE;YACjG,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,SAAS,GAAW,OAAO,MAAM,UAAU,CAAC;YAElD,mBAAmB,CACjB,cAAc,EACd,SAAS,CACV;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,eAAe,CAAC,kBAAkB,CAAC,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;YAEL,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAe,EAAE,OAAiB,EAAQ,EAAE;gBACjE,IAAI,GAAG,EAAE,CAAC;oBACT,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAgB,EAAE,QAAkB,EAAQ,EAAE;wBACnE,IAAI,IAAI,EAAE,CAAC;4BACV,IAAI,CAAC,QAAQ;gCAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACnD,MAAM,CAAC,IAAI,CAAC,CAAC;4BACb,OAAO;wBACR,CAAC;wBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACpC,OAAO,CAAC,QAAQ,CAAC,CAAC;wBACnB,CAAC;wBACD,OAAO;oBACR,CAAC,CAAC,CAAC;oBACH,OAAO;gBACR,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBACpC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Expiry } from '../classes/expiry';
|
|
2
|
-
import { DatabaseService } from '../services/database.service';
|
|
1
|
+
import { Expiry } from '../classes/expiry.mjs';
|
|
2
|
+
import { DatabaseService } from '../services/database.service.mjs';
|
|
3
3
|
export declare class Expirer {
|
|
4
4
|
private expiry;
|
|
5
5
|
private database;
|