hydra-crawler 1.4.4 → 2.0.0

package/dist/classes/crawler.js

@@ -1,20 +1,26 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Crawler = void 0;
-const url_1 = require("url");
-const http = require("http");
-const https = require("https");
-const tscommons_core_1 = require("tscommons-core");
-const tscommons_async_1 = require("tscommons-async");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_process_1 = require("nodecommons-process");
-const nodecommons_security_1 = require("nodecommons-security");
-const robot_1 = require("../classes/robot");
-const random_1 = require("../helpers/random");
-const elist_1 = require("../enums/elist");
-const dns_1 = require("./dns");
-class Crawler {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { URL } from 'url';
+import * as http from 'http';
+import * as https from 'https';
+import { commonsMapToObject, commonsTypeHasPropertyString } from 'tscommons-es-core';
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-es-async';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputAlert, commonsOutputCompleted, commonsOutputDebug, commonsOutputError, commonsOutputInfo } from 'nodecommons-es-cli';
+import { commonsGracefulAbortAddCallback, commonsGracefulAbortRemoveCallback } from 'nodecommons-es-process';
+import { commonsHashMd5 } from 'nodecommons-es-security';
+import { Robot } from '../classes/robot';
+import { Random } from '../helpers/random';
+import { EList } from '../enums/elist';
+import { Dns } from './dns';
+export class Crawler {
     constructor(domain, database, crawlConfig, parsersConfig, robotsConfig, parsers, lists, tracker) {
         this.domain = domain;
         this.database = database;
@@ -41,11 +47,11 @@ class Crawler {
             let timedOut = false;
             const timedOutMessage = `Timeout on crawl of ${url}`;
             const secondaryTimeoutId = `crawler_${url}_secondary_timeout`;
-            tscommons_async_1.CommonsAsync.timeout(connectTimeout + 60000, secondaryTimeoutId)
+            commonsAsyncTimeout(connectTimeout + 60000, secondaryTimeoutId)
                 .then(() => {
                 if (timedOut || completed)
                     return;
-                nodecommons_cli_1.CommonsOutput.debug(`Secondary timeout for ${url}`);
+                commonsOutputDebug(`Secondary timeout for ${url}`);
                 timedOut = true;
                 reject(new Error(timedOutMessage));
                 return;
@@ -53,7 +59,7 @@ class Crawler {
                 .catch((e) => {
                 if (e.message === 'abortTimeout called')
                     return;
-                nodecommons_cli_1.CommonsOutput.debug('debug position 10');
+                commonsOutputDebug('debug position 10');
                 console.log(e);
                 throw e;
             });
@@ -71,7 +77,7 @@ class Crawler {
                         if (!exceeded) {
                             exceeded = true;
                             res.destroy();
-                            nodecommons_cli_1.CommonsOutput.alert(`Exceeded maxFileSize for URL: ${url}`);
+                            commonsOutputAlert(`Exceeded maxFileSize for URL: ${url}`);
                         }
                         return;
                     }
@@ -84,9 +90,10 @@ class Crawler {
                     if (timedOut || completed)
                         return;
                     completed = true;
-                    tscommons_async_1.CommonsAsync.abortTimeout(secondaryTimeoutId);
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
                     resolve({
                         latency: latency,
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
                         ip: res['remoteIp'],
                         statusCode: res.statusCode || 599,
                         headers: res.headers,
@@ -98,7 +105,7 @@ class Crawler {
                 res.on('error', (err) => {
                     if (timedOut || completed)
                         return;
-                    tscommons_async_1.CommonsAsync.abortTimeout(secondaryTimeoutId);
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
                     reject(err);
                 });
             });
@@ -108,8 +115,8 @@ class Crawler {
                 if (timedOut || completed)
                     return;
                 timedOut = true;
-                req.abort();
-                tscommons_async_1.CommonsAsync.abortTimeout(secondaryTimeoutId);
+                req.destroy();
+                commonsAsyncAbortTimeout(secondaryTimeoutId);
                 reject(new Error(timedOutMessage));
             });
             req.on('response', (res) => {
@@ -118,23 +125,17 @@ class Crawler {
             req.on('error', (err) => {
                 if (timedOut || completed)
                     return;
-                tscommons_async_1.CommonsAsync.abortTimeout(secondaryTimeoutId);
+                commonsAsyncAbortTimeout(secondaryTimeoutId);
                 reject(err);
             });
-            //			req.on(
-            //					'data',
-            //					(_: unknown): void => {
-            //						req.abort();
-            //					}
-            //			);
             req.on('socket', (socket) => {
                 socket.setTimeout(connectTimeout);
                 socket.on('timeout', () => {
                     if (timedOut || completed)
                         return;
-                    tscommons_async_1.CommonsAsync.abortTimeout(secondaryTimeoutId);
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
                     timedOut = true;
-                    req.abort();
+                    req.destroy();
                     reject(new Error(timedOutMessage));
                 });
             });
@@ -151,6 +152,8 @@ class Crawler {
         for (const key of keys) {
             if (!Object.keys(headers).includes(key.key))
                 continue;
+            if (!commonsTypeHasPropertyString(headers, key.key))
+                continue;
             const value = headers[key.key];
             switch (key.type) {
                 case 'string':
@@ -169,247 +172,251 @@ class Crawler {
     }
     abort() {
         this.isAborted = true;
-        tscommons_async_1.CommonsAsync.abortTimeout(`crawl_${this.domain}`);
+        commonsAsyncAbortTimeout(`crawl_${this.domain}`);
     }
     pause() {
-        nodecommons_cli_1.CommonsOutput.alert(`Pausing crawler for ${this.domain}`);
+        commonsOutputAlert(`Pausing crawler for ${this.domain}`);
         this.isPaused = true;
     }
     resume() {
-        nodecommons_cli_1.CommonsOutput.completed(`Resuming crawler for ${this.domain}`);
+        commonsOutputCompleted(`Resuming crawler for ${this.domain}`);
         this.isPaused = false;
     }
-    async fetch(url, setDomainIp) {
-        nodecommons_cli_1.CommonsOutput.info(`Fetching ${url}`);
-        const whatwg = new url_1.URL(url);
-        let handler;
-        switch (whatwg.protocol) {
-            case 'http:':
-                handler = http;
-                break;
-            case 'https:':
-                handler = https;
-                break;
-            default:
-                throw new Error(`unable to handle protocol ${whatwg.protocol}`);
-        }
-        let outcome;
-        try {
-            outcome = await Crawler.request(handler, url, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
-        }
-        catch (ex) {
-            //if (!/^Timeout on crawl of/.test(ex.message)) console.error(ex);
-            throw ex;
-        }
-        if (setDomainIp) {
-            await this.database.domain(whatwg.hostname, outcome.ip);
-            if (this.tracker)
-                this.tracker.domain();
-        }
-        await this.database.setStatusCode(url, outcome.statusCode);
-        const headers = Crawler.pruneHeaders(outcome.headers, this.crawlConfig.keepHeaders);
-        if (!headers.has('content-length') && outcome.data !== undefined)
-            headers.set('content-length', outcome.data.length);
-        await this.database.setHeaders(url, headers);
-        const outgoing = [];
-        if (outcome.statusCode >= 300
-            && outcome.statusCode < 400
-            && headers.has('location')
-            && this.lists.match(elist_1.EList.ALLOW, url) // only queue redirects for allowlist urls
-        ) {
-            // redirect codes
-            const location = new url_1.URL(headers.get('location'), url);
-            if (location.protocol.match(/^http(s?):$/)) {
-                const added = await this.database.queue(location.toString(), this.lists.match(elist_1.EList.DENY, location.toString()));
-                if (added && this.tracker)
-                    this.tracker.delta(this.lists.match(elist_1.EList.DENY, location.toString()) ? hydra_crawler_ts_assets_1.EStatus.DENY : hydra_crawler_ts_assets_1.EStatus.QUEUED, 1);
-                if (location.toString() !== url)
-                    outgoing.push(location.toString());
+    fetch(url, setDomainIp) {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputInfo(`Fetching ${url}`);
+            const whatwg = new URL(url);
+            let handler;
+            switch (whatwg.protocol) {
+                case 'http:':
+                    handler = http;
+                    break;
+                case 'https:':
+                    handler = https;
+                    break;
+                default:
+                    throw new Error(`unable to handle protocol ${whatwg.protocol}`);
             }
-        }
-        if (outcome.data) {
-            const hash = nodecommons_security_1.CommonsHash.md5(outcome.data);
-            const existing = await this.database.getHash(url);
-            if (hash !== existing)
-                await this.database.setHash(url, hash);
-        }
-        if (outcome.data && outcome.data.length && headers.has('content-type')) {
-            const contentType = headers.get('content-type');
-            const isAllow = this.lists.match(elist_1.EList.ALLOW, url);
-            for (const ctor of this.parsers) {
-                const parser = new ctor(url, outcome, this.parsersConfig);
-                if (!parser.isEnabled())
-                    continue;
-                if (!parser.supports(contentType, isAllow))
-                    continue;
-                try {
-                    await parser.parse(this.database);
-                }
-                catch (ex) {
-                    nodecommons_cli_1.CommonsOutput.debug('debug position 3');
-                    console.log(ex);
+            let outcome;
+            try {
+                outcome = yield Crawler.request(handler, url, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
+            }
+            catch (ex) {
+                //if (!/^Timeout on crawl of/.test(ex.message)) console.error(ex);
+                throw ex;
+            }
+            if (setDomainIp) {
+                yield this.database.domain(whatwg.hostname, outcome.ip);
+                if (this.tracker)
+                    this.tracker.domain();
+            }
+            yield this.database.setStatusCode(url, outcome.statusCode);
+            const headers = Crawler.pruneHeaders(outcome.headers, this.crawlConfig.keepHeaders);
+            if (!headers.has('content-length') && outcome.data !== undefined)
+                headers.set('content-length', outcome.data.length);
+            yield this.database.setHeaders(url, commonsMapToObject(headers));
+            const outgoing = [];
+            if (outcome.statusCode >= 300
+                && outcome.statusCode < 400
+                && headers.has('location')
+                && this.lists.match(EList.ALLOW, url) // only queue redirects for allowlist urls
+            ) {
+                // redirect codes
+                const location = new URL(headers.get('location'), url);
+                if (location.protocol.match(/^http(s?):$/)) {
+                    const added = yield this.database.queue(location.toString(), this.lists.match(EList.DENY, location.toString()));
+                    if (added && this.tracker)
+                        this.tracker.delta(this.lists.match(EList.DENY, location.toString()) ? EStatus.DENY : EStatus.QUEUED, 1);
+                    if (location.toString() !== url)
+                        outgoing.push(location.toString());
                 }
-                if (isAllow) {
+            }
+            if (outcome.data) {
+                const hash = commonsHashMd5(outcome.data);
+                const existing = yield this.database.getHash(url);
+                if (hash !== existing)
+                    yield this.database.setHash(url, hash);
+            }
+            if (outcome.data && outcome.data.length && headers.has('content-type')) {
+                const contentType = headers.get('content-type');
+                const isAllow = this.lists.match(EList.ALLOW, url);
+                for (const ctor of this.parsers) {
+                    const parser = new ctor(url, outcome, this.parsersConfig);
+                    if (!parser.isEnabled())
+                        continue;
+                    if (!parser.supports(contentType, isAllow))
+                        continue;
                     try {
-                        const links = await parser.links();
-                        if (links.length === 0)
-                            continue;
-                        for (const link of links) {
-                            if (link === url)
-                                continue;
-                            try {
-                                outgoing.push(link);
-                            }
-                            catch (ex) { /* do nothing */ }
-                        }
+                        yield parser.parse(this.database);
                     }
                     catch (ex) {
-                        nodecommons_cli_1.CommonsOutput.debug('debug position 2');
+                        commonsOutputDebug('debug position 3');
                         console.log(ex);
                     }
+                    if (isAllow) {
+                        try {
+                            const links = yield parser.links();
+                            if (links.length === 0)
+                                continue;
+                            for (const link of links) {
+                                if (link === url)
+                                    continue;
+                                try {
+                                    outgoing.push(link);
+                                }
+                                catch (ex) { /* do nothing */ }
+                            }
+                        }
+                        catch (ex) {
+                            commonsOutputDebug('debug position 2');
+                            console.log(ex);
+                        }
+                    }
                 }
             }
-        }
-        for (const link of outgoing) {
-            const added = await this.database.queue(link, this.lists.match(elist_1.EList.DENY, link));
-            if (added && this.tracker)
-                this.tracker.delta(this.lists.match(elist_1.EList.DENY, link) ? hydra_crawler_ts_assets_1.EStatus.DENY : hydra_crawler_ts_assets_1.EStatus.QUEUED, 1);
-        }
-        await this.database.link(url, outgoing);
-        if (this.tracker)
-            this.tracker.link(outgoing.length);
-        return outcome.statusCode;
-    }
-    async crawl() {
-        const gracefulAbortId = nodecommons_process_1.CommonsGracefulAbort.addCallback(() => {
-            nodecommons_cli_1.CommonsOutput.alert(`SIGINT abort flag is set. Aborting crawler ${this.domain}.`);
-            this.abort();
+            for (const link of outgoing) {
+                const added = yield this.database.queue(link, this.lists.match(EList.DENY, link));
+                if (added && this.tracker)
+                    this.tracker.delta(this.lists.match(EList.DENY, link) ? EStatus.DENY : EStatus.QUEUED, 1);
+            }
+            yield this.database.link(url, outgoing);
+            if (this.tracker)
+                this.tracker.link(outgoing.length);
+            return outcome.statusCode;
         });
-        try {
+    }
+    crawl() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const gracefulAbortId = commonsGracefulAbortAddCallback(() => {
+                commonsOutputAlert(`SIGINT abort flag is set. Aborting crawler ${this.domain}.`);
+                this.abort();
+            });
             try {
-                await dns_1.Dns.resolve(this.domain, this.crawlConfig.connectTimeout);
-            }
-            catch (ex) {
-                if (ex.message === 'DNS timeout' && !this.crawlConfig.treatTimeoutDnsAsDead) {
-                    nodecommons_cli_1.CommonsOutput.error(`DNS timed out for ${this.domain}. Will allow reattempts`);
-                    return;
-                }
-                nodecommons_cli_1.CommonsOutput.error(`Domain ${this.domain} is dead.`);
-                await this.database.markDead(this.domain);
-                return;
-            }
-            const robots = new robot_1.Robot(this.domain, this.crawlConfig, this.robotsConfig, this.tracker);
-            await robots.load();
-            let tally = 0;
-            while (true) {
-                if (this.isAborted)
-                    break;
-                if (this.isPaused) {
-                    await tscommons_async_1.CommonsAsync.timeout(1000);
-                    continue;
+                try {
+                    yield Dns.resolve(this.domain, this.crawlConfig.connectTimeout);
                 }
-                const url = await this.database.next(this.domain);
-                if (!url) {
-                    nodecommons_cli_1.CommonsOutput.completed(`Crawl for ${this.domain} completed.`);
-                    break;
+                catch (ex) {
+                    if (ex.message === 'DNS timeout' && !this.crawlConfig.treatTimeoutDnsAsDead) {
+                        commonsOutputError(`DNS timed out for ${this.domain}. Will allow reattempts`);
+                        return;
+                    }
+                    commonsOutputError(`Domain ${this.domain} is dead.`);
+                    yield this.database.markDead(this.domain);
+                    return;
                 }
-                await this.database.unsetFailReason(url);
-                if (this.tracker)
-                    this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.QUEUED, -1);
-                if (this.lists.match(elist_1.EList.DENY, url)) {
-                    this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.DENY);
-                    this.database.unsetTtl(url);
+                const robots = new Robot(this.domain, this.crawlConfig, this.robotsConfig, this.tracker);
+                yield robots.load();
+                let tally = 0;
+                while (true) {
+                    if (this.isAborted)
+                        break;
+                    if (this.isPaused) {
+                        yield commonsAsyncTimeout(1000);
+                        continue;
+                    }
+                    const url = yield this.database.next(this.domain);
+                    if (!url) {
+                        commonsOutputCompleted(`Crawl for ${this.domain} completed.`);
+                        break;
+                    }
+                    yield this.database.unsetFailReason(url);
                     if (this.tracker)
-                        this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.DENY, 1);
-                    // To get here, a url was queued but is now denylisted.
-                    // It may have had outgoings, so remove any that were there.
-                    await this.database.link(url, []);
-                    continue;
-                }
-                if (robots.isDisallowed(url)) {
-                    this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.DISALLOWED);
-                    this.database.unsetTtl(url);
+                        this.tracker.delta(EStatus.QUEUED, -1);
+                    if (this.lists.match(EList.DENY, url)) {
+                        yield this.database.setStatus(url, EStatus.DENY);
+                        yield this.database.unsetTtl(url);
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.DENY, 1);
+                        // To get here, a url was queued but is now denylisted.
+                        // It may have had outgoings, so remove any that were there.
+                        yield this.database.link(url, []);
+                        continue;
+                    }
+                    if (robots.isDisallowed(url)) {
+                        yield this.database.setStatus(url, EStatus.DISALLOWED);
+                        yield this.database.unsetTtl(url);
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.DISALLOWED, 1);
+                        // To get here, a url was queued but is now disallowed.
+                        // It may have had outgoings, so remove any that were there.
+                        yield this.database.link(url, []);
+                        continue;
+                    }
+                    if (!(yield this.database.setStatus(url, EStatus.ACTIVE))) {
+                        commonsOutputError(`Unable to set status to active for ${url}. Aborting crawler.`);
+                        break;
+                    }
                     if (this.tracker)
-                        this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.DISALLOWED, 1);
-                    // To get here, a url was queued but is now disallowed.
-                    // It may have had outgoings, so remove any that were there.
-                    await this.database.link(url, []);
-                    continue;
-                }
-                if (!await this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.ACTIVE)) {
-                    nodecommons_cli_1.CommonsOutput.error(`Unable to set status to active for ${url}. Aborting crawler.`);
-                    break;
-                }
-                if (this.tracker)
-                    this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.ACTIVE, 1);
-                if (tally > 0) {
-                    try {
-                        const timeout = random_1.Random.randomise(this.crawlConfig.betweenFetchDelay, this.crawlConfig);
-                        await tscommons_async_1.CommonsAsync.timeout(timeout, `crawl_${this.domain}`);
-                        while (this.isPaused && !this.isAborted) {
-                            await tscommons_async_1.CommonsAsync.timeout(1000);
+                        this.tracker.delta(EStatus.ACTIVE, 1);
+                    if (tally > 0) {
+                        try {
+                            const timeout = Random.randomise(this.crawlConfig.betweenFetchDelay, this.crawlConfig);
+                            yield commonsAsyncTimeout(timeout, `crawl_${this.domain}`);
+                            while (this.isPaused && !this.isAborted) {
+                                yield commonsAsyncTimeout(1000);
+                            }
                         }
+                        catch (ex) { /* ignore */ }
                     }
-                    catch (ex) { /* ignore */ }
-                }
-                if (this.isAborted)
-                    break;
-                if (this.tracker)
-                    this.tracker.fetching(url);
-                try {
-                    const statusCode = await this.fetch(url, tally === 0);
-                    if (this.tracker)
-                        this.tracker.done(url, statusCode);
-                    await this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.DONE);
-                    this.database.unsetTtl(url);
-                    if (this.tracker)
-                        this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.DONE, 1);
-                }
-                catch (ex) {
+                    if (this.isAborted)
+                        break;
                     if (this.tracker)
-                        this.tracker.failed(url);
-                    const ttl = await this.database.getTtl(url) || 0;
-                    if (ttl < this.crawlConfig.maxFailedTtl) {
-                        await this.database.setTtl(url, ttl + 1);
-                        await this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.QUEUED);
+                        yield this.tracker.fetching(url);
+                    try {
+                        const statusCode = yield this.fetch(url, tally === 0);
                         if (this.tracker)
-                            this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.QUEUED, 1);
+                            yield this.tracker.done(url, statusCode);
+                        yield this.database.setStatus(url, EStatus.DONE);
+                        yield this.database.unsetTtl(url);
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.DONE, 1);
                     }
-                    else {
-                        await this.database.setStatus(url, hydra_crawler_ts_assets_1.EStatus.FAILED);
+                    catch (ex) {
                         if (this.tracker)
-                            this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.FAILED, 1);
-                        if (tscommons_core_1.CommonsType.hasPropertyString(ex, 'code')) {
-                            await this.database.setFailReason(url, ex.code);
+                            yield this.tracker.failed(url);
+                        const ttl = (yield this.database.getTtl(url)) || 0;
+                        if (ttl < this.crawlConfig.maxFailedTtl) {
+                            yield this.database.setTtl(url, ttl + 1);
+                            yield this.database.setStatus(url, EStatus.QUEUED);
+                            if (this.tracker)
+                                this.tracker.delta(EStatus.QUEUED, 1);
                         }
-                        else if (tscommons_core_1.CommonsType.hasPropertyString(ex, 'message')) {
-                            if (/^Timeout on crawl/.test(ex.message)) {
-                                await this.database.setFailReason(url, 'TIMEOUT');
+                        else {
+                            yield this.database.setStatus(url, EStatus.FAILED);
+                            if (this.tracker)
+                                this.tracker.delta(EStatus.FAILED, 1);
+                            if (commonsTypeHasPropertyString(ex, 'code')) {
+                                yield this.database.setFailReason(url, ex.code);
                             }
-                            else {
-                                await this.database.setFailReason(url, ex.message);
+                            else if (commonsTypeHasPropertyString(ex, 'message')) {
+                                if (/^Timeout on crawl/.test(ex.message)) {
+                                    yield this.database.setFailReason(url, 'TIMEOUT');
+                                }
+                                else {
+                                    yield this.database.setFailReason(url, ex.message);
+                                }
                             }
                         }
                     }
-                }
-                finally {
-                    if (this.tracker)
-                        this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.ACTIVE, -1);
-                }
-                tally++;
-                if (tally >= this.crawlConfig.maxFetchesPerCrawl) {
-                    nodecommons_cli_1.CommonsOutput.alert(`Crawl for ${this.domain} reached the maximum permitted fetches for this session.`);
-                    break;
+                    finally {
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.ACTIVE, -1);
+                    }
+                    tally++;
+                    if (tally >= this.crawlConfig.maxFetchesPerCrawl) {
+                        commonsOutputAlert(`Crawl for ${this.domain} reached the maximum permitted fetches for this session.`);
+                        break;
+                    }
                 }
             }
-        }
-        catch (e) {
-            nodecommons_cli_1.CommonsOutput.debug('debug position 1');
-            console.log(e);
-        }
-        finally {
-            nodecommons_process_1.CommonsGracefulAbort.removeCallback(gracefulAbortId);
-        }
+            catch (e) {
+                commonsOutputDebug('debug position 1');
+                console.log(e);
+            }
+            finally {
+                commonsGracefulAbortRemoveCallback(gracefulAbortId);
+            }
+        });
     }
 }
-exports.Crawler = Crawler;
+//# sourceMappingURL=crawler.js.map

package/dist/classes/crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/classes/crawler.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAErF,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3I,OAAO,EAAE,+BAA+B,EAAE,kCAAkC,EAAE,MAAM,wBAAwB,CAAC;AAC7G,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAKzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAU3C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC;AAE5B,MAAM,OAAO,OAAO;IAqMnB,YACU,MAAc,EACd,QAAyB,EACzB,WAAyB,EACzB,aAAwC,EACxC,YAA2B,EAC3B,OAAsB,EACtB,KAAY,EACZ,OAAiB;QAPjB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAA2B;QACxC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAe;QACtB,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAU;QAXnB,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAY,KAAK,CAAC;IAWhC,CAAC;IA7MG,MAAM,CAAC,sBAAsB,CAAC,OAA2B;QAC/D,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,uFAAuF,CAAC,CAAC;QACrH,OAAO,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,qJAAqJ,CAAC,CAAC;IACxL,CAAC;IAEM,MAAM,CAAC,OAAO,CACnB,OAAiC,EACjC,GAAW,EACX,cAAsB,EACtB,WAAmB,EACnB,OAAiB;QAElB,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAqC,EAAE,MAA0B,EAAQ,EAAE;YAC/G,MAAM,KAAK,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,GAAY,KAAK,CAAC;YAE9B,IAAI,SAAS,GAAY,KAAK,CAAC;YAC/B,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,eAAe,GAAW,uBAAuB,GAAG,EAAE,CAAC;YAE7D,MAAM,kBAAkB,GAAW,WAAW,GAAG,oBAAoB,CAAC;YACtE,mBAAmB,CACjB,cAAc,GAAG,KAAK,EACtB,kBAAkB,CACnB;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,kBAAkB,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBAEnD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACnC,OAAO;YACR,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC;YACT,CAAC,CAAC,CAAC;YAEL,MAAM,GAAG,GAAuB,OAAO,CAAC,OAAO,CAC7C,GAAG,EACH,CAAC,GAAyB,EAAQ,EAAE;gBACnC,4EAA4E;gBAC5E,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAW,GAAG,GAAG,KAAK,CAAC;gBAEpC,gDAAgD;gBAEhD,IAAI,IAAI,GAAW,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,GAAG,CAAC,EAAE,CACJ,MAAM,EACN,CAAC,KAAa,EAAQ,EAAE;oBACvB,IAAI,IAAI,GAAG,WAAW,EAAE;wBACvB,IAAI,CAAC,QAAQ,EAAE;4BACd,QAAQ,GAAG,IAAI,CAAC;4BAChB,GAAG,CAAC,OAAO,EAAE,CAAC;4BAEd,kBAAkB,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;yBAC3D;wBAED,OAAO;qBACP;oBACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,OAAO;wBAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CACF,CAAC;gBACF,GAAG,CAAC,EAAE,CACJ,KAAK,EACL,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,SAAS,GAAG,IAAI,CAAC;oBAEjB,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,OAAO,CAAC;wBACN,OAAO,EAAE,OAAO;wBAChB,mEAAmE;wBACnE,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC;wBACnB,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG;wBACjC,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACJ,CAAC,CACF,CAAC;gBAEF,wEAAwE;gBACxE,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;oBACpB,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAEpC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAS,EAAE;gBAC5B,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,QAAQ,GAAG,IAAI,CAAC;gBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CACJ,UAAU,EACV,CAAC,GAAyB,EAAQ,EAAE;gBACnC,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YAChD,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;gBACpB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;YACb,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,QAAQ,EACR,CAAC,MAAc,EAAQ,EAAE;gBACxB,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CACP,SAAS,EACT,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,QAAQ,GAAG,IAAI,CAAC;oBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACpC,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAA4B,EAAE,WAAqB;QAC9E,MAAM,GAAG,GAAoC,IAAI,GAAG,EAA8B,CAAC;QAEnF,MAAM,IAAI,GAAoC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE;YAC5B,MAAM,EAAE,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACvC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACtD,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE9D,MAAM,KAAK,GAAW,OAAO,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;YACjD,QAAQ,GAAG,CAAC,IAAI,EAAE;gBACjB,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM;gBACzD,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBAAC,MAAM;gBAC5D,KAAK,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAC,MAAM;gBACtD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;aACpD;SACD;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAgBO,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK;QACX,kBAAkB,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAEM,MAAM;QACZ,sBAAsB,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE9D,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IAEY,KAAK,CAAC,GAAW,EAAE,WAAoB;;YACnD,iBAAiB,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YAErC,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAEjC,IAAI,OAA2C,CAAC;YAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE;gBACxB,KAAK,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBACpC,KAAK,QAAQ;oBAAE,OAAO,GAAG,KAAK,CAAC;oBAAC,MAAM;gBACtC;oBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjE;YAED,IAAI,OAAwB,CAAC;YAC7B,IAAI;gBACH,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,OAAO,EACP,GAAG,EACH,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;aACF;YAAC,OAAO,EAAE,EAAE;gBACZ,kEAAkE;gBAClE,MAAM,EAAE,CAAC;aACT;YAED,IAAI,WAAW,EAAE;gBAChB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;gBACxD,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACxC;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAE3D,MAAM,OAAO,GAAoC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACrH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrH,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;YAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;YAE9B,IACC,OAAO,CAAC,UAAU,IAAI,GAAG;mBACtB,OAAO,CAAC,UAAU,GAAG,GAAG;mBACxB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;mBACvB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,0CAA0C;cAC/E;gBACD,iBAAiB;gBACjB,MAAM,QAAQ,GAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAY,EAAE,GAAG,CAAC,CAAC;gBAEvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;oBAC3C,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBACzH,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAEpI,IAAI,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG;wBAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;iBACpE;aACD;YAED,IAAI,OAAO,CAAC,IAAI,EAAE;gBACjB,MAAM,IAAI,GAAW,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAClD,MAAM,QAAQ,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEpE,IAAI,IAAI,KAAK,QAAQ;oBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;aAC9D;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;gBACvE,MAAM,WAAW,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAY,CAAC;gBACnE,MAAM,OAAO,GAAY,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAE5D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE;oBAChC,MAAM,MAAM,GAAY,IAAI,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;oBACnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;wBAAE,SAAS;oBAElC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;wBAAE,SAAS;oBAErD,IAAI;wBACH,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;qBAClC;oBAAC,OAAO,EAAE,EAAE;wBACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;wBACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qBAChB;oBAED,IAAI,OAAO,EAAE;wBACZ,IAAI;4BACH,MAAM,KAAK,GAAa,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;4BAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gCAAE,SAAS;4BAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gCACzB,IAAI,IAAI,KAAK,GAAG;oCAAE,SAAS;gCAC3B,IAAI;oCACH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iCACpB;gCAAC,OAAO,EAAE,EAAE,EAAE,gBAAgB,EAAE;6BACjC;yBACD;wBAAC,OAAO,EAAE,EAAE;4BACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;yBAChB;qBACD;iBACD;aACD;YAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE;gBAC5B,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC3F,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;aACrH;YACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAExC,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAErD,OAAO,OAAO,CAAC,UAAU,CAAC;QAC3B,CAAC;KAAA;IAEY,KAAK;;YACjB,MAAM,eAAe,GAAW,+BAA+B,CAAC,GAAS,EAAE;gBAC1E,kBAAkB,CAAC,8CAA8C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEjF,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,IAAI;gBACH,IAAI;oBACH,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;iBAChE;gBAAC,OAAO,EAAE,EAAE;oBACZ,IAAK,EAAY,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE;wBACvF,kBAAkB,CAAC,qBAAqB,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;wBAC9E,OAAO;qBACP;oBAED,kBAAkB,CAAC,UAAU,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;oBACrD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC1C,OAAO;iBACP;gBAED,MAAM,MAAM,GAAU,IAAI,KAAK,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CACb,CAAC;gBACF,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEpB,IAAI,KAAK,GAAW,CAAC,CAAC;gBACtB,OAAO,IAAI,EAAE;oBACZ,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,QAAQ,EAAE;wBAClB,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBAChC,SAAS;qBACT;oBAED,MAAM,GAAG,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpE,IAAI,CAAC,GAAG,EAAE;wBACT,sBAAsB,CAAC,aAAa,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;wBAC9D,MAAM;qBACN;oBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBAEzC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;oBAEzD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;wBACtC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;wBAEtD,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE;wBAC7B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;wBACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;wBAE5D,uDAAuD;wBACvD,4DAA4D;wBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBAClC,SAAS;qBACT;oBAED,IAAI,CAAC,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA,EAAE;wBACxD,kBAAkB,CAAC,sCAAsC,GAAG,qBAAqB,CAAC,CAAC;wBACnF,MAAM;qBACN;oBACD,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBAExD,IAAI,KAAK,GAAG,CAAC,EAAE;wBACd,IAAI;4BACH,MAAM,OAAO,GAAW,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;4BAE/F,MAAM,mBAAmB,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;4BAE3D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;gCACxC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;6BAChC;yBACD;wBAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;qBAC7B;oBACD,IAAI,IAAI,CAAC,SAAS;wBAAE,MAAM;oBAE1B,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACnD,IAAI;wBACH,MAAM,UAAU,GAAW,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;wBAE9D,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;wBAC3D,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;qBACtD;oBAAC,OAAO,EAAE,EAAE;wBACZ,IAAI,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAEjD,MAAM,GAAG,GAAW,CAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAI,CAAC,CAAC;wBACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;4BACxC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;4BACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;yBACxD;6BAAM;4BACN,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;4BACnD,IAAI,IAAI,CAAC,OAAO;gCAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;4BAExD,IAAI,4BAA4B,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE;gCAC7C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAAuB,CAAC,IAAI,CAAC,CAAC;6BACtE;iCAAM,IAAI,4BAA4B,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE;gCACvD,IAAI,mBAAmB,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,EAAE;oCAClE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;iCAClD;qCAAM;oCACN,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAA0B,CAAC,OAAO,CAAC,CAAC;iCAC5E;6BACD;yBACD;qBACD;4BAAS;wBACT,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;qBACzD;oBAED,KAAK,EAAE,CAAC;oBACR,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE;wBACjD,kBAAkB,CAAC,aAAa,IAAI,CAAC,MAAM,0DAA0D,CAAC,CAAC;wBACvG,MAAM;qBACN;iBACD;aACD;YAAC,OAAO,CAAC,EAAE;gBACX,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACf;oBAAS;gBACT,kCAAkC,CAAC,eAAe,CAAC,CAAC;aACpD;QACF,CAAC;KAAA;CACD"}

package/dist/classes/dns.d.ts

@@ -0,0 +1,3 @@
+export declare abstract class Dns {
+    static resolve(domain: string, connectTimeout: number): Promise<string[]>;
+}