hydra-crawler 1.4.6 → 2.0.0

package/dist/classes/dns.js

@@ -1,16 +1,13 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Dns = void 0;
-const dns = require("dns");
-const tscommons_async_1 = require("tscommons-async");
-const nodecommons_cli_1 = require("nodecommons-cli");
-class Dns {
+import * as dns from 'dns';
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-es-async';
+import { commonsOutputDebug } from 'nodecommons-es-cli';
+export class Dns {
     static resolve(domain, connectTimeout) {
         // try IP4 first, then IP6
         return new Promise((resolve, reject) => {
             let timedout = false;
             const timeoutId = `dns_${domain}_timeout`;
-            tscommons_async_1.CommonsAsync.timeout(connectTimeout, timeoutId)
+            commonsAsyncTimeout(connectTimeout, timeoutId)
                 .then(() => {
                 timedout = true;
                 reject(new Error('DNS timeout'));
@@ -18,7 +15,7 @@ class Dns {
                 .catch((e) => {
                 if (e.message === 'abortTimeout called')
                     return;
-                nodecommons_cli_1.CommonsOutput.debug('debug position 4');
+                commonsOutputDebug('debug position 4');
                 console.log(e);
                 reject(e);
             });
@@ -27,12 +24,12 @@ class Dns {
                     dns.resolve6(domain, (err2, address2) => {
                         if (err2) {
                             if (!timedout)
-                                tscommons_async_1.CommonsAsync.abortTimeout(timeoutId);
+                                commonsAsyncAbortTimeout(timeoutId);
                             reject(err2);
                             return;
                         }
                         if (!timedout) {
-                            tscommons_async_1.CommonsAsync.abortTimeout(timeoutId);
+                            commonsAsyncAbortTimeout(timeoutId);
                             resolve(address2);
                         }
                         return;
@@ -40,11 +37,11 @@ class Dns {
                     return;
                 }
                 if (!timedout) {
-                    tscommons_async_1.CommonsAsync.abortTimeout(timeoutId);
+                    commonsAsyncAbortTimeout(timeoutId);
                     resolve(address);
                 }
             });
         });
     }
 }
-exports.Dns = Dns;
+//# sourceMappingURL=dns.js.map

package/dist/classes/dns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dns.js","sourceRoot":"","sources":["../../src/classes/dns.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAE3B,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExD,MAAM,OAAgB,GAAG;IACjB,MAAM,CAAC,OAAO,CACnB,MAAc,EACd,cAAsB;QAEvB,0BAA0B;QAE1B,OAAO,IAAI,OAAO,CAAW,CAAC,OAA8B,EAAE,MAA0B,EAAQ,EAAE;YACjG,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,SAAS,GAAW,OAAO,MAAM,UAAU,CAAC;YAElD,mBAAmB,CACjB,cAAc,EACd,SAAS,CACV;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;YAEL,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAe,EAAE,OAAiB,EAAQ,EAAE;gBACjE,IAAI,GAAG,EAAE;oBACR,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAgB,EAAE,QAAkB,EAAQ,EAAE;wBACnE,IAAI,IAAI,EAAE;4BACT,IAAI,CAAC,QAAQ;gCAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACnD,MAAM,CAAC,IAAI,CAAC,CAAC;4BACb,OAAO;yBACP;wBACD,IAAI,CAAC,QAAQ,EAAE;4BACd,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACpC,OAAO,CAAC,QAAQ,CAAC,CAAC;yBAClB;wBACD,OAAO;oBACR,CAAC,CAAC,CAAC;oBACH,OAAO;iBACP;gBACD,IAAI,CAAC,QAAQ,EAAE;oBACd,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBACpC,OAAO,CAAC,OAAO,CAAC,CAAC;iBACjB;YACF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}

package/dist/classes/expirer.d.ts

@@ -0,0 +1,10 @@
+import { Expiry } from '../classes/expiry';
+import { DatabaseService } from '../services/database.service';
+export declare class Expirer {
+    private expiry;
+    private database;
+    constructor(expiry: Expiry, database: DatabaseService);
+    private listQueuedAndActive;
+    expire(limit?: number): Promise<boolean>;
+    expireFixed(threshold: Date): Promise<boolean>;
+}

package/dist/classes/expirer.js

@@ -1,105 +1,118 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Expirer = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const tscommons_core_2 = require("tscommons-core");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const database_service_1 = require("../services/database.service");
-class Expirer {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsArrayChunk, commonsDateDateToYmdHis, commonsTypeHasPropertyDate } from 'tscommons-es-core';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputAlert, commonsOutputDoing, commonsOutputProgress, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { isTMongoIdRow } from '../services/database.service';
+export class Expirer {
     constructor(expiry, database) {
         this.expiry = expiry;
         this.database = database;
     }
-    async listQueuedAndActive() {
-        nodecommons_cli_1.CommonsOutput.doing(`Enumering queued and active domains`);
-        const results = this.database.getUrls().aggregate([
-            { $match: { status: { $in: [hydra_crawler_ts_assets_1.EStatus.QUEUED, hydra_crawler_ts_assets_1.EStatus.ACTIVE] } } },
-            { $group: { _id: '$domain' } }
-        ], { allowDiskUse: true });
-        const queued = await this.database.listQueryResults(results, database_service_1.isTMongoIdRow);
-        nodecommons_cli_1.CommonsOutput.success(queued.length);
-        return queued
-            .map((q) => q._id);
+    listQueuedAndActive() {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputDoing('Enumering queued and active domains');
+            const results = this.database.getUrls().aggregate([
+                { $match: { status: { $in: [EStatus.QUEUED, EStatus.ACTIVE] } } },
+                { $group: { _id: '$domain' } }
+            ], { allowDiskUse: true });
+            const queued = yield this.database.listQueryResults(results, isTMongoIdRow);
+            commonsOutputSuccess(queued.length);
+            return queued
+                // eslint-disable-next-line no-underscore-dangle
+                .map((q) => q._id);
+        });
     }
-    async expire(limit) {
-        nodecommons_cli_1.CommonsOutput.alert(`Running expiry thread`);
-        const queued = await this.listQueuedAndActive();
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for expired URLs`);
-        const results = this.database.getUrls().find({ $and: [
-                { status: { $nin: [
-                            hydra_crawler_ts_assets_1.EStatus.ARCHIVED,
-                            hydra_crawler_ts_assets_1.EStatus.QUEUED,
-                            hydra_crawler_ts_assets_1.EStatus.ACTIVE
-                        ] } },
-                { domain: { $nin: queued } },
-                { attempted: { $exists: true } }
-            ] });
-        const now = new Date().getTime() / 1000;
-        const expired = [];
-        let tally = 0;
-        let found = 0;
-        while (true) {
-            tally++;
-            if ((tally % 1000) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}, ${found}`);
-            if (limit !== undefined && tally > limit)
-                break;
-            const row = await results.next();
-            if (row === null)
-                break;
-            const interval = this.expiry.getBestExpiry(row.url);
-            if (!tscommons_core_1.CommonsType.hasPropertyDate(row, 'attempted'))
-                continue;
-            const attempted = row['attempted'].getTime() / 1000;
-            if ((attempted + interval) < now) {
-                expired.push(row['_id']);
-                found++;
+    expire(limit) {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputAlert('Running expiry thread');
+            const queued = yield this.listQueuedAndActive();
+            commonsOutputDoing('Searching for expired URLs');
+            const results = this.database.getUrls().find({ $and: [
+                    { status: { $nin: [
+                                EStatus.ARCHIVED,
+                                EStatus.QUEUED,
+                                EStatus.ACTIVE
+                            ] } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } }
+                ] }, {});
+            const now = new Date().getTime() / 1000;
+            const expired = [];
+            let tally = 0;
+            let found = 0;
+            while (true) {
+                tally++;
+                if ((tally % 1000) === 0)
+                    commonsOutputProgress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = yield results.next();
+                if (row === null)
+                    break;
+                const interval = this.expiry.getBestExpiry(row.url);
+                if (!commonsTypeHasPropertyDate(row, 'attempted'))
+                    continue;
+                const attempted = row['attempted'].getTime() / 1000;
+                if ((attempted + interval) < now) {
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+                    expired.push(row['_id']);
+                    found++;
+                }
+            }
+            commonsOutputSuccess(found);
+            if (found === 0)
+                return false;
+            const batches = commonsArrayChunk(expired, 100);
+            commonsOutputDoing('Re-queuing expired');
+            tally = 0;
+            for (const batch of batches) {
+                yield this.database.getUrls().updateMany({ _id: { $in: batch } }, {
+                    $set: { status: EStatus.QUEUED },
+                    $unset: { ttl: true }
+                });
+                tally += 100;
+                commonsOutputProgress(tally);
             }
-        }
-        nodecommons_cli_1.CommonsOutput.success(found);
-        if (found === 0)
-            return false;
-        const batches = tscommons_core_2.CommonsArray.chunk(expired, 100);
-        nodecommons_cli_1.CommonsOutput.doing(`Re-queuing expired`);
-        tally = 0;
-        for (const batch of batches) {
-            await this.database.getUrls().updateMany({ _id: { $in: batch } }, {
-                $set: { status: hydra_crawler_ts_assets_1.EStatus.QUEUED },
+            commonsOutputSuccess();
+            return true;
+        });
+    }
+    expireFixed(threshold) {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputAlert('Running fixed date expiry thread');
+            const queued = yield this.listQueuedAndActive();
+            commonsOutputDoing(`Searching attempted before date ${commonsDateDateToYmdHis(threshold)}`);
+            const count = yield this.database.getUrls().find({ $and: [
+                    { status: { $ne: EStatus.ARCHIVED } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } },
+                    { attempted: { $lt: threshold } }
+                ] }, {}).count();
+            commonsOutputSuccess(count);
+            if (count === 0)
+                return false;
+            commonsOutputDoing('Re-queuing expired');
+            yield this.database.getUrls().updateMany({ $and: [
+                    { status: { $ne: EStatus.ARCHIVED } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } },
+                    { attempted: { $lt: threshold } }
+                ] }, {
+                $set: { status: EStatus.QUEUED },
                 $unset: { ttl: true }
             });
-            tally += 100;
-            nodecommons_cli_1.CommonsOutput.progress(tally);
-        }
-        nodecommons_cli_1.CommonsOutput.success();
-        return true;
-    }
-    async expireFixed(threshold) {
-        nodecommons_cli_1.CommonsOutput.alert(`Running fixed date expiry thread`);
-        const queued = await this.listQueuedAndActive();
-        nodecommons_cli_1.CommonsOutput.doing(`Searching attempted before date ${threshold}`);
-        const count = await this.database.getUrls().find({ $and: [
-                { status: { $ne: hydra_crawler_ts_assets_1.EStatus.ARCHIVED } },
-                { domain: { $nin: queued } },
-                { attempted: { $exists: true } },
-                { attempted: { $lt: threshold } }
-            ] }).count();
-        nodecommons_cli_1.CommonsOutput.success(count);
-        if (count === 0)
-            return false;
-        nodecommons_cli_1.CommonsOutput.doing(`Re-queuing expired`);
-        await this.database.getUrls().updateMany({ $and: [
-                { status: { $ne: hydra_crawler_ts_assets_1.EStatus.ARCHIVED } },
-                { domain: { $nin: queued } },
-                { attempted: { $exists: true } },
-                { attempted: { $lt: threshold } }
-            ] }, {
-            $set: { status: hydra_crawler_ts_assets_1.EStatus.QUEUED },
-            $unset: { ttl: true }
+            commonsOutputSuccess();
+            commonsOutputDoing('Searching for expired URLs');
+            return true;
         });
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for expired URLs`);
-        return true;
     }
 }
-exports.Expirer = Expirer;
+//# sourceMappingURL=expirer.js.map

package/dist/classes/expirer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expirer.js","sourceRoot":"","sources":["../../src/classes/expirer.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AAE3G,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAIzH,OAAO,EAAgC,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAE3F,MAAM,OAAO,OAAO;IACnB,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEU,mBAAmB;;YAChC,kBAAkB,CAAC,qCAAqC,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAmC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;gBAC7F,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;gBACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;aAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;YAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAE3F,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpC,OAAO,MAAM;gBACX,gDAAgD;iBAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,MAAM,CAAC,KAAc;;YACjC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAE5C,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAiB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACxD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE5D,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,MAAM,SAAS,GAAY,GAAG,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;gBACtE,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE;oBACjC,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzB,KAAK,EAAE,CAAC;iBACR;aACD;YACD,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAE5D,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;YAEvB,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;IAEY,WAAW,CAAC,SAAe;;YACvC,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;YAEvD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAE1D,kBAAkB,CAAC,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,oBAAoB,CAAC,KAAK,CAAC,CAAC;YAE5B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE9B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,OAAO,IAAI,CAAC;QACb,CAAC;KAAA;CACD"}

package/dist/classes/expiry.d.ts

@@ -0,0 +1,8 @@
+import { IExpiry } from '../interfaces/iexpiry';
+export declare class Expiry {
+    static loadFromFile(file: string): IExpiry[];
+    private static getScore;
+    private expiries;
+    add(expiries: IExpiry[]): void;
+    getBestExpiry(url: string): number;
+}

package/dist/classes/expiry.js

@@ -1,41 +1,38 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Expiry = void 0;
-const url_1 = require("url");
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_file_1 = require("nodecommons-file");
-const matcher_1 = require("../helpers/matcher");
-const iexpiry_1 = require("../interfaces/iexpiry");
-class Expiry {
+import { URL } from 'url';
+import { commonsTypeHasPropertyString, commonsTypeIsArray, commonsTypeIsTArray } from 'tscommons-es-core';
+import { commonsFileReadJsonFile } from 'nodecommons-es-file';
+import { Matcher } from '../helpers/matcher';
+import { isIExpiry, toIExpiry } from '../interfaces/iexpiry';
+export class Expiry {
     constructor() {
         this.expiries = [];
     }
     static loadFromFile(file) {
-        const json = nodecommons_file_1.CommonsFile.readJsonFile(file);
-        if (!tscommons_core_1.CommonsType.isArray(json))
+        const json = commonsFileReadJsonFile(file);
+        if (!commonsTypeIsArray(json))
             throw new Error('Invalid expiry list');
         const expiries = json
             .map((entry) => {
             try {
-                return iexpiry_1.toIExpiry(entry);
+                return toIExpiry(entry);
             }
             catch (e) {
                 return undefined;
             }
         });
-        if (!tscommons_core_1.CommonsType.isTArray(expiries, iexpiry_1.isIExpiry))
+        if (!commonsTypeIsTArray(expiries, isIExpiry))
             throw new Error(`Invalid JSON list in ${file}`);
         return expiries;
     }
     static getScore(expiry) {
         let total = 0;
-        if (tscommons_core_1.CommonsType.hasPropertyString(expiry.match, 'search')) {
+        if (commonsTypeHasPropertyString(expiry.match, 'search')) {
             total += 1000 + expiry.match.search.length;
         }
-        if (tscommons_core_1.CommonsType.hasPropertyString(expiry.match, 'pathname')) {
+        if (commonsTypeHasPropertyString(expiry.match, 'pathname')) {
             total += 1000 + expiry.match.pathname.length;
         }
-        if (tscommons_core_1.CommonsType.hasPropertyString(expiry.match, 'hostname')) {
+        if (commonsTypeHasPropertyString(expiry.match, 'hostname')) {
             total += 1000 + expiry.match.hostname.length;
         }
         return total;
@@ -46,11 +43,11 @@ class Expiry {
         }
     }
     getBestExpiry(url) {
-        const whatwg = new url_1.URL(url);
+        const whatwg = new URL(url);
         let best;
         let bestScore;
         for (const expiry of this.expiries) {
-            if (!matcher_1.Matcher.matches(whatwg, expiry.match))
+            if (!Matcher.matches(whatwg, expiry.match))
                 continue;
             const score = Expiry.getScore(expiry);
             if (bestScore === undefined || score > bestScore) {
@@ -63,4 +60,4 @@ class Expiry {
         return best.expiry;
     }
 }
-exports.Expiry = Expiry;
+//# sourceMappingURL=expiry.js.map

package/dist/classes/expiry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expiry.js","sourceRoot":"","sources":["../../src/classes/expiry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,4BAA4B,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAE1G,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAE9D,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAW,SAAS,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEtE,MAAM,OAAO,MAAM;IAAnB;QAmCS,aAAQ,GAAc,EAAE,CAAC;IA6BlC,CAAC;IA/DO,MAAM,CAAC,YAAY,CAAC,IAAY;QACtC,MAAM,IAAI,GAAY,uBAAuB,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAEtE,MAAM,QAAQ,GAA0B,IAAI;aACzC,GAAG,CAAC,CAAC,KAAc,EAAqB,EAAE;YAC1C,IAAI;gBACH,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;aACxB;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,SAAS,CAAC;aACjB;QACF,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,mBAAmB,CAAU,QAAQ,EAAE,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAExG,OAAO,QAAQ,CAAC;IACjB,CAAC;IAEO,MAAM,CAAC,QAAQ,CAAC,MAAe;QACtC,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;YACzD,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAO,CAAC,MAAM,CAAC;SAC5C;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE;YAC3D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;SAC9C;QACD,IAAI,4BAA4B,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE;YAC3D,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,QAAS,CAAC,MAAM,CAAC;SAC9C;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAIM,GAAG,CAAC,QAAmB;QAC7B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE;YAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC3B;IACF,CAAC;IAEM,aAAa,CAAC,GAAW;QAC/B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,IAAuB,CAAC;QAC5B,IAAI,SAA2B,CAAC;QAEhC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;YACnC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;gBAAE,SAAS;YAErD,MAAM,KAAK,GAAW,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,GAAG,SAAS,EAAE;gBACjD,IAAI,GAAG,MAAM,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;aAClB;SACD;QAED,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAEtE,OAAO,IAAI,CAAC,MAAM,CAAC;IACpB,CAAC;CACD"}

package/dist/classes/lists.d.ts

@@ -0,0 +1,9 @@
+import { IMatch } from '../interfaces/imatch';
+import { EList } from '../enums/elist';
+export declare class Lists {
+    private lists;
+    constructor();
+    add(list: EList, entries: IMatch[]): void;
+    match(list: EList, url: string): boolean;
+    listHostnames(list: EList): string[];
+}

package/dist/classes/lists.js

@@ -1,12 +1,9 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Lists = void 0;
-const matcher_1 = require("../helpers/matcher");
-const elist_1 = require("../enums/elist");
-class Lists {
+import { Matcher } from '../helpers/matcher';
+import { ELISTS } from '../enums/elist';
+export class Lists {
     constructor() {
         this.lists = new Map();
-        for (const list of elist_1.ELISTS) {
+        for (const list of ELISTS) {
             this.lists.set(list, []);
         }
     }
@@ -19,7 +16,7 @@ class Lists {
         const entries = this.lists.get(list);
         if (!entries)
             throw new Error('Unknown list');
-        return matcher_1.Matcher.isAnyMatch(url, entries);
+        return Matcher.isAnyMatch(url, entries);
     }
     listHostnames(list) {
         const entries = this.lists.get(list);
@@ -28,16 +25,14 @@ class Lists {
         return entries
             .filter((item) => item.hostname !== undefined)
             .map((item) => item.hostname)
-            .map((hostname) => {
-            return hostname
-                .replace(/\(\^\|\\.\)/g, '')
-                .replace(/\\./g, '.')
-                .replace(/[$^]/g, '')
-                .replace(/[^-a-z0-9.]/gi, '')
-                .replace('www.', '')
-                .trim();
-        })
+            .map((hostname) => hostname
+            .replace(/\(\^\|\\.\)/g, '')
+            .replace(/\\./g, '.')
+            .replace(/[$^]/g, '')
+            .replace(/[^-a-z0-9.]/gi, '')
+            .replace('www.', '')
+            .trim())
             .filter((hostname) => hostname !== '');
     }
 }
-exports.Lists = Lists;
+//# sourceMappingURL=lists.js.map

package/dist/classes/lists.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lists.js","sourceRoot":"","sources":["../../src/classes/lists.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAI7C,OAAO,EAAS,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE/C,MAAM,OAAO,KAAK;IAGjB;QAFQ,UAAK,GAAyB,IAAI,GAAG,EAAmB,CAAC;QAGhE,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;SACzB;IACF,CAAC;IAEM,GAAG,CAAC,IAAW,EAAE,OAAiB;QACxC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC5B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SAClC;IACF,CAAC;IAEM,KAAK,CAAC,IAAW,EAAE,GAAW;QACpC,MAAM,OAAO,GAAuB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;QAE9C,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAEM,aAAa,CAAC,IAAW;QAC/B,MAAM,OAAO,GAAuB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;QAE9C,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC;aAC9D,GAAG,CAAC,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,QAAS,CAAC;aAC7C,GAAG,CAAC,CAAC,QAAgB,EAAU,EAAE,CAAC,QAAQ;aACxC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;aAC3B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;aAC5B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;aACnB,IAAI,EAAE,CACR;aACA,MAAM,CAAC,CAAC,QAAgB,EAAW,EAAE,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC;IAC3D,CAAC;CACD"}

package/dist/classes/robot.d.ts

@@ -0,0 +1,15 @@
+import { TCrawlConfig } from '../types/tcrawl-config';
+import { TRobotsConfig } from '../types/trobots-config';
+import { Tracker } from './tracker';
+export declare class Robot {
+    private domain;
+    private crawlConfig;
+    private robotsConfig;
+    private tracker?;
+    static parse(robotstxt: string, robotsConfig: TRobotsConfig): string[];
+    private paths;
+    constructor(domain: string, crawlConfig: TCrawlConfig, robotsConfig: TRobotsConfig, tracker?: Tracker | undefined);
+    private checkDomain;
+    load(): Promise<void>;
+    isDisallowed(url: string): boolean;
+}

package/dist/classes/robot.js

@@ -1,11 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Robot = void 0;
-const http = require("http");
-const https = require("https");
-const url_1 = require("url");
-const crawler_1 = require("./crawler");
-class Robot {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as http from 'http';
+import * as https from 'https';
+import { URL } from 'url';
+import { Crawler } from './crawler';
+export class Robot {
     constructor(domain, crawlConfig, robotsConfig, tracker) {
         this.domain = domain;
         this.crawlConfig = crawlConfig;
@@ -32,31 +38,35 @@ class Robot {
         }
         return [...new Set(paths)];
     }
-    async checkDomain(protocol) {
-        let outcome;
-        try {
-            switch (protocol) {
-                case 'http':
-                    outcome = await crawler_1.Crawler.request(http, `http://${this.domain}/robots.txt`, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
-                    break;
-                case 'https':
-                    outcome = await crawler_1.Crawler.request(https, `https://${this.domain}/robots.txt`, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
-                    break;
-                default: throw new Error('Unknown protocol');
+    checkDomain(protocol) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let outcome;
+            try {
+                switch (protocol) {
+                    case 'http':
+                        outcome = yield Crawler.request(http, `http://${this.domain}/robots.txt`, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
+                        break;
+                    case 'https':
+                        outcome = yield Crawler.request(https, `https://${this.domain}/robots.txt`, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
+                        break;
+                    default: throw new Error('Unknown protocol');
+                }
             }
-        }
-        catch (ex) { /* ignore */ }
-        if (!outcome || !outcome.data)
-            return [];
-        return Robot.parse(outcome.data.toString(), this.robotsConfig);
+            catch (ex) { /* ignore */ }
+            if (!outcome || !outcome.data)
+                return [];
+            return Robot.parse(outcome.data.toString(), this.robotsConfig);
+        });
     }
-    async load() {
-        for (const protocol of ['http', 'https']) {
-            this.paths.set(protocol, await this.checkDomain(protocol));
-        }
+    load() {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (const protocol of ['http', 'https']) {
+                this.paths.set(protocol, yield this.checkDomain(protocol));
+            }
+        });
     }
     isDisallowed(url) {
-        const whatwg = new url_1.URL(url);
+        const whatwg = new URL(url);
         const protocol = whatwg.protocol.match(/^(http(?:s?)):$/);
         if (!protocol)
             return false;
@@ -69,4 +79,4 @@ class Robot {
         return false;
     }
 }
-exports.Robot = Robot;
+//# sourceMappingURL=robot.js.map

package/dist/classes/robot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robot.js","sourceRoot":"","sources":["../../src/classes/robot.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAO1B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,KAAK;IAyBjB,YACU,MAAc,EACd,WAAyB,EACzB,YAA2B,EAC3B,OAAiB;QAHjB,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAc;QACzB,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAU;QAE1B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,CAAC;IA/BM,MAAM,CAAC,KAAK,CAAC,SAAiB,EAAE,YAA2B;QACjE,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,MAAM,QAAQ,GAAa,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtE,KAAK,IAAI,OAAO,IAAI,QAAQ,EAAE;YAC7B,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAEzB,MAAM,KAAK,GAA0B,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAClF,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBAC/C,MAAM,KAAK,GAA0B,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAC9E,IAAI,CAAC,KAAK;oBAAE,SAAS;gBAErB,IAAI,YAAY,CAAC,yBAAyB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;oBAAE,SAAS;gBACzE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;aACrB;SACD;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5B,CAAC;IAaa,WAAW,CAAC,QAAgB;;YACzC,IAAI,OAAkC,CAAC;YACvC,IAAI;gBACH,QAAQ,QAAQ,EAAE;oBACjB,KAAK,MAAM;wBACV,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,IAAI,EACJ,UAAU,IAAI,CAAC,MAAM,aAAa,EAClC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,KAAK,OAAO;wBACX,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,KAAK,EACL,WAAW,IAAI,CAAC,MAAM,aAAa,EACnC,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;wBACF,MAAM;oBACP,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;iBAC7C;aACD;YAAC,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE;YAE7B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YAEzC,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAChE,CAAC;KAAA;IAEY,IAAI;;YAChB,KAAK,MAAM,QAAQ,IAAI,CAAE,MAAM,EAAE,OAAO,CAAE,EAAE;gBAC3C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;aAC3D;QACF,CAAC;KAAA;IAEM,YAAY,CAAC,GAAW;QAC9B,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,MAAM,QAAQ,GAA0B,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,EAAE;YAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}

package/dist/classes/tracker.d.ts

@@ -0,0 +1,25 @@
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { SocketIoServer } from '../servers/socket-io.server';
+import { DatabaseService } from '../services/database.service';
+export declare class Tracker {
+    private databaseService;
+    private socketIoServer;
+    private map;
+    private current;
+    private index;
+    private fetches;
+    private linkTallies;
+    private domainTallies;
+    constructor(databaseService: DatabaseService, socketIoServer: SocketIoServer);
+    fetching(url: string): Promise<void>;
+    private outcome;
+    done(url: string, statusCode: number): Promise<void>;
+    failed(url: string): Promise<void>;
+    bandwidth(size: number): void;
+    delta(status: EStatus, delta: number): void;
+    abort(): void;
+    link(count?: number): void;
+    domain(): void;
+    private sync;
+    start(): Promise<void>;
+}