hydra-crawler 1.4.6 → 2.0.0

package/dist/classes/cleaner.js

@@ -1,237 +1,257 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Cleaner = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const elist_1 = require("../enums/elist");
-class Cleaner {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeHasPropertyNumber } from 'tscommons-es-core';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { EList } from '../enums/elist';
+export class Cleaner {
     constructor(lists, databaseService) {
         this.lists = lists;
         this.databaseService = databaseService;
     }
-    async detectStatusOrphans(statuses) {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Detecting ${statuses.join(', ')} orphan urls`);
-        while (true) {
-            const urls = this.databaseService.getUrls()
-                .find({
-                $and: [
-                    { status: { $in: statuses } },
-                    { $or: [
-                            { orphan: null },
-                            { orphan: false }
-                        ] }
-                ]
-            })
-                .sort({ _id: 1 })
-                .skip(tally);
-            try {
-                while (true) {
-                    tally++;
-                    if ((tally % 100) === 0)
-                        nodecommons_cli_1.CommonsOutput.progress(`${tally} urls, ${found} orphans`);
-                    const row = await urls.next();
-                    if (row === null)
-                        break;
-                    const incoming = await this.databaseService.getLinks().find({ outgoing: row.url }).count();
-                    if (incoming > 0)
-                        continue;
-                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
-                    found++;
+    detectStatusOrphans(statuses) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing(`Detecting ${statuses.join(', ')} orphan urls`);
+            while (true) {
+                const urls = this.databaseService.getUrls()
+                    .find({
+                    $and: [
+                        { status: { $in: statuses } },
+                        { $or: [
+                                { orphan: null },
+                                { orphan: false }
+                            ] }
+                    ]
+                }, {})
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            commonsOutputProgress(`${tally} urls, ${found} orphans`);
+                        const row = yield urls.next();
+                        if (row === null)
+                            break;
+                        const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url }).count();
+                        if (incoming > 0)
+                            continue;
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                        yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                        found++;
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
                 }
-                break;
-            }
-            catch (err) {
-                if (err.code !== 43)
-                    throw err;
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(`${tally} urls, ${found} orphans`);
+            commonsOutputResult(`${tally} urls, ${found} orphans`);
+        });
     }
-    async detectNonAllowlistOrphans(statuses) {
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Detecting non-allowlist ${statuses.join(', ')} orphan urls`);
-        while (true) {
-            const urls = this.databaseService.getUrls()
-                .find({
-                $and: [
-                    { status: { $in: statuses } },
-                    { $or: [
-                            { orphan: null },
-                            { orphan: false }
-                        ] }
-                ]
-            })
-                .sort({ _id: 1 })
-                .skip(tally);
-            try {
-                while (true) {
-                    tally++;
-                    if ((tally % 100) === 0)
-                        nodecommons_cli_1.CommonsOutput.progress(`${tally} urls, ${found} orphans`);
-                    const row = await urls.next();
-                    if (row === null)
-                        break;
-                    if (this.lists.match(elist_1.EList.ALLOW, row.url))
-                        continue;
-                    const incoming = await this.databaseService.getLinks().find({ outgoing: row.url }).count();
-                    if (incoming > 0)
-                        continue;
-                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
-                    found++;
+    detectNonAllowlistOrphans(statuses) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing(`Detecting non-allowlist ${statuses.join(', ')} orphan urls`);
+            while (true) {
+                const urls = this.databaseService.getUrls()
+                    .find({
+                    $and: [
+                        { status: { $in: statuses } },
+                        { $or: [
+                                { orphan: null },
+                                { orphan: false }
+                            ] }
+                    ]
+                }, {})
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            commonsOutputProgress(`${tally} urls, ${found} orphans`);
+                        const row = yield urls.next();
+                        if (row === null)
+                            break;
+                        if (this.lists.match(EList.ALLOW, row.url))
+                            continue;
+                        const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url }).count();
+                        if (incoming > 0)
+                            continue;
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                        yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                        found++;
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
                 }
-                break;
-            }
-            catch (err) {
-                if (err.code !== 43)
-                    throw err;
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(`${tally} urls, ${found} orphans`);
+            commonsOutputResult(`${tally} urls, ${found} orphans`);
+        });
     }
-    async detectStatusCodeOrphans(gt, lt) {
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Detecting DONE status code between ${gt}<${lt} orphan urls`);
-        while (true) {
-            const urls = this.databaseService.getUrls()
-                .find({
-                $and: [
-                    { status: hydra_crawler_ts_assets_1.EStatus.DONE },
-                    { statusCode: { $gt: gt } },
-                    { statusCode: { $lt: lt } },
-                    { $or: [
-                            { orphan: null },
-                            { orphan: false }
-                        ] }
-                ]
-            })
-                .sort({ _id: 1 })
-                .skip(tally);
-            try {
-                while (true) {
-                    tally++;
-                    if ((tally % 100) === 0)
-                        nodecommons_cli_1.CommonsOutput.progress(`${tally} urls, ${found} orphans`);
-                    const row = await urls.next();
-                    if (row === null)
-                        break;
-                    const incoming = await this.databaseService.getLinks().find({ outgoing: row.url }).count();
-                    if (incoming > 0)
-                        continue;
-                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
-                    found++;
+    detectStatusCodeOrphans(gt, lt) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing(`Detecting DONE status code between ${gt}<${lt} orphan urls`);
+            while (true) {
+                const urls = this.databaseService.getUrls()
+                    .find({
+                    $and: [
+                        { status: EStatus.DONE },
+                        { statusCode: { $gt: gt } },
+                        { statusCode: { $lt: lt } },
+                        { $or: [
+                                { orphan: null },
+                                { orphan: false }
+                            ] }
+                    ]
+                }, {})
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            commonsOutputProgress(`${tally} urls, ${found} orphans`);
+                        const row = yield urls.next();
+                        if (row === null)
+                            break;
+                        const incoming = yield this.databaseService.getLinks().find({ outgoing: row.url }).count();
+                        if (incoming > 0)
+                            continue;
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                        yield this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                        found++;
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
                 }
-                break;
             }
-            catch (err) {
-                if (err.code !== 43)
-                    throw err;
+            commonsOutputResult(`${tally} urls, ${found} orphans`);
+        });
+    }
+    purgeOrphanUrls() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.detectStatusOrphans([
+                EStatus.DENY,
+                EStatus.FAILED,
+                EStatus.DEAD,
+                EStatus.DISALLOWED
+            ]);
+            yield this.detectNonAllowlistOrphans([
+                EStatus.QUEUED,
+                EStatus.DONE
+            ]);
+            yield this.detectStatusCodeOrphans(300, 310);
+            yield this.detectStatusCodeOrphans(400, 500);
+            let tally = 0;
+            commonsOutputDoing('Removing orphan outgoing links');
+            while (true) {
+                const urls2 = this.databaseService.getUrls()
+                    .find({ orphan: true }, {})
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            commonsOutputProgress(tally);
+                        const row = yield urls2.next();
+                        if (row === null)
+                            break;
+                        yield this.databaseService.getLinks().deleteMany({ url: row.url });
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
+                }
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(`${tally} urls, ${found} orphans`);
+            commonsOutputResult(tally);
+            commonsOutputDoing('Archiving orphans');
+            yield this.databaseService.getUrls().updateMany({ orphan: true }, {
+                $set: { status: EStatus.ARCHIVED },
+                $unset: {
+                    orphan: true,
+                    headers: true,
+                    server: true,
+                    hash: true,
+                    hashSet: true,
+                    ttl: true,
+                    reason: true,
+                    links: true,
+                    title: true
+                }
+            });
+            commonsOutputSuccess();
+        });
     }
-    async purgeOrphanUrls() {
-        await this.detectStatusOrphans([
-            hydra_crawler_ts_assets_1.EStatus.DENY,
-            hydra_crawler_ts_assets_1.EStatus.FAILED,
-            hydra_crawler_ts_assets_1.EStatus.DEAD,
-            hydra_crawler_ts_assets_1.EStatus.DISALLOWED
-        ]);
-        await this.detectNonAllowlistOrphans([
-            hydra_crawler_ts_assets_1.EStatus.QUEUED,
-            hydra_crawler_ts_assets_1.EStatus.DONE
-        ]);
-        await this.detectStatusCodeOrphans(300, 310);
-        await this.detectStatusCodeOrphans(400, 500);
-        let tally = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Removing orphan outgoing links`);
-        while (true) {
-            const urls2 = this.databaseService.getUrls()
-                .find({ orphan: true })
-                .sort({ _id: 1 })
-                .skip(tally);
+    purgeEmptyDomains() {
+        return __awaiter(this, void 0, void 0, function* () {
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing('Enumerating domains');
+            const results = this.databaseService.getDomains().find({}, {});
+            const domains = [];
             try {
+                tally = 0;
                 while (true) {
                     tally++;
                     if ((tally % 100) === 0)
-                        nodecommons_cli_1.CommonsOutput.progress(tally);
-                    const row = await urls2.next();
+                        commonsOutputProgress(tally);
+                    const row = yield results.next();
                     if (row === null)
                         break;
-                    await this.databaseService.getLinks().deleteMany({ url: row.url });
+                    domains.push(row.domain);
                 }
-                break;
             }
             catch (err) {
-                if (err.code !== 43)
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
                     throw err;
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
-        nodecommons_cli_1.CommonsOutput.doing(`Archiving orphans`);
-        await this.databaseService.getUrls().updateMany({ orphan: true }, {
-            $set: { status: hydra_crawler_ts_assets_1.EStatus.ARCHIVED },
-            $unset: {
-                orphan: true,
-                headers: true,
-                server: true,
-                hash: true,
-                hashSet: true,
-                ttl: true,
-                reason: true,
-                links: true,
-                title: true
-            }
-        });
-        nodecommons_cli_1.CommonsOutput.success();
-    }
-    async purgeEmptyDomains() {
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing('Enumerating domains');
-        const results = this.databaseService.getDomains().find({});
-        const domains = [];
-        try {
+            commonsOutputResult(tally);
             tally = 0;
-            while (true) {
+            found = 0;
+            commonsOutputDoing('Detecting and removing empty domains');
+            for (const domain of domains) {
                 tally++;
-                if ((tally % 100) === 0)
-                    nodecommons_cli_1.CommonsOutput.progress(tally);
-                const row = await results.next();
-                if (row === null)
-                    break;
-                domains.push(row.domain);
+                if ((tally % 10) === 0)
+                    commonsOutputProgress(`${tally} domains, ${found} removed`);
+                const urls = yield this.databaseService.getUrls()
+                    .find({
+                    domain: domain,
+                    status: { $ne: EStatus.ARCHIVED }
+                }, {})
+                    .count();
+                if (urls > 0)
+                    continue;
+                yield this.databaseService.getDomains().deleteOne({ domain: domain });
+                found++;
             }
-        }
-        catch (err) {
-            if (err.code !== 43)
-                throw err;
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
-        tally = 0;
-        found = 0;
-        nodecommons_cli_1.CommonsOutput.doing('Detecting and removing empty domains');
-        for (const domain of domains) {
-            tally++;
-            if ((tally % 10) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally} domains, ${found} removed`);
-            const urls = await this.databaseService.getUrls()
-                .find({
-                domain: domain,
-                status: { $ne: hydra_crawler_ts_assets_1.EStatus.ARCHIVED }
-            })
-                .count();
-            if (urls > 0)
-                continue;
-            await this.databaseService.getDomains().deleteOne({ domain: domain });
-            found++;
-        }
-        nodecommons_cli_1.CommonsOutput.result(`${tally} domains, ${found} removed`);
+            commonsOutputResult(`${tally} domains, ${found} removed`);
+        });
     }
 }
-exports.Cleaner = Cleaner;
+//# sourceMappingURL=cleaner.js.map

package/dist/classes/cleaner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../src/classes/cleaner.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAGjE,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAM1H,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,OAAO;IACnB,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEU,mBAAmB,CAAC,QAAmB;;YACpD,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEnE,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBACnG,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,yBAAyB,CAAC,QAAmB;;YAC1D,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;wBAC7B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAErD,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBACnG,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEa,uBAAuB,CAAC,EAAU,EAAE,EAAU;;YAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,sCAAsC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;YAEjF,OAAO,IAAI,EAAE;gBACZ,MAAM,IAAI,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CACH;oBACE,IAAI,EAAE;wBACJ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;wBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;wBAC3B,EAAE,GAAG,EAAE;gCACL,EAAE,MAAM,EAAE,IAAI,EAAE;gCAChB,EAAE,MAAM,EAAE,KAAK,EAAE;6BAClB,EAAE;qBACJ;iBACF,EACD,EAAE,CACH;qBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;wBAElF,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;wBACzC,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,QAAQ,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBACnG,IAAI,QAAQ,GAAG,CAAC;4BAAE,SAAS;wBAE3B,mEAAmE;wBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;wBAChG,KAAK,EAAE,CAAC;qBACR;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;QACxD,CAAC;KAAA;IAEY,eAAe;;YAC3B,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;aACnB,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,yBAAyB,CAAC;gBACnC,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;aACb,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAE7C,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;YACrD,OAAO,IAAI,EAAE;gBACZ,MAAM,KAAK,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACvD,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;iBACZ;aACF,CACF,CAAC;YACF,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,iBAAiB;;YAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAoB,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAEzF,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI;gBACH,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;oBAEtD,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;iBACzB;aACD;YAAC,OAAO,GAAG,EAAE;gBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;aAC7E;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;YAC3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAEpF,MAAM,IAAI,GAAW,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACtD,IAAI,CAAO;oBACV,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;iBAClC,EAAE,EAAE,CAAC;qBACL,KAAK,EAAE,CAAC;gBAEX,IAAI,IAAI,GAAG,CAAC;oBAAE,SAAS;gBAEvB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAC3D,CAAC;KAAA;CAED"}

package/dist/classes/crawler.d.ts

@@ -0,0 +1,34 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import * as http from 'http';
+import * as https from 'https';
+import { TKeyObject } from 'tscommons-es-core';
+import { Lists } from '../classes/lists';
+import { Tracker } from '../classes/tracker';
+import { DatabaseService } from '../services/database.service';
+import { IRequestOutcome } from '../interfaces/irequest-outcome';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { TCrawlConfig } from '../types/tcrawl-config';
+import { TRobotsConfig } from '../types/trobots-config';
+import { TParserCtor } from '../types/tparser-ctor';
+export declare class Crawler {
+    private domain;
+    private database;
+    private crawlConfig;
+    private parsersConfig;
+    private robotsConfig;
+    private parsers;
+    private lists;
+    private tracker?;
+    static applyMasqueradeHeaders(request: http.ClientRequest): void;
+    static request(handler: typeof http | typeof https, url: string, connectTimeout: number, maxFileSize: number, tracker?: Tracker): Promise<IRequestOutcome>;
+    private static pruneHeaders;
+    private isPaused;
+    private isAborted;
+    constructor(domain: string, database: DatabaseService, crawlConfig: TCrawlConfig, parsersConfig: TKeyObject<IParserConfig>, robotsConfig: TRobotsConfig, parsers: TParserCtor[], lists: Lists, tracker?: Tracker | undefined);
+    private abort;
+    pause(): void;
+    resume(): void;
+    fetch(url: string, setDomainIp: boolean): Promise<number>;
+    crawl(): Promise<void>;
+}