hydra-crawler 1.4.5 → 2.0.1

package/dist/apps/startup.app.js

@@ -1,10 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.StartupApp = void 0;
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsOutputDoing, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
 // load the tables into Mongo's memory
-class StartupApp extends nodecommons_app_1.CommonsApp {
+export class StartupApp extends CommonsApp {
     constructor() {
         super('hydra-crawler');
     }
@@ -14,38 +20,45 @@ class StartupApp extends nodecommons_app_1.CommonsApp {
     setDatabaseService(databaseService) {
         this.databaseService = databaseService;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const dbo = this.databaseService.getRawDatabase();
-        for (const collection of 'domains,urls,links'.split(',')) {
-            nodecommons_cli_1.CommonsOutput.doing(`Loading ${collection} collection into memory (simple enumeration)`);
-            const tally = await dbo.collection(collection).find({}).count();
-            nodecommons_cli_1.CommonsOutput.result(tally);
-        }
-        nodecommons_cli_1.CommonsOutput.doing(`Loading domains collection into memory (regex enumeration)`);
-        await this.databaseService.getDomains().find({
-            domain: /[0-9]temp[0-9]/
-        }).count();
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing(`Loading urls collection into memory (regex enumeration)`);
-        await this.databaseService.getUrls().find({
-            url: /[0-9]temp[0-9]/
-        }).count();
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing(`Loading links collection into memory (regex enumeration)`);
-        await this.databaseService.getLinks().find({
-            outgoing: /[0-9]temp[0-9]/
-        }).count();
-        nodecommons_cli_1.CommonsOutput.success();
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const dbo = this.databaseService.getRawDatabase();
+            for (const collection of 'domains,urls,links'.split(',')) {
+                commonsOutputDoing(`Loading ${collection} collection into memory (simple enumeration)`);
+                const tally = yield dbo.collection(collection).find({}).count();
+                commonsOutputResult(tally);
+            }
+            commonsOutputDoing('Loading domains collection into memory (regex enumeration)');
+            yield this.databaseService.getDomains().find({
+                domain: /[0-9]temp[0-9]/
+            }).count();
+            commonsOutputSuccess();
+            commonsOutputDoing('Loading urls collection into memory (regex enumeration)');
+            yield this.databaseService.getUrls().find({
+                url: /[0-9]temp[0-9]/
+            }).count();
+            commonsOutputSuccess();
+            commonsOutputDoing('Loading links collection into memory (regex enumeration)');
+            yield this.databaseService.getLinks().find({
+                outgoing: /[0-9]temp[0-9]/
+            }).count();
+            commonsOutputSuccess();
+        });
     }
 }
-exports.StartupApp = StartupApp;
+//# sourceMappingURL=startup.app.js.map

package/dist/apps/startup.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"startup.app.js","sourceRoot":"","sources":["../../src/apps/startup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,sCAAsC;AAEtC,MAAM,OAAO,UAAW,SAAQ,UAAU;IAGzC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,GAAG,GAAO,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,CAAC;YAEtD,KAAK,MAAM,UAAU,IAAI,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACzD,kBAAkB,CAAC,WAAW,UAAU,8CAA8C,CAAC,CAAC;gBACxF,MAAM,KAAK,GAAW,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACxE,mBAAmB,CAAC,KAAK,CAAC,CAAC;aAC3B;YAED,kBAAkB,CAAC,4DAA4D,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC;gBAC3C,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,yDAAyD,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACxC,GAAG,EAAE,gBAAgB;aACtB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,0DAA0D,CAAC,CAAC;YAC/E,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;aAC3B,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/apps/unarchive.app.d.ts

@@ -0,0 +1,15 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+export declare class UnarchiveApp extends CommonsApp implements IInternalHydraCommonListApp {
+    private databaseService;
+    private lists;
+    constructor();
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/unarchive.app.js

@@ -1,15 +1,21 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnarchiveApp = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const elist_1 = require("../enums/elist");
-class UnarchiveApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputFound, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { EList } from '../enums/elist';
+export class UnarchiveApp extends CommonsApp {
     constructor() {
         super('hydra-crawler');
-        this.lists = new lists_1.Lists();
+        this.lists = new Lists();
     }
     getAppName() {
         return 'Hydra - Unarchive';
@@ -20,50 +26,57 @@ class UnarchiveApp extends nodecommons_app_1.CommonsApp {
     addToList(list, entries) {
         this.lists.add(list, entries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Enumerating ARCHIVED URLS`);
-        const results = this.databaseService.getUrls()
-            .find({ status: hydra_crawler_ts_assets_1.EStatus.ARCHIVED });
-        const urls = [];
-        tally = 0;
-        found = 0;
-        while (true) {
-            if ((tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.found(tally, found);
-            tally++;
-            const row = await results.next();
-            if (row === null)
-                break;
-            if (!this.lists.match(elist_1.EList.ALLOW, row.url))
-                continue;
-            urls.push(row.url);
-            found++;
-        }
-        nodecommons_cli_1.CommonsOutput.result(found);
-        nodecommons_cli_1.CommonsOutput.doing('Re-QUEUEING matched URLs');
-        tally = 0;
-        while (urls.length > 0) {
-            nodecommons_cli_1.CommonsOutput.progress(tally);
-            tally += 100;
-            const batch = urls.slice(0, 100);
-            await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, {
-                $set: { status: hydra_crawler_ts_assets_1.EStatus.QUEUED }
-            });
-            urls.splice(0, 100);
-        }
-        nodecommons_cli_1.CommonsOutput.success();
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing('Enumerating ARCHIVED URLS');
+            const results = this.databaseService.getUrls()
+                .find({ status: EStatus.ARCHIVED }, {});
+            const urls = [];
+            tally = 0;
+            found = 0;
+            while (true) {
+                if ((tally % 100) === 0)
+                    commonsOutputFound(tally, found);
+                tally++;
+                const row = yield results.next();
+                if (row === null)
+                    break;
+                if (!this.lists.match(EList.ALLOW, row.url))
+                    continue;
+                urls.push(row.url);
+                found++;
+            }
+            commonsOutputResult(found);
+            commonsOutputDoing('Re-QUEUEING matched URLs');
+            tally = 0;
+            while (urls.length > 0) {
+                commonsOutputProgress(tally);
+                tally += 100;
+                const batch = urls.slice(0, 100);
+                yield this.databaseService.getUrls().updateMany({ url: { $in: batch } }, {
+                    $set: { status: EStatus.QUEUED }
+                });
+                urls.splice(0, 100);
+            }
+            commonsOutputSuccess();
+        });
     }
 }
-exports.UnarchiveApp = UnarchiveApp;
+//# sourceMappingURL=unarchive.app.js.map

package/dist/apps/unarchive.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unarchive.app.js","sourceRoot":"","sources":["../../src/apps/unarchive.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,YAAa,SAAQ,UAAU;IAK3C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzD,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YAErB,OAAO,IAAI,EAAE;gBACZ,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC1D,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YAC/C,KAAK,GAAG,CAAC,CAAC;YACV,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAC7B,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aACpB;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/classes/cleaner.d.ts

@@ -0,0 +1,12 @@
+import { Lists } from '../classes/lists';
+import { DatabaseService } from '../services/database.service';
+export declare class Cleaner {
+    private lists;
+    private databaseService;
+    constructor(lists: Lists, databaseService: DatabaseService);
+    private detectStatusOrphans;
+    private detectNonAllowlistOrphans;
+    private detectStatusCodeOrphans;
+    purgeOrphanUrls(): Promise<void>;
+    purgeEmptyDomains(): Promise<void>;
+}