hydra-crawler 1.4.6 → 2.0.0

package/dist/apps/denylist.app.js

@@ -1,17 +1,23 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DenylistApp = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_cli_2 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const elist_1 = require("../enums/elist");
-class DenylistApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsArrayChunk } from 'tscommons-es-core';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsArgs } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { EList } from '../enums/elist';
+export class DenylistApp extends CommonsApp {
     constructor() {
         super('hydra-crawler');
-        this.lists = new lists_1.Lists();
+        this.lists = new Lists();
     }
     getAppName() {
         return 'Hydra - Denylist';
@@ -22,94 +28,105 @@ class DenylistApp extends nodecommons_app_1.CommonsApp {
     addToList(list, entries) {
         this.lists.add(list, entries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async forward(limit) {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for non-DENY urls on the denylist`);
-        const urls = [];
-        const result = this.databaseService.getUrls()
-            .find({ status: { $nin: [hydra_crawler_ts_assets_1.EStatus.DENY, hydra_crawler_ts_assets_1.EStatus.ARCHIVED] } });
-        while (true) {
-            tally++;
-            if ((tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}, ${found}`);
-            if (limit !== undefined && tally > limit)
-                break;
-            const row = await result.next();
-            if (row === null)
-                break;
-            if (!this.lists.match(elist_1.EList.DENY, row.url))
-                continue;
-            found++;
-            urls.push(row.url);
-        }
-        nodecommons_cli_1.CommonsOutput.result(found);
-        nodecommons_cli_1.CommonsOutput.doing(`Marking detected as DENY`);
-        const batches = tscommons_core_1.CommonsArray.chunk(urls, 100);
-        tally = 0;
-        for (const batch of batches) {
-            await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
-                    status: hydra_crawler_ts_assets_1.EStatus.DENY
-                }, $unset: {
-                    ttl: true
-                } });
-            tally += 100;
-            nodecommons_cli_1.CommonsOutput.progress(tally);
-        }
-        nodecommons_cli_1.CommonsOutput.success();
+    forward(limit) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing('Searching for non-DENY urls on the denylist');
+            const urls = [];
+            const result = this.databaseService.getUrls()
+                .find({ status: { $nin: [EStatus.DENY, EStatus.ARCHIVED] } }, {});
+            while (true) {
+                tally++;
+                if ((tally % 100) === 0)
+                    commonsOutputProgress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = yield result.next();
+                if (row === null)
+                    break;
+                if (!this.lists.match(EList.DENY, row.url))
+                    continue;
+                found++;
+                urls.push(row.url);
+            }
+            commonsOutputResult(found);
+            commonsOutputDoing('Marking detected as DENY');
+            const batches = commonsArrayChunk(urls, 100);
+            tally = 0;
+            for (const batch of batches) {
+                yield this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
+                        status: EStatus.DENY
+                    }, $unset: {
+                        ttl: true
+                    } });
+                tally += 100;
+                commonsOutputProgress(tally);
+            }
+            commonsOutputSuccess();
+        });
     }
-    async reverse(limit) {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        let tally = 0;
-        let found = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for DENY urls not on the denylist`);
-        const urls = [];
-        const result = this.databaseService.getUrls()
-            .find({ status: hydra_crawler_ts_assets_1.EStatus.DENY });
-        while (true) {
-            tally++;
-            if ((tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}, ${found}`);
-            if (limit !== undefined && tally > limit)
-                break;
-            const row = await result.next();
-            if (row === null)
-                break;
-            if (this.lists.match(elist_1.EList.DENY, row.url))
-                continue;
-            found++;
-            urls.push(row.url);
-        }
-        nodecommons_cli_1.CommonsOutput.result(found);
-        nodecommons_cli_1.CommonsOutput.doing(`Marking detected as QUEUED`);
-        const batches = tscommons_core_1.CommonsArray.chunk(urls, 100);
-        tally = 0;
-        for (const batch of batches) {
-            await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
-                    status: hydra_crawler_ts_assets_1.EStatus.QUEUED
-                }, $unset: {
-                    ttl: true
-                } });
-            tally += 100;
-            nodecommons_cli_1.CommonsOutput.progress(tally);
-        }
-        nodecommons_cli_1.CommonsOutput.success();
+    reverse(limit) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            let tally = 0;
+            let found = 0;
+            commonsOutputDoing('Searching for DENY urls not on the denylist');
+            const urls = [];
+            const result = this.databaseService.getUrls()
+                .find({ status: EStatus.DENY }, {});
+            while (true) {
+                tally++;
+                if ((tally % 100) === 0)
+                    commonsOutputProgress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = yield result.next();
+                if (row === null)
+                    break;
+                if (this.lists.match(EList.DENY, row.url))
+                    continue;
+                found++;
+                urls.push(row.url);
+            }
+            commonsOutputResult(found);
+            commonsOutputDoing('Marking detected as QUEUED');
+            const batches = commonsArrayChunk(urls, 100);
+            tally = 0;
+            for (const batch of batches) {
+                yield this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
+                        status: EStatus.QUEUED
+                    }, $unset: {
+                        ttl: true
+                    } });
+                tally += 100;
+                commonsOutputProgress(tally);
+            }
+            commonsOutputSuccess();
+        });
     }
-    async run() {
-        const args = new nodecommons_cli_2.CommonsArgs();
-        await this.forward(args.getNumberOrUndefined('limit'));
-        await this.reverse(args.getNumberOrUndefined('limit'));
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const args = new CommonsArgs();
+            yield this.forward(args.getNumberOrUndefined('limit'));
+            yield this.reverse(args.getNumberOrUndefined('limit'));
+        });
     }
 }
-exports.DenylistApp = DenylistApp;
+//# sourceMappingURL=denylist.app.js.map

package/dist/apps/denylist.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"denylist.app.js","sourceRoot":"","sources":["../../src/apps/denylist.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,WAAY,SAAQ,UAAU;IAK1C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACxD,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAE,EAAE,EAAE,EACxD,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACxD,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EACxB,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;YAE5C,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;KAAA;CACD"}

package/dist/apps/expire.app.d.ts

@@ -0,0 +1,19 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { IExpiry } from '../interfaces/iexpiry';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp } from './internal-hydra-common.app';
+export declare class ExpireApp extends CommonsApp implements IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp {
+    private threshold;
+    private databaseService;
+    private lists;
+    private expiry;
+    constructor(threshold: Date | undefined);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    addToExpiry(expiries: IExpiry[]): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/expire.app.js

@@ -1,18 +1,24 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExpireApp = void 0;
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_cli_2 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const expirer_1 = require("../classes/expirer");
-const expiry_1 = require("../classes/expiry");
-class ExpireApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsArgs } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { Expirer } from '../classes/expirer';
+import { Expiry } from '../classes/expiry';
+export class ExpireApp extends CommonsApp {
     constructor(threshold) {
         super('hydra-crawler');
         this.threshold = threshold;
-        this.lists = new lists_1.Lists();
-        this.expiry = new expiry_1.Expiry();
+        this.lists = new Lists();
+        this.expiry = new Expiry();
     }
     getAppName() {
         return 'Hydra - Expire';
@@ -26,25 +32,32 @@ class ExpireApp extends nodecommons_app_1.CommonsApp {
     addToExpiry(expiries) {
         this.expiry.add(expiries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const args = new nodecommons_cli_2.CommonsArgs();
-        const expirer = new expirer_1.Expirer(this.expiry, this.databaseService);
-        if (this.threshold !== undefined) {
-            await expirer.expireFixed(this.threshold);
-        }
-        else {
-            await expirer.expire(args.getNumberOrUndefined('limit'));
-        }
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const args = new CommonsArgs();
+            const expirer = new Expirer(this.expiry, this.databaseService);
+            if (this.threshold !== undefined) {
+                yield expirer.expireFixed(this.threshold);
+            }
+            else {
+                yield expirer.expire(args.getNumberOrUndefined('limit'));
+            }
+        });
     }
 }
-exports.ExpireApp = ExpireApp;
+//# sourceMappingURL=expire.app.js.map

package/dist/apps/expire.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"expire.app.js","sourceRoot":"","sources":["../../src/apps/expire.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAW3C,MAAM,OAAO,SAAU,SAAQ,UAAU;IAKxC,YACU,SAAyB;QAElC,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAAgB;QAIlC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;YAE5C,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAExE,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;gBACjC,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC1C;iBAAM;gBACN,MAAM,OAAO,CAAC,MAAM,CAClB,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CACnC,CAAC;aACF;QACF,CAAC;KAAA;CACD"}

package/dist/apps/extract-text.app.d.ts

@@ -0,0 +1,8 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { IInternalHydraCommonApp } from './internal-hydra-common.app';
+export declare class ExtractTextApp extends CommonsApp implements IInternalHydraCommonApp {
+    private url;
+    constructor(url: string);
+    getAppName(): string;
+    run(): Promise<void>;
+}

package/dist/apps/extract-text.app.js

@@ -1,22 +1,28 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExtractTextApp = void 0;
-const http = require("http");
-const https = require("https");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const crawler_1 = require("../classes/crawler");
-const text_parser_1 = require("../parsers/text.parser");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as http from 'http';
+import * as https from 'https';
+import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Crawler } from '../classes/crawler';
+import { TextParser } from '../parsers/text.parser';
 // extract the text from the HTML of a URL
-class TextParserInstance extends text_parser_1.TextParser {
-    constructor(outcome) {
-        super(outcome);
-    }
-    async parseText(_database, text) {
-        console.log(text);
+class TextParserInstance extends TextParser {
+    // eslint-disable-next-line @typescript-eslint/require-await
+    parseText(_database, text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log(text);
+        });
     }
 }
-class ExtractTextApp extends nodecommons_app_1.CommonsApp {
+export class ExtractTextApp extends CommonsApp {
     constructor(url) {
         super('hydra-crawler');
         this.url = url;
@@ -24,24 +30,26 @@ class ExtractTextApp extends nodecommons_app_1.CommonsApp {
     getAppName() {
         return 'Hydra - Extract Text';
     }
-    async run() {
-        nodecommons_cli_1.CommonsOutput.doing(`Downloading from URL at ${this.url}`);
-        const whatwg = new URL(this.url);
-        let handler;
-        switch (whatwg.protocol) {
-            case 'http:':
-                handler = http;
-                break;
-            case 'https:':
-                handler = https;
-                break;
-            default:
-                throw new Error(`unable to handle protocol ${whatwg.protocol}`);
-        }
-        const outcome = await crawler_1.Crawler.request(handler, this.url, 10000, 4096000);
-        nodecommons_cli_1.CommonsOutput.success();
-        const parser = new TextParserInstance(outcome);
-        await parser.parse(undefined);
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputDoing(`Downloading from URL at ${this.url}`);
+            const whatwg = new URL(this.url);
+            let handler;
+            switch (whatwg.protocol) {
+                case 'http:':
+                    handler = http;
+                    break;
+                case 'https:':
+                    handler = https;
+                    break;
+                default:
+                    throw new Error(`unable to handle protocol ${whatwg.protocol}`);
+            }
+            const outcome = yield Crawler.request(handler, this.url, 10000, 4096000);
+            commonsOutputSuccess();
+            const parser = new TextParserInstance(outcome);
+            yield parser.parse(undefined);
+        });
     }
 }
-exports.ExtractTextApp = ExtractTextApp;
+//# sourceMappingURL=extract-text.app.js.map

package/dist/apps/extract-text.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-text.app.js","sourceRoot":"","sources":["../../src/apps/extract-text.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAOpD,0CAA0C;AAE1C,MAAM,kBAAmB,SAAQ,UAAyB;IACzD,4DAA4D;IAC5C,SAAS,CACvB,SAA0B,EAC1B,IAAY;;YAEb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;KAAA;CACD;AAED,MAAM,OAAO,cAAe,SAAQ,UAAU;IAC7C,YACU,GAAW;QAEpB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,QAAG,GAAH,GAAG,CAAQ;IAGrB,CAAC;IAEM,UAAU;QAChB,OAAO,sBAAsB,CAAC;IAC/B,CAAC;IAEY,GAAG;;YACf,kBAAkB,CAAC,2BAA2B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEtC,IAAI,OAA2C,CAAC;YAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE;gBACxB,KAAK,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBACpC,KAAK,QAAQ;oBAAE,OAAO,GAAG,KAAK,CAAC;oBAAC,MAAM;gBACtC;oBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjE;YAED,MAAM,OAAO,GAAoB,MAAM,OAAO,CAAC,OAAO,CACpD,OAAO,EACP,IAAI,CAAC,GAAG,EACR,KAAK,EACL,OAAO,CACR,CAAC;YACF,oBAAoB,EAAE,CAAC;YAEvB,MAAM,MAAM,GAAuB,IAAI,kBAAkB,CACvD,OAAO,CACR,CAAC;YACF,MAAM,MAAM,CAAC,KAAK,CAAC,SAAuC,CAAC,CAAC;QAC7D,CAAC;KAAA;CACD"}

package/dist/apps/hydra.app.d.ts

@@ -0,0 +1,34 @@
+import { TCommonsScheduleTime } from 'tscommons-es-async';
+import { ICommonsExpressConfig } from 'nodecommons-es-express';
+import { CommonsStrictExpressServer } from 'nodecommons-es-express';
+import { CommonsSocketIoApp } from 'nodecommons-es-app-socket-io';
+import { SocketIoServer } from '../servers/socket-io.server';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { IExpiry } from '../interfaces/iexpiry';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp, IInternalHydraCommonMaintenanceApp } from './internal-hydra-common.app';
+export declare class HydraApp extends CommonsSocketIoApp<SocketIoServer> implements IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp, IInternalHydraCommonMaintenanceApp {
+    private databaseService;
+    private lists;
+    private expiry;
+    private hydraConfig;
+    private crawlConfig;
+    private parsersConfig;
+    private robotsConfig;
+    private crawlServer;
+    private maintenanceTimes;
+    private maintenanceServer;
+    private tracker;
+    constructor();
+    protected buildSocketIoServer(expressServer: CommonsStrictExpressServer, expressConfig: ICommonsExpressConfig): SocketIoServer;
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    addToExpiry(expiries: IExpiry[]): void;
+    setMaintenanceSchedule(times: TCommonsScheduleTime[]): void;
+    init(): Promise<void>;
+    protected listening(): void;
+    run(): Promise<void>;
+    protected shutdown(): Promise<void>;
+}