hydra-crawler 1.4.6 → 2.0.0

package/dist/apps/query.app.js

@@ -1,23 +1,29 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.QueryApp = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const nodecommons_file_1 = require("nodecommons-file");
-const lists_1 = require("../classes/lists");
-const expiry_1 = require("../classes/expiry");
-const iparser_config_1 = require("../interfaces/iparser-config");
-class QueryApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeIsTKeyObject } from 'tscommons-es-core';
+import { commonsOutputDebug, commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { commonsFileCwdRelativeOrAbsolutePath, commonsFileIsDirectory, commonsFileIsFile, commonsFileRootRelativePath } from 'nodecommons-es-file';
+import { Lists } from '../classes/lists';
+import { Expiry } from '../classes/expiry';
+import { isIParserConfig } from '../interfaces/iparser-config';
+export class QueryApp extends CommonsApp {
     constructor(queryName) {
         super('hydra-crawler');
         this.queryName = queryName;
         if (/[^-a-z0-9]/i.test(queryName))
             throw new Error('Invalid query name');
-        this.lists = new lists_1.Lists();
-        this.expiry = new expiry_1.Expiry();
+        this.lists = new Lists();
+        this.expiry = new Expiry();
         const parsersConfig = this.getConfigArea('parsers');
-        if (!tscommons_core_1.CommonsType.isTKeyObject(parsersConfig, iparser_config_1.isIParserConfig))
+        if (!commonsTypeIsTKeyObject(parsersConfig, isIParserConfig))
             throw new Error('Parsers config is not valid');
         this.parsersConfig = parsersConfig;
     }
@@ -33,41 +39,49 @@ class QueryApp extends nodecommons_app_1.CommonsApp {
     addToExpiry(expiries) {
         this.expiry.add(expiries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const paths = [];
-        const cwdNodeModulesConfig = nodecommons_file_1.CommonsFile.cwdRelativeOrAbsolutePath(`dist/queries`);
-        nodecommons_cli_1.CommonsOutput.debug(`Calculated cwd node_modules path for queries is ${cwdNodeModulesConfig}`);
-        if (nodecommons_file_1.CommonsFile.isDirectory(cwdNodeModulesConfig))
-            paths.push(cwdNodeModulesConfig);
-        const localNodeModulesConfig = nodecommons_file_1.CommonsFile.rootRelativePath(`dist/queries`);
-        nodecommons_cli_1.CommonsOutput.debug(`Calculated global node_modules path for queries is ${localNodeModulesConfig}`);
-        if (nodecommons_file_1.CommonsFile.isDirectory(localNodeModulesConfig))
-            paths.push(localNodeModulesConfig);
-        const globalNodeModulesConfig = nodecommons_file_1.CommonsFile.rootRelativePath(`node_modules/hydra-crawler/dist/queries`);
-        nodecommons_cli_1.CommonsOutput.debug(`Calculated global node_modules path for queries is ${globalNodeModulesConfig}`);
-        if (nodecommons_file_1.CommonsFile.isDirectory(globalNodeModulesConfig))
-            paths.push(globalNodeModulesConfig);
-        const exists = paths
-            .filter((p) => nodecommons_file_1.CommonsFile.isDirectory(p));
-        if (exists.length === 0)
-            throw new Error('Unable to derive queries path');
-        nodecommons_cli_1.CommonsOutput.debug(`Selected path is: ${exists[0]}`);
-        const path = `${exists[0]}/${this.queryName}.query.js`;
-        nodecommons_cli_1.CommonsOutput.debug(`Query file path is: ${path}`);
-        if (!nodecommons_file_1.CommonsFile.isFile(path))
-            throw new Error('No such query exists');
-        const query = require(path).default;
-        await query(this.getArgs(), this.databaseService, this.lists, this.expiry, this.parsersConfig);
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const paths = [];
+            const cwdNodeModulesConfig = commonsFileCwdRelativeOrAbsolutePath('dist/queries');
+            commonsOutputDebug(`Calculated cwd node_modules path for queries is ${cwdNodeModulesConfig}`);
+            if (commonsFileIsDirectory(cwdNodeModulesConfig))
+                paths.push(cwdNodeModulesConfig);
+            const localNodeModulesConfig = commonsFileRootRelativePath('dist/queries');
+            commonsOutputDebug(`Calculated global node_modules path for queries is ${localNodeModulesConfig}`);
+            if (commonsFileIsDirectory(localNodeModulesConfig))
+                paths.push(localNodeModulesConfig);
+            const globalNodeModulesConfig = commonsFileRootRelativePath('node_modules/hydra-crawler/dist/queries');
+            commonsOutputDebug(`Calculated global node_modules path for queries is ${globalNodeModulesConfig}`);
+            if (commonsFileIsDirectory(globalNodeModulesConfig))
+                paths.push(globalNodeModulesConfig);
+            const exists = paths
+                .filter((p) => commonsFileIsDirectory(p));
+            if (exists.length === 0)
+                throw new Error('Unable to derive queries path');
+            commonsOutputDebug(`Selected path is: ${exists[0]}`);
+            const path = `${exists[0]}/${this.queryName}.query.js`;
+            commonsOutputDebug(`Query file path is: ${path}`);
+            if (!commonsFileIsFile(path))
+                throw new Error('No such query exists');
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-var-requires
+            const query = require(path).default;
+            yield query(this.getArgs(), this.databaseService, this.lists, this.expiry, this.parsersConfig);
+        });
     }
 }
-exports.QueryApp = QueryApp;
+//# sourceMappingURL=query.app.js.map

package/dist/apps/query.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.app.js","sourceRoot":"","sources":["../../src/apps/query.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAExE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,oCAAoC,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAC;AAEnJ,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAI3C,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAU9E,MAAM,OAAO,QAAS,SAAQ,UAAU;IAQvC,YACU,SAAiB;QAE1B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAAQ;QAI1B,IAAI,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEzE,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACpC,CAAC;IAEM,UAAU;QAChB,OAAO,eAAe,CAAC;IACxB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,KAAK,GAAa,EAAE,CAAC;YAE3B,MAAM,oBAAoB,GAAW,oCAAoC,CAAC,cAAc,CAAC,CAAC;YAC1F,kBAAkB,CAAC,mDAAmD,oBAAoB,EAAE,CAAC,CAAC;YAC9F,IAAI,sBAAsB,CAAC,oBAAoB,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAEnF,MAAM,sBAAsB,GAAW,2BAA2B,CAAC,cAAc,CAAC,CAAC;YACnF,kBAAkB,CAAC,sDAAsD,sBAAsB,EAAE,CAAC,CAAC;YACnG,IAAI,sBAAsB,CAAC,sBAAsB,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;YAEvF,MAAM,uBAAuB,GAAW,2BAA2B,CAAC,yCAAyC,CAAC,CAAC;YAC/G,kBAAkB,CAAC,sDAAsD,uBAAuB,EAAE,CAAC,CAAC;YACpG,IAAI,sBAAsB,CAAC,uBAAuB,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAEzF,MAAM,MAAM,GAAa,KAAK;iBAC3B,MAAM,CAAC,CAAC,CAAS,EAAW,EAAE,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC;YAE7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC1E,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAErD,MAAM,IAAI,GAAW,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,WAAW,CAAC;YAC/D,kBAAkB,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;YAElD,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAEtE,mJAAmJ;YACnJ,MAAM,KAAK,GAAW,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;YAE5C,MAAM,KAAK,CACT,IAAI,CAAC,OAAO,EAAE,EACd,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,aAAa,CACnB,CAAC;QACH,CAAC;KAAA;CACD"}

package/dist/apps/reattempt.app.d.ts

@@ -0,0 +1,17 @@
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+export declare class ReattemptApp extends CommonsApp implements IInternalHydraCommonListApp {
+    private include;
+    private databaseService;
+    private lists;
+    constructor(include: EStatus[]);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/reattempt.app.js

@@ -1,16 +1,22 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ReattemptApp = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const elist_1 = require("../enums/elist");
-class ReattemptApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { EList } from '../enums/elist';
+export class ReattemptApp extends CommonsApp {
     constructor(include) {
         super('hydra-crawler');
         this.include = include;
-        this.lists = new lists_1.Lists();
+        this.lists = new Lists();
     }
     getAppName() {
         return 'Hydra - Reattempt';
@@ -21,50 +27,57 @@ class ReattemptApp extends nodecommons_app_1.CommonsApp {
     addToList(list, entries) {
         this.lists.add(list, entries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        let tally = 0;
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for allowlisted ${this.include.join(' and ')}`);
-        const results = this.databaseService.getUrls()
-            .find({
-            status: { $in: this.include }
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            let tally = 0;
+            commonsOutputDoing(`Searching for allowlisted ${this.include.join(' and ')}`);
+            const results = this.databaseService.getUrls()
+                .find({
+                status: { $in: this.include }
+            }, {});
+            const urls = [];
+            tally = 0;
+            while (true) {
+                if ((tally % 100) === 0)
+                    commonsOutputProgress(tally);
+                const row = yield results.next();
+                if (row === null)
+                    break;
+                if (!this.lists.match(EList.ALLOW, row.url))
+                    continue;
+                urls.push(row.url);
+                tally++;
+            }
+            commonsOutputResult(tally);
+            commonsOutputDoing('Requeuing matches');
+            tally = 0;
+            for (const url of urls) {
+                tally++;
+                if ((tally % 10) === 0)
+                    commonsOutputProgress(tally);
+                yield this.databaseService.getUrls()
+                    .updateOne({ url: url }, {
+                    $set: { status: EStatus.QUEUED },
+                    $unset: { ttl: 1 }
+                });
+            }
+            commonsOutputResult(tally);
         });
-        const urls = [];
-        tally = 0;
-        while (true) {
-            if ((tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(tally);
-            const row = await results.next();
-            if (row === null)
-                break;
-            if (!this.lists.match(elist_1.EList.ALLOW, row.url))
-                continue;
-            urls.push(row.url);
-            tally++;
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
-        nodecommons_cli_1.CommonsOutput.doing(`Requeuing matches`);
-        tally = 0;
-        for (const url of urls) {
-            tally++;
-            if ((tally % 10) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(tally);
-            await this.databaseService.getUrls()
-                .updateOne({ url: url }, {
-                $set: { status: hydra_crawler_ts_assets_1.EStatus.QUEUED },
-                $unset: { ttl: 1 }
-            });
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
     }
 }
-exports.ReattemptApp = ReattemptApp;
+//# sourceMappingURL=reattempt.app.js.map

package/dist/apps/reattempt.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reattempt.app.js","sourceRoot":"","sources":["../../src/apps/reattempt.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,YAAa,SAAQ,UAAU;IAK3C,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9E,MAAM,OAAO,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACzD,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE;gBACZ,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAEtD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;aACL;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/requeue-domain.app.d.ts

@@ -0,0 +1,13 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class RequeueDomainApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private domain;
+    private done200Only;
+    private databaseService;
+    constructor(domain: string, done200Only: boolean);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/requeue-domain.app.js

@@ -1,10 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RequeueDomainApp = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-class RequeueDomainApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+export class RequeueDomainApp extends CommonsApp {
     constructor(domain, done200Only) {
         super('hydra-crawler');
         this.domain = domain;
@@ -16,36 +22,43 @@ class RequeueDomainApp extends nodecommons_app_1.CommonsApp {
     setDatabaseService(databaseService) {
         this.databaseService = databaseService;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const conditions = {
-            domain: this.domain,
-            status: { $in: [] }
-        };
-        if (this.done200Only) {
-            conditions.status.$in = [hydra_crawler_ts_assets_1.EStatus.DONE];
-            conditions.statusCode = 200;
-            nodecommons_cli_1.CommonsOutput.doing(`Marking all 200 DONE as QUEUED for ${this.domain}`);
-        }
-        else {
-            conditions.status.$in = [
-                hydra_crawler_ts_assets_1.EStatus.DONE,
-                hydra_crawler_ts_assets_1.EStatus.FAILED
-            ];
-            nodecommons_cli_1.CommonsOutput.doing(`Marking all DONE and FAILED as QUEUED for ${this.domain}`);
-        }
-        await this.databaseService.getUrls()
-            .updateMany(conditions, { $set: { status: hydra_crawler_ts_assets_1.EStatus.QUEUED } });
-        nodecommons_cli_1.CommonsOutput.success();
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const conditions = {
+                domain: this.domain,
+                status: { $in: [] }
+            };
+            if (this.done200Only) {
+                conditions.status.$in = [EStatus.DONE];
+                conditions.statusCode = 200;
+                commonsOutputDoing(`Marking all 200 DONE as QUEUED for ${this.domain}`);
+            }
+            else {
+                conditions.status.$in = [
+                    EStatus.DONE,
+                    EStatus.FAILED
+                ];
+                commonsOutputDoing(`Marking all DONE and FAILED as QUEUED for ${this.domain}`);
+            }
+            yield this.databaseService.getUrls()
+                .updateMany(conditions, { $set: { status: EStatus.QUEUED } });
+            commonsOutputSuccess();
+        });
     }
 }
-exports.RequeueDomainApp = RequeueDomainApp;
+//# sourceMappingURL=requeue-domain.app.js.map

package/dist/apps/requeue-domain.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requeue-domain.app.js","sourceRoot":"","sources":["../../src/apps/requeue-domain.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAG/C,YACU,MAAc,EACd,WAAoB;QAE7B,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAS;IAG9B,CAAC;IAEM,UAAU;QAChB,OAAO,wBAAwB,CAAC;IACjC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAMZ;gBACF,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,EAAE;gBACrB,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAE,OAAO,CAAC,IAAI,CAAE,CAAC;gBACzC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC;gBAE5B,kBAAkB,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aACxE;iBAAM;gBACN,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG;oBACtB,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,CAAC;gBAEF,kBAAkB,CAAC,6CAA6C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aAC/E;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,UAAU,EACR,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CACvC,CAAC;YAEJ,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/apps/seed.app.d.ts

@@ -0,0 +1,15 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+export declare class SeedApp extends CommonsApp implements IInternalHydraCommonListApp {
+    private databaseService;
+    private lists;
+    constructor();
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/seed.app.js

@@ -1,14 +1,20 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SeedApp = void 0;
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const elist_1 = require("../enums/elist");
-class SeedApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { EList } from '../enums/elist';
+export class SeedApp extends CommonsApp {
     constructor() {
         super('hydra-crawler');
-        this.lists = new lists_1.Lists();
+        this.lists = new Lists();
     }
     getAppName() {
         return 'Hydra - Seed';
@@ -19,38 +25,45 @@ class SeedApp extends nodecommons_app_1.CommonsApp {
     addToList(list, entries) {
         this.lists.add(list, entries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        nodecommons_cli_1.CommonsOutput.doing('Building allowlist hostname seeds');
-        const hostnames = this.lists.listHostnames(elist_1.EList.ALLOW);
-        const unique = [];
-        for (const hostname of hostnames) {
-            if (!unique.includes(hostname))
-                unique.push(`http://${hostname}/`);
-            if (!unique.includes(`www.${hostname}`))
-                unique.push(`http://www.${hostname}/`);
-            nodecommons_cli_1.CommonsOutput.progress(unique.length);
-        }
-        nodecommons_cli_1.CommonsOutput.result(unique.length);
-        nodecommons_cli_1.CommonsOutput.doing('Seeding into queue');
-        let tally = 0;
-        for (const url of unique) {
-            const outcome = await this.databaseService.queue(url);
-            if (outcome) {
-                tally++;
-                nodecommons_cli_1.CommonsOutput.progress(tally);
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            commonsOutputDoing('Building allowlist hostname seeds');
+            const hostnames = this.lists.listHostnames(EList.ALLOW);
+            const unique = [];
+            for (const hostname of hostnames) {
+                if (!unique.includes(hostname))
+                    unique.push(`http://${hostname}/`);
+                if (!unique.includes(`www.${hostname}`))
+                    unique.push(`http://www.${hostname}/`);
+                commonsOutputProgress(unique.length);
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
+            commonsOutputResult(unique.length);
+            commonsOutputDoing('Seeding into queue');
+            let tally = 0;
+            for (const url of unique) {
+                const outcome = yield this.databaseService.queue(url);
+                if (outcome) {
+                    tally++;
+                    commonsOutputProgress(tally);
+                }
+            }
+            commonsOutputResult(tally);
+        });
     }
 }
-exports.SeedApp = SeedApp;
+//# sourceMappingURL=seed.app.js.map

package/dist/apps/seed.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed.app.js","sourceRoot":"","sources":["../../src/apps/seed.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,OAAQ,SAAQ,UAAU;IAKtC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,cAAc,CAAC;IACvB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;YACxD,MAAM,SAAS,GAAa,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAElE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;gBACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,QAAQ,EAAE,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,GAAG,CAAC,CAAC;gBAChF,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACrC;YAED,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;gBACzB,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,qBAAqB,CAAC,KAAK,CAAC,CAAC;iBAC7B;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/startup.app.d.ts

@@ -0,0 +1,11 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class StartupApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private databaseService;
+    constructor();
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}