hydra-crawler 1.4.4 → 2.0.0

package/dist/apps/hydra.app.js

@@ -1,63 +1,69 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.HydraApp = void 0;
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_angular_1 = require("nodecommons-app-angular");
-const autocomplete_api_1 = require("../apis/autocomplete.api");
-const bugs_api_1 = require("../apis/bugs.api");
-const crawl_api_1 = require("../apis/crawl.api");
-const domains_api_1 = require("../apis/domains.api");
-const images_api_1 = require("../apis/images.api");
-const statistics_api_1 = require("../apis/statistics.api");
-const test_api_1 = require("../apis/test.api");
-const urls_api_1 = require("../apis/urls.api");
-const lists_1 = require("../classes/lists");
-const expiry_1 = require("../classes/expiry");
-const tracker_1 = require("../classes/tracker");
-const crawler_1 = require("../classes/crawler");
-const socket_io_server_1 = require("../servers/socket-io.server");
-const crawl_server_1 = require("../servers/crawl.server");
-const maintenance_server_1 = require("../servers/maintenance.server");
-const server_parser_1 = require("../parsers/server.parser");
-const hyperlinks_parser_1 = require("../parsers/hyperlinks.parser");
-const image_tags_parser_1 = require("../parsers/image-tags.parser");
-const jpeg_parser_1 = require("../parsers/jpeg.parser");
-const spelling_parser_1 = require("../parsers/spelling.parser");
-const bad_words_parser_1 = require("../parsers/bad-words.parser");
-const php_error_parser_1 = require("../parsers/php-error.parser");
-const asp_error_parser_1 = require("../parsers/asp-error.parser");
-const complex_english_parser_1 = require("../parsers/complex-english.parser");
-const accessibility_metrics_parser_1 = require("../parsers/accessibility-metrics.parser");
-const iparser_config_1 = require("../interfaces/iparser-config");
-const thydra_config_1 = require("../types/thydra-config");
-const tcrawl_config_1 = require("../types/tcrawl-config");
-const trobots_config_1 = require("../types/trobots-config");
-class HydraApp extends nodecommons_app_angular_1.CommonsAngularApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeIsTKeyObject } from 'tscommons-es-core';
+import { commonsOutputDebug, commonsOutputDoing, commonsOutputInfo, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsSocketIoApp } from 'nodecommons-es-app-socket-io';
+import { AutocompleteApi } from '../apis/autocomplete.api';
+import { BugsApi } from '../apis/bugs.api';
+import { CrawlApi } from '../apis/crawl.api';
+import { DomainsApi } from '../apis/domains.api';
+import { ImagesApi } from '../apis/images.api';
+import { StatisticsApi } from '../apis/statistics.api';
+import { TestApi } from '../apis/test.api';
+import { UrlsApi } from '../apis/urls.api';
+import { Lists } from '../classes/lists';
+import { Expiry } from '../classes/expiry';
+import { Tracker } from '../classes/tracker';
+import { Crawler } from '../classes/crawler';
+import { SocketIoServer } from '../servers/socket-io.server';
+import { CrawlServer } from '../servers/crawl.server';
+import { MaintenanceServer } from '../servers/maintenance.server';
+import { ServerParser } from '../parsers/server.parser';
+import { HyperlinksParser } from '../parsers/hyperlinks.parser';
+import { ImageTagsParser } from '../parsers/image-tags.parser';
+import { JpegParser } from '../parsers/jpeg.parser';
+import { SpellingParser } from '../parsers/spelling.parser';
+import { BadWordsParser } from '../parsers/bad-words.parser';
+import { PhpErrorParser } from '../parsers/php-error.parser';
+import { AspErrorParser } from '../parsers/asp-error.parser';
+import { ComplexEnglishParser } from '../parsers/complex-english.parser';
+import { AccessibilityMetricsParser } from '../parsers/accessibility-metrics.parser';
+import { isIParserConfig } from '../interfaces/iparser-config';
+import { isTHydraConfig } from '../types/thydra-config';
+import { isTCrawlConfig } from '../types/tcrawl-config';
+import { isTRobotsConfig } from '../types/trobots-config';
+export class HydraApp extends CommonsSocketIoApp {
     constructor() {
-        super('hydra-crawler', true);
+        super('hydra-crawler');
         this.maintenanceTimes = [];
-        this.lists = new lists_1.Lists();
-        this.expiry = new expiry_1.Expiry();
+        this.lists = new Lists();
+        this.expiry = new Expiry();
         const hydraConfig = this.getConfigArea('hydra');
-        if (!thydra_config_1.isTHydraConfig(hydraConfig))
+        if (!isTHydraConfig(hydraConfig))
             throw new Error('Hydra config is not valid');
         this.hydraConfig = hydraConfig;
         const crawlConfig = this.getConfigArea('crawl');
-        if (!tcrawl_config_1.isTCrawlConfig(crawlConfig))
+        if (!isTCrawlConfig(crawlConfig))
             throw new Error('Crawl config is not valid');
         this.crawlConfig = crawlConfig;
         const parsersConfig = this.getConfigArea('parsers');
-        if (!tscommons_core_1.CommonsType.isTKeyObject(parsersConfig, iparser_config_1.isIParserConfig))
+        if (!commonsTypeIsTKeyObject(parsersConfig, isIParserConfig))
             throw new Error('Parsers config is not valid');
         this.parsersConfig = parsersConfig;
         const robotsConfig = this.getConfigArea('robots');
-        if (!trobots_config_1.isTRobotsConfig(robotsConfig))
+        if (!isTRobotsConfig(robotsConfig))
             throw new Error('Robots config is not valid');
         this.robotsConfig = robotsConfig;
     }
     buildSocketIoServer(expressServer, expressConfig) {
-        return new socket_io_server_1.SocketIoServer(expressServer, expressConfig);
+        return new SocketIoServer(expressServer, true, expressConfig);
     }
     getAppName() {
         return 'Hydra';
@@ -74,107 +80,114 @@ class HydraApp extends nodecommons_app_angular_1.CommonsAngularApp {
     setMaintenanceSchedule(times) {
         this.maintenanceTimes = times;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        this.install((restServer, path) => {
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
             if (!this.databaseService)
-                throw new Error('Database service has not been defined yet');
-            nodecommons_cli_1.CommonsOutput.doing('Installing auto-complete API');
-            // tslint:disable-next-line:no-unused-expression
-            new autocomplete_api_1.AutocompleteApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing bugs API');
-            // tslint:disable-next-line:no-unused-expression
-            new bugs_api_1.BugsApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing crawl API');
-            // tslint:disable-next-line:no-unused-expression
-            new crawl_api_1.CrawlApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing domains API');
-            // tslint:disable-next-line:no-unused-expression
-            new domains_api_1.DomainsApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing images API');
-            // tslint:disable-next-line:no-unused-expression
-            new images_api_1.ImagesApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing statistics API');
-            // tslint:disable-next-line:no-unused-expression
-            new statistics_api_1.StatisticsApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing test API');
-            // tslint:disable-next-line:no-unused-expression
-            new test_api_1.TestApi(restServer, path);
-            nodecommons_cli_1.CommonsOutput.success();
-            nodecommons_cli_1.CommonsOutput.doing('Installing urls API');
-            // tslint:disable-next-line:no-unused-expression
-            new urls_api_1.UrlsApi(restServer, this.databaseService, path);
-            nodecommons_cli_1.CommonsOutput.success();
+                throw new Error('Database service has not been set yet');
+            this.install((restServer, path) => {
+                if (!this.databaseService)
+                    throw new Error('Database service has not been defined yet');
+                commonsOutputDoing('Installing auto-complete API');
+                new AutocompleteApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing bugs API');
+                new BugsApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing crawl API');
+                new CrawlApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing domains API');
+                new DomainsApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing images API');
+                new ImagesApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing statistics API');
+                new StatisticsApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing test API');
+                new TestApi(restServer, path);
+                commonsOutputSuccess();
+                commonsOutputDoing('Installing urls API');
+                new UrlsApi(restServer, this.databaseService, path);
+                commonsOutputSuccess();
+            });
+            yield _super.init.call(this);
+            const socketIoServer = this.getSocketIoServer();
+            if (!socketIoServer)
+                throw new Error('SocketIo server has not been set yet');
+            this.tracker = new Tracker(this.databaseService, socketIoServer);
+            this.crawlServer = new CrawlServer(this.databaseService, this.hydraConfig, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.lists, this.tracker);
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            for (const parser of [
+                ServerParser,
+                HyperlinksParser,
+                ImageTagsParser,
+                JpegParser,
+                SpellingParser,
+                BadWordsParser,
+                PhpErrorParser,
+                AspErrorParser,
+                ComplexEnglishParser,
+                AccessibilityMetricsParser
+            ]) {
+                commonsOutputDebug(`Installing parser ${parser.name}`);
+                this.crawlServer.addParser(parser);
+                yield this.databaseService.initParser(parser);
+            }
+            this.maintenanceServer = new MaintenanceServer(this.maintenanceTimes, this.expiry, this.lists, this.databaseService, this.crawlServer);
         });
-        await super.init();
-        const socketIoServer = this.getSocketIoServer();
-        if (!socketIoServer)
-            throw new Error('SocketIo server has not been set yet');
-        this.tracker = new tracker_1.Tracker(this.databaseService, socketIoServer);
-        this.crawlServer = new crawl_server_1.CrawlServer(this.databaseService, this.hydraConfig, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.lists, this.tracker);
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        for (const parser of [
-            server_parser_1.ServerParser,
-            hyperlinks_parser_1.HyperlinksParser,
-            image_tags_parser_1.ImageTagsParser,
-            jpeg_parser_1.JpegParser,
-            spelling_parser_1.SpellingParser,
-            bad_words_parser_1.BadWordsParser,
-            php_error_parser_1.PhpErrorParser,
-            asp_error_parser_1.AspErrorParser,
-            complex_english_parser_1.ComplexEnglishParser,
-            accessibility_metrics_parser_1.AccessibilityMetricsParser
-        ]) {
-            nodecommons_cli_1.CommonsOutput.debug(`Installing parser ${parser.name}`);
-            this.crawlServer.addParser(parser);
-            this.databaseService.initParser(parser);
-        }
-        this.maintenanceServer = new maintenance_server_1.MaintenanceServer(this.maintenanceTimes, this.expiry, this.lists, this.databaseService, this.crawlServer);
     }
     listening() {
-        (async () => {
+        void (() => __awaiter(this, void 0, void 0, function* () {
             if (!this.crawlServer)
                 throw new Error('No crawl server instantiated! This should not be possible');
-            await this.crawlServer.start();
+            yield this.crawlServer.start();
             this.abort();
-        })();
+        }))();
     }
-    async run() {
-        const singleUrl = this.getArgs().getStringOrUndefined('single-url');
-        if (singleUrl) {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            if (!this.crawlServer)
-                throw new Error('Crawl server has not been set yet');
-            const whatwg = new URL(singleUrl);
-            const crawler = new crawler_1.Crawler(whatwg.hostname, this.databaseService, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.crawlServer.listParsers(), this.lists);
-            nodecommons_cli_1.CommonsOutput.info(`Running single crawl of: ${singleUrl}`);
-            const outcome = await crawler.fetch(singleUrl, false);
-            nodecommons_cli_1.CommonsOutput.info(`HTTP status code outcome was ${outcome}`);
-            return;
-        }
-        if (!this.tracker)
-            throw new Error('Tracker has not been defined! This should not be possible');
-        if (!this.maintenanceServer)
-            throw new Error('Maintenance server has not been defined! This should not be possible');
-        this.maintenanceServer.start();
-        this.tracker.start();
-        await super.run();
+    run() {
+        const _super = Object.create(null, {
+            run: { get: () => super.run }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            const singleUrl = this.getArgs().getStringOrUndefined('single-url');
+            if (singleUrl) {
+                if (!this.databaseService)
+                    throw new Error('Database service has not been set yet');
+                if (!this.crawlServer)
+                    throw new Error('Crawl server has not been set yet');
+                const whatwg = new URL(singleUrl);
+                const crawler = new Crawler(whatwg.hostname, this.databaseService, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.crawlServer.listParsers(), this.lists);
+                commonsOutputInfo(`Running single crawl of: ${singleUrl}`);
+                const outcome = yield crawler.fetch(singleUrl, false);
+                commonsOutputInfo(`HTTP status code outcome was ${outcome}`);
+                return;
+            }
+            if (!this.tracker)
+                throw new Error('Tracker has not been defined! This should not be possible');
+            if (!this.maintenanceServer)
+                throw new Error('Maintenance server has not been defined! This should not be possible');
+            this.maintenanceServer.start();
+            void this.tracker.start();
+            yield _super.run.call(this);
+        });
     }
-    async shutdown() {
-        if (!this.crawlServer)
-            throw new Error('No crawl server instantiated! This should not be possible');
-        await this.crawlServer.shutdown();
-        await super.shutdown();
+    shutdown() {
+        const _super = Object.create(null, {
+            shutdown: { get: () => super.shutdown }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.crawlServer)
+                throw new Error('No crawl server instantiated! This should not be possible');
+            yield this.crawlServer.shutdown();
+            yield _super.shutdown.call(this);
+        });
     }
 }
-exports.HydraApp = HydraApp;
+//# sourceMappingURL=hydra.app.js.map

package/dist/apps/hydra.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGxE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAIrH,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AAErF,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,IAAI,EACJ,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAA6B,EAAE,IAAY,EAAQ,EAAE;gBACrD,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;aAC3B,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,CACjB,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/apps/import.app.d.ts

@@ -0,0 +1,11 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class ImportApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private databaseService;
+    constructor();
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/import.app.js

@@ -1,10 +1,15 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ImportApp = void 0;
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_cli_2 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-class ImportApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsInputReadAllStdIn, commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+export class ImportApp extends CommonsApp {
     constructor() {
         super('hydra-crawler');
     }
@@ -14,32 +19,39 @@ class ImportApp extends nodecommons_app_1.CommonsApp {
     setDatabaseService(databaseService) {
         this.databaseService = databaseService;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        nodecommons_cli_1.CommonsOutput.doing('Loading URLs from stdin');
-        const urls = await nodecommons_cli_2.CommonsInput.readAllStdIn();
-        nodecommons_cli_1.CommonsOutput.result(urls.length);
-        nodecommons_cli_1.CommonsOutput.doing('Seeding into queue');
-        let tally = 0;
-        let done = 0;
-        for (const url of urls) {
-            if ((++tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}, imported ${done}`);
-            const outcome = await this.databaseService.queue(url);
-            if (outcome) {
-                done++;
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            commonsOutputDoing('Loading URLs from stdin');
+            const urls = yield commonsInputReadAllStdIn();
+            commonsOutputResult(urls.length);
+            commonsOutputDoing('Seeding into queue');
+            let tally = 0;
+            let done = 0;
+            for (const url of urls) {
+                if ((++tally % 100) === 0)
+                    commonsOutputProgress(`${tally}, imported ${done}`);
+                const outcome = yield this.databaseService.queue(url);
+                if (outcome) {
+                    done++;
+                }
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(done);
+            commonsOutputResult(done);
+        });
     }
 }
-exports.ImportApp = ImportApp;
+//# sourceMappingURL=import.app.js.map

package/dist/apps/import.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.app.js","sourceRoot":"","sources":["../../src/apps/import.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACpJ,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAGxC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,yBAAyB,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAa,MAAM,wBAAwB,EAAE,CAAC;YACxD,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,IAAI,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,CAAC,CAAC;gBAE/E,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE;oBACZ,IAAI,EAAE,CAAC;iBACP;aACD;YACD,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;KAAA;CACD"}

package/dist/apps/internal-hydra-common.app.d.ts

@@ -0,0 +1,28 @@
+import { TCommonsScheduleTime } from 'tscommons-es-async';
+import { CommonsConfigFile } from 'nodecommons-es-config';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { IExpiry } from '../interfaces/iexpiry';
+import { EList } from '../enums/elist';
+export interface IInternalHydraCommonApp {
+    loadConfigFile(file: string): CommonsConfigFile;
+    loadRawJsonConfigFile(file: string): unknown;
+    start(): Promise<void>;
+    getAppName(): string;
+}
+export interface IInternalHydraCommonDbApp extends IInternalHydraCommonApp {
+    setDatabaseService(databaseService: DatabaseService): void;
+}
+export declare function isIInternalHydraCommonDbApp(test: IInternalHydraCommonApp): test is IInternalHydraCommonDbApp;
+export interface IInternalHydraCommonListApp extends IInternalHydraCommonDbApp {
+    addToList(list: EList, entries: IMatch[]): void;
+}
+export declare function isIInternalHydraCommonListApp(test: IInternalHydraCommonApp): test is IInternalHydraCommonListApp;
+export interface IInternalHydraCommonExpiryApp extends IInternalHydraCommonDbApp {
+    addToExpiry(expiries: IExpiry[]): void;
+}
+export declare function isIInternalHydraCommonExpiryApp(test: IInternalHydraCommonApp): test is IInternalHydraCommonExpiryApp;
+export interface IInternalHydraCommonMaintenanceApp extends IInternalHydraCommonDbApp {
+    setMaintenanceSchedule(times: TCommonsScheduleTime[]): void;
+}
+export declare function isIInternalHydraCommonMaintenanceApp(test: IInternalHydraCommonApp): test is IInternalHydraCommonMaintenanceApp;

package/dist/apps/internal-hydra-common.app.js

@@ -1,25 +1,19 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isIInternalHydraCommonMaintenanceApp = exports.isIInternalHydraCommonExpiryApp = exports.isIInternalHydraCommonListApp = exports.isIInternalHydraCommonDbApp = void 0;
-function isIInternalHydraCommonDbApp(test) {
+export function isIInternalHydraCommonDbApp(test) {
     return 'setDatabaseService' in test;
 }
-exports.isIInternalHydraCommonDbApp = isIInternalHydraCommonDbApp;
-function isIInternalHydraCommonListApp(test) {
+export function isIInternalHydraCommonListApp(test) {
     if (!isIInternalHydraCommonDbApp(test))
         return false;
     return 'addToList' in test;
 }
-exports.isIInternalHydraCommonListApp = isIInternalHydraCommonListApp;
-function isIInternalHydraCommonExpiryApp(test) {
+export function isIInternalHydraCommonExpiryApp(test) {
     if (!isIInternalHydraCommonDbApp(test))
         return false;
     return 'addToExpiry' in test;
 }
-exports.isIInternalHydraCommonExpiryApp = isIInternalHydraCommonExpiryApp;
-function isIInternalHydraCommonMaintenanceApp(test) {
+export function isIInternalHydraCommonMaintenanceApp(test) {
     if (!isIInternalHydraCommonDbApp(test))
         return false;
     return 'setMaintenanceSchedule' in test;
 }
-exports.isIInternalHydraCommonMaintenanceApp = isIInternalHydraCommonMaintenanceApp;
+//# sourceMappingURL=internal-hydra-common.app.js.map

package/dist/apps/internal-hydra-common.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"internal-hydra-common.app.js","sourceRoot":"","sources":["../../src/apps/internal-hydra-common.app.ts"],"names":[],"mappings":"AAyBA,MAAM,UAAU,2BAA2B,CAAC,IAA6B;IACxE,OAAO,oBAAoB,IAAI,IAAI,CAAC;AACrC,CAAC;AAQD,MAAM,UAAU,6BAA6B,CAAC,IAA6B;IAC1E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,WAAW,IAAI,IAAI,CAAC;AAC5B,CAAC;AAKD,MAAM,UAAU,+BAA+B,CAAC,IAA6B;IAC5E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,aAAa,IAAI,IAAI,CAAC;AAC9B,CAAC;AAKD,MAAM,UAAU,oCAAoC,CAAC,IAA6B;IACjF,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,wBAAwB,IAAI,IAAI,CAAC;AACzC,CAAC"}

package/dist/apps/query.app.d.ts

@@ -0,0 +1,20 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { IExpiry } from '../interfaces/iexpiry';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class QueryApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private queryName;
+    private databaseService;
+    private lists;
+    private expiry;
+    private parsersConfig;
+    constructor(queryName: string);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    addToExpiry(expiries: IExpiry[]): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}