hydra-crawler 1.4.5 → 2.0.1

package/dist/queries/linking-to-domains.query.js

@@ -1,10 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_2 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_3 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_4 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { isIUrl } from 'hydra-crawler-ts-assets';
+import { isTLink } from 'hydra-crawler-ts-assets';
+import { isTDomain } from 'hydra-crawler-ts-assets';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
 const domainSort = (a, b) => {
     if (a.startsWith('http://'))
         a = a.substr(7);
@@ -24,31 +31,31 @@ const domainSort = (a, b) => {
         return 1;
     return 0;
 };
-const query = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+export const query = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
     const regexStr = args.getString('regex');
     if (!regexStr) {
-        nodecommons_cli_1.CommonsOutput.error('No domain regex specified');
+        commonsOutputError('No domain regex specified');
         return;
     }
     const regex = new RegExp(regexStr);
-    nodecommons_cli_1.CommonsOutput.doing('Searching for matching domains');
+    commonsOutputDoing('Searching for matching domains');
     const results = databaseService.getDomains().find({
         domain: regex
-    });
-    const domains = await databaseService.listQueryResults(results, hydra_crawler_ts_assets_3.isTDomain);
-    nodecommons_cli_1.CommonsOutput.result(domains.length);
+    }, {});
+    const domains = yield databaseService.listQueryResults(results, isTDomain);
+    commonsOutputResult(domains.length);
     const domainUrls = new Map();
     for (const domain of domains) {
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for URLs for ${domain.domain}`);
+        commonsOutputDoing(`Searching for URLs for ${domain.domain}`);
         const results2 = databaseService.getUrls().find({
             domain: domain.domain,
-            status: { $ne: hydra_crawler_ts_assets_4.EStatus.ARCHIVED }
-        });
-        const urls = await databaseService.listQueryResults(results2, hydra_crawler_ts_assets_1.isIUrl);
-        nodecommons_cli_1.CommonsOutput.result(urls.length);
+            status: { $ne: EStatus.ARCHIVED }
+        }, {});
+        const urls = yield databaseService.listQueryResults(results2, isIUrl);
+        commonsOutputResult(urls.length);
         domainUrls.set(domain, urls);
     }
-    nodecommons_cli_1.CommonsOutput.doing(`Searching for outgoing links to URLs`);
+    commonsOutputDoing('Searching for outgoing links to URLs');
     let tally = 0;
     const domainUrlLinks = new Map();
     for (const domain of domains) {
@@ -59,20 +66,20 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
         for (const url of urls) {
             const results3 = databaseService.getLinks().find({
                 outgoing: url.url
-            });
-            const links = await databaseService.listQueryResults(results3, hydra_crawler_ts_assets_2.isTLink);
+            }, {});
+            const links = yield databaseService.listQueryResults(results3, isTLink);
             if (links.length === 0)
                 continue;
             tally += links.length;
-            nodecommons_cli_1.CommonsOutput.progress(tally);
+            commonsOutputProgress(tally);
             map.set(url, links);
         }
         if (map.size === 0)
             continue;
         domainUrlLinks.set(domain, map);
     }
-    nodecommons_cli_1.CommonsOutput.result(tally);
-    nodecommons_cli_1.CommonsOutput.doing(`Constructing JSON object`);
+    commonsOutputResult(tally);
+    commonsOutputDoing('Constructing JSON object');
     const outcome = {};
     for (const domain of domainUrlLinks.keys()) {
         outcome[domain.domain] = {};
@@ -81,7 +88,7 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
                 .map((link) => link.url);
         }
     }
-    nodecommons_cli_1.CommonsOutput.success();
+    commonsOutputSuccess();
     if (args.hasAttribute('domains-only')) {
         Object.keys(outcome)
             .sort(domainSort)
@@ -119,5 +126,6 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
     else if (args.hasAttribute('json-dump')) {
         console.log(JSON.stringify(outcome));
     }
-};
-exports.default = query;
+});
+// export default query;
+//# sourceMappingURL=linking-to-domains.query.js.map

package/dist/queries/linking-to-domains.query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAoB,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAChE;QACE,MAAM,EAAE,KAAK;KACd,EACD,EAAE,CACH,CAAC;IACF,MAAM,OAAO,GAAc,MAAM,eAAe,CAAC,gBAAgB,CAC/D,OAAO,EACP,SAAS,CACV,CAAC;IACF,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC,MAAM,UAAU,GAAyB,IAAI,GAAG,EAAmB,CAAC;IACpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,kBAAkB,CAAC,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAiB,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3D;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;SAClC,EACD,EAAE,CACH,CAAC;QACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,QAAQ,EACR,MAAM,CACP,CAAC;QACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;KAC7B;IAED,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,cAAc,GAAqC,IAAI,GAAG,EAA+B,CAAC;IAChG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,MAAM,IAAI,GAAqB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAuB,IAAI,GAAG,EAAiB,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,MAAM,QAAQ,GAAkB,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAC7D;gBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;aAClB,EACD,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE7B,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACpB;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAE7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE;QAC3C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE5B,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,EAAE;YACrD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAE;iBACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAC1C;KACD;IACD,oBAAoB,EAAE,CAAC;IAEvB,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE;QACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,CAAC,CAAC,MAAc,EAAQ,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;KACL;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,UAAU,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEtE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE;gBAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACxE;aACD;SACD;QAED,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aACjD,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEtC,IAAI,KAAK,GAAY,IAAI,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE;gBACvC,MAAM,GAAG,GAAa;oBACpB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChB,GAAG;iBACJ,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE5B,KAAK,GAAG,KAAK,CAAC;aACd;SACD;KACD;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;KACrC;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}

package/dist/queries/readability-score.query.d.ts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery';
+export declare const query: TQuery;

package/dist/queries/readability-score.query.js

@@ -1,17 +1,24 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const query = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult } from 'nodecommons-es-cli';
+export const query = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
     const domain = args.getString('domain');
     if (!domain) {
-        nodecommons_cli_1.CommonsOutput.error('No domain specified');
+        commonsOutputError('No domain specified');
         return;
     }
-    nodecommons_cli_1.CommonsOutput.doing(`Scoring and ordering readability of DONE URLs for domain ${domain}`);
+    commonsOutputDoing(`Scoring and ordering readability of DONE URLs for domain ${domain}`);
     const results = databaseService.getUrls().aggregate([
         { $match: {
-                status: { $ne: hydra_crawler_ts_assets_1.EStatus.ARCHIVED },
+                status: { $ne: EStatus.ARCHIVED },
                 domain: domain,
                 'headers.content-type': 'text/html'
             } },
@@ -57,16 +64,17 @@ const query = async (args, databaseService, _lists, _expiry, _parsersConfig) =>
     while (true) {
         tally++;
         if ((tally % 100) === 0)
-            nodecommons_cli_1.CommonsOutput.progress(`${tally}`);
-        const row = await results.next();
+            commonsOutputProgress(`${tally}`);
+        const row = yield results.next();
         if (row === null)
             break;
         matches.push(row);
     }
-    nodecommons_cli_1.CommonsOutput.result(tally);
+    commonsOutputResult(tally);
     console.log('url\tparagraphs\tsentences\twords\tfkre\tari\tscore');
     for (const match of matches) {
         console.log(`${match.url}\t${match.stats.paragraphs}\t${match.stats.sentences}\t${match.stats.words}\t${match.stats.fkre}\t${match.stats.ari}\t${match.score}`);
     }
-};
-exports.default = query;
+});
+// export default query;
+//# sourceMappingURL=readability-score.query.js.map

package/dist/queries/readability-score.query.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability-score.query.js","sourceRoot":"","sources":["../../src/queries/readability-score.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAuBrI,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,kBAAkB,CAAC,4DAA4D,MAAM,EAAE,CAAC,CAAC;IAEzF,MAAM,OAAO,GAA+B,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAU;QACvF,EAAE,MAAM,EAAE;gBACR,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,sBAAsB,EAAE,WAAW;aACpC,EAAE;QACH,EAAE,MAAM,EAAE;gBACR,2BAA2B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;aAC3C,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE;oBACL,UAAU,EAAE,kCAAkC;oBAC9C,SAAS,EAAE,iCAAiC;oBAC5C,KAAK,EAAE,6BAA6B;oBACpC,IAAI,EAAE,4BAA4B;oBAClC,GAAG,EAAE,2BAA2B;iBACjC;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,CAAE,EAAE;oBACtD,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,EAAE,EAAE,YAAY,CAAE,EAAE,CAAE,EAAE;iBACpD;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE;oBACR,IAAI,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,EAAE,GAAG,CAAE,EAAE;oBACjE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,YAAY,EAAE,CAAC,CAAE,EAAE,EAAE,EAAE,CAAE,EAAE;iBAC7D;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,IAAI,EAAE,CAAE,gBAAgB,EAAE,eAAe,CAAE,EAAE,EAAE,CAAC,CAAE,EAAE;aAC3E,EAAE;QACH,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KAClB;IAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;KAChK;AACF,CAAC,CAAA,CAAC;AAEF,wBAAwB"}

package/dist/servers/crawl.server.d.ts

@@ -0,0 +1,35 @@
+import { TKeyObject } from 'tscommons-es-core';
+import { Lists } from '../classes/lists';
+import { Tracker } from '../classes/tracker';
+import { DatabaseService } from '../services/database.service';
+import { IParserConfig } from '../interfaces/iparser-config';
+import { THydraConfig } from '../types/thydra-config';
+import { TCrawlConfig } from '../types/tcrawl-config';
+import { TRobotsConfig } from '../types/trobots-config';
+import { TParserCtor } from '../types/tparser-ctor';
+export declare class CrawlServer {
+    private database;
+    private hydraConfig;
+    private crawlConfig;
+    private parsersConfig;
+    private robotsConfig;
+    private lists;
+    private tracker;
+    private parsers;
+    private crawlers;
+    private strategies;
+    private isAborted;
+    private preDelayIds;
+    private isPaused;
+    constructor(database: DatabaseService, hydraConfig: THydraConfig, crawlConfig: TCrawlConfig, parsersConfig: TKeyObject<IParserConfig>, robotsConfig: TRobotsConfig, lists: Lists, tracker: Tracker);
+    addParser(parser: TParserCtor): void;
+    listParsers(): TParserCtor[];
+    pause(): void;
+    resume(): void;
+    private abort;
+    terminate(): Promise<void>;
+    private pauseCrawl;
+    private resumeCrawl;
+    start(): Promise<void>;
+    shutdown(): Promise<void>;
+}

package/dist/servers/crawl.server.js

@@ -1,13 +1,19 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CrawlServer = void 0;
-const tscommons_async_1 = require("tscommons-async");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_process_1 = require("nodecommons-process");
-const crawler_1 = require("../classes/crawler");
-const eavailable_strategy_1 = require("../enums/eavailable-strategy");
-class CrawlServer {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-es-async';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputAlert, commonsOutputCompleted, commonsOutputDebug, commonsOutputDoing, commonsOutputProgress, commonsOutputStarting, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { commonsGracefulAbortAddCallback } from 'nodecommons-es-process';
+import { Crawler } from '../classes/crawler';
+import { EAvailableStrategy } from '../enums/eavailable-strategy';
+export class CrawlServer {
     constructor(database, hydraConfig, crawlConfig, parsersConfig, robotsConfig, lists, tracker) {
         this.database = database;
         this.hydraConfig = hydraConfig;
@@ -22,11 +28,11 @@ class CrawlServer {
         this.isPaused = false;
         this.crawlers = new Map();
         this.strategies = new Map();
-        nodecommons_process_1.CommonsGracefulAbort.addCallback(() => {
-            nodecommons_cli_1.CommonsOutput.alert(`SIGINT abort flag is set. Aborting crawl server.`);
+        commonsGracefulAbortAddCallback(() => {
+            commonsOutputAlert('SIGINT abort flag is set. Aborting crawl server.');
             this.abort();
             for (const preDelayId of this.preDelayIds)
-                tscommons_async_1.CommonsAsync.abortTimeout(preDelayId);
+                commonsAsyncAbortTimeout(preDelayId);
         });
     }
     addParser(parser) {
@@ -36,14 +42,14 @@ class CrawlServer {
         return this.parsers.slice();
     }
     pause() {
-        nodecommons_cli_1.CommonsOutput.alert(`Pausing crawl server`);
+        commonsOutputAlert('Pausing crawl server');
         this.isPaused = true;
         for (const domain of this.crawlers.keys()) {
             this.pauseCrawl(domain);
         }
     }
     resume() {
-        nodecommons_cli_1.CommonsOutput.completed(`Resuming crawl server`);
+        commonsOutputCompleted('Resuming crawl server');
         this.isPaused = false;
         for (const domain of this.crawlers.keys()) {
             this.resumeCrawl(domain);
@@ -51,132 +57,138 @@ class CrawlServer {
     }
     abort() {
         this.isAborted = true;
-        tscommons_async_1.CommonsAsync.abortTimeout('find-new-available');
+        commonsAsyncAbortTimeout('find-new-available');
     }
-    async terminate() {
-        this.abort();
-        await this.database.close();
-        process.exit(0);
+    terminate() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.abort();
+            yield this.database.close();
+            process.exit(0);
+        });
     }
-    async pauseCrawl(domain) {
+    pauseCrawl(domain) {
         if (!this.crawlers.has(domain))
             return;
-        nodecommons_cli_1.CommonsOutput.alert(`Pausing crawler for ${domain}`);
+        commonsOutputAlert(`Pausing crawler for ${domain}`);
         this.crawlers.get(domain).pause();
     }
-    async resumeCrawl(domain) {
+    resumeCrawl(domain) {
         if (!this.crawlers.has(domain))
             return;
-        nodecommons_cli_1.CommonsOutput.completed(`Resuming crawler for ${domain}`);
+        commonsOutputCompleted(`Resuming crawler for ${domain}`);
         this.crawlers.get(domain).resume();
     }
-    async start() {
-        await this.database.resetActive();
-        const added = await this.database.queue(this.hydraConfig.startUrl);
-        if (added)
-            this.tracker.delta(hydra_crawler_ts_assets_1.EStatus.QUEUED, 1);
-        while (!this.isAborted) {
-            if (!this.isPaused) {
-                const existing = [...this.crawlers.keys()];
-                const space = this.crawlConfig.maxCrawlers - existing.length;
-                if (space > 0) {
-                    nodecommons_cli_1.CommonsOutput.debug(`Space for ${space} new crawls available`);
-                    const largestStrategyDomains = existing
-                        .filter((domain) => this.strategies.get(domain) === eavailable_strategy_1.EAvailableStrategy.LARGEST);
-                    const smallestStrategyDomains = existing
-                        .filter((domain) => this.strategies.get(domain) === eavailable_strategy_1.EAvailableStrategy.SMALLEST);
-                    nodecommons_cli_1.CommonsOutput.debug(`Current crawl has LARGEST=${largestStrategyDomains.length}; SMALLEST=${smallestStrategyDomains.length}`);
-                    let spaceForLargestDomains = 0;
-                    let spaceForSmallestDomains = space;
-                    if (largestStrategyDomains.length >= Math.floor(this.crawlConfig.maxCrawlers / 2)) {
-                        // already 50% large crawling, so all remaining space goes to small crawls
-                    }
-                    else {
-                        // allow a new large crawl, if enough space
-                        if (space > 1) {
-                            spaceForLargestDomains = 1;
-                            spaceForSmallestDomains--;
+    start() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.database.resetActive();
+            const added = yield this.database.queue(this.hydraConfig.startUrl);
+            if (added)
+                this.tracker.delta(EStatus.QUEUED, 1);
+            while (!this.isAborted) {
+                if (!this.isPaused) {
+                    const existing = [...this.crawlers.keys()];
+                    const space = this.crawlConfig.maxCrawlers - existing.length;
+                    if (space > 0) {
+                        commonsOutputDebug(`Space for ${space} new crawls available`);
+                        const largestStrategyDomains = existing
+                            .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.LARGEST);
+                        const smallestStrategyDomains = existing
+                            .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.SMALLEST);
+                        commonsOutputDebug(`Current crawl has LARGEST=${largestStrategyDomains.length}; SMALLEST=${smallestStrategyDomains.length}`);
+                        let spaceForLargestDomains = 0;
+                        let spaceForSmallestDomains = space;
+                        if (largestStrategyDomains.length >= Math.floor(this.crawlConfig.maxCrawlers / 2)) {
+                            // already 50% large crawling, so all remaining space goes to small crawls
                         }
-                    }
-                    nodecommons_cli_1.CommonsOutput.debug(`Space for LARGEST=${spaceForLargestDomains}; SMALLEST=${spaceForSmallestDomains}`);
-                    const availablesLargest = await this.database.available(eavailable_strategy_1.EAvailableStrategy.LARGEST, this.crawlConfig.availableStrategyThreshold, spaceForLargestDomains, existing);
-                    existing.push(...availablesLargest);
-                    const availablesSmallest = await this.database.available(eavailable_strategy_1.EAvailableStrategy.SMALLEST, this.crawlConfig.availableStrategyThreshold, spaceForSmallestDomains, existing);
-                    const availables = [
-                        ...availablesSmallest,
-                        ...availablesLargest
-                    ];
-                    if (!this.isAborted) {
-                        for (const domain of availables) {
-                            nodecommons_cli_1.CommonsOutput.starting(`Creating new crawl head for ${domain}`);
-                            const crawler = new crawler_1.Crawler(domain, this.database, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.parsers, this.lists, this.tracker);
-                            this.crawlers.set(domain, crawler);
-                            if (availablesLargest.includes(domain)) {
-                                this.strategies.set(domain, eavailable_strategy_1.EAvailableStrategy.LARGEST);
+                        else {
+                            // allow a new large crawl, if enough space
+                            if (space > 1) {
+                                spaceForLargestDomains = 1;
+                                spaceForSmallestDomains--;
                             }
-                            else {
-                                this.strategies.set(domain, eavailable_strategy_1.EAvailableStrategy.SMALLEST);
-                            }
-                            // called without await in order to do parallel crawls
-                            (async () => {
-                                try {
-                                    // delay randomly to prevent network socket request spikes
-                                    const preDelayId = `predelay_${domain}`;
-                                    this.preDelayIds.push(preDelayId);
+                        }
+                        commonsOutputDebug(`Space for LARGEST=${spaceForLargestDomains}; SMALLEST=${spaceForSmallestDomains}`);
+                        const availablesLargest = yield this.database.available(EAvailableStrategy.LARGEST, this.crawlConfig.availableStrategyThreshold, spaceForLargestDomains, existing);
+                        existing.push(...availablesLargest);
+                        const availablesSmallest = yield this.database.available(EAvailableStrategy.SMALLEST, this.crawlConfig.availableStrategyThreshold, spaceForSmallestDomains, existing);
+                        const availables = [
+                            ...availablesSmallest,
+                            ...availablesLargest
+                        ];
+                        if (!this.isAborted) {
+                            for (const domain of availables) {
+                                commonsOutputStarting(`Creating new crawl head for ${domain}`);
+                                const crawler = new Crawler(domain, this.database, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.parsers, this.lists, this.tracker);
+                                this.crawlers.set(domain, crawler);
+                                if (availablesLargest.includes(domain)) {
+                                    this.strategies.set(domain, EAvailableStrategy.LARGEST);
+                                }
+                                else {
+                                    this.strategies.set(domain, EAvailableStrategy.SMALLEST);
+                                }
+                                // called without await in order to do parallel crawls
+                                void (() => __awaiter(this, void 0, void 0, function* () {
                                     try {
-                                        await tscommons_async_1.CommonsAsync.timeout(Math.random() * (this.crawlConfig.findNewAvailableDelay - 1000), preDelayId);
-                                    }
-                                    catch (e) {
-                                        if (e.message === 'abortTimeout called')
+                                        // delay randomly to prevent network socket request spikes
+                                        const preDelayId = `predelay_${domain}`;
+                                        this.preDelayIds.push(preDelayId);
+                                        try {
+                                            yield commonsAsyncTimeout(Math.random() * (this.crawlConfig.findNewAvailableDelay - 1000), preDelayId);
+                                        }
+                                        catch (e) {
+                                            if (e.message === 'abortTimeout called')
+                                                return;
+                                            throw e;
+                                        }
+                                        this.preDelayIds = this.preDelayIds
+                                            .filter((pid) => pid !== preDelayId);
+                                        if (this.isAborted)
                                             return;
-                                        throw e;
+                                        if (this.isPaused)
+                                            this.pauseCrawl(domain);
+                                        commonsOutputStarting(`Starting crawler for ${domain}`);
+                                        yield crawler.crawl();
                                     }
-                                    this.preDelayIds = this.preDelayIds
-                                        .filter((pid) => pid !== preDelayId);
-                                    if (this.isAborted)
-                                        return;
-                                    if (this.isPaused)
-                                        this.pauseCrawl(domain);
-                                    nodecommons_cli_1.CommonsOutput.starting(`Starting crawler for ${domain}`);
-                                    await crawler.crawl();
-                                }
-                                catch (ex) {
-                                    nodecommons_cli_1.CommonsOutput.debug('debug position 6');
-                                    console.log(ex);
-                                }
-                                finally {
-                                    this.crawlers.delete(domain); // doesn't get called until after doCrawl() is called, so have to do it here if aborted
-                                    this.strategies.delete(domain);
-                                }
-                            })();
+                                    catch (ex) {
+                                        commonsOutputDebug('debug position 6');
+                                        console.log(ex);
+                                    }
+                                    finally {
+                                        this.crawlers.delete(domain); // doesn't get called until after doCrawl() is called, so have to do it here if aborted
+                                        this.strategies.delete(domain);
+                                    }
+                                }))();
+                            }
                         }
                     }
                 }
+                try {
+                    yield commonsAsyncTimeout(this.crawlConfig.findNewAvailableDelay, 'find-new-available');
+                }
+                catch (ex) {
+                    // ignore
+                }
+                if (this.isAborted) {
+                    commonsOutputAlert('SIGINT abort is set. Aborting new head loop.');
+                    break;
+                }
             }
-            try {
-                await tscommons_async_1.CommonsAsync.timeout(this.crawlConfig.findNewAvailableDelay, 'find-new-available');
-            }
-            catch (ex) {
-                // ignore
-            }
-            if (this.isAborted) {
-                nodecommons_cli_1.CommonsOutput.alert(`SIGINT abort is set. Aborting new head loop.`);
-                break;
-            }
-        }
+        });
     }
-    async shutdown() {
-        nodecommons_cli_1.CommonsOutput.doing('Waiting for all crawlers to abort');
-        while (this.crawlers.size > 0) {
-            nodecommons_cli_1.CommonsOutput.progress(this.crawlers.size);
-            try {
-                await tscommons_async_1.CommonsAsync.timeout(1000);
-            }
-            catch (ex) {
-                // ignore
+    shutdown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            commonsOutputDoing('Waiting for all crawlers to abort');
+            while (this.crawlers.size > 0) {
+                commonsOutputProgress(this.crawlers.size);
+                try {
+                    yield commonsAsyncTimeout(1000);
+                }
+                catch (ex) {
+                    // ignore
+                }
             }
-        }
-        nodecommons_cli_1.CommonsOutput.success();
+            commonsOutputSuccess();
+        });
     }
 }
-exports.CrawlServer = CrawlServer;
+//# sourceMappingURL=crawl.server.js.map

package/dist/servers/crawl.server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.server.js","sourceRoot":"","sources":["../../src/servers/crawl.server.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC5L,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAEzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAa7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,OAAO,WAAW;IAUvB,YACU,QAAyB,EACzB,WAAyB,EACzB,WAAyB,EACzB,aAAwC,EACxC,YAA2B,EAC3B,KAAY,EACZ,OAAgB;QANhB,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAA2B;QACxC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAS;QAhBlB,YAAO,GAAkB,EAAE,CAAC;QAG5B,cAAS,GAAY,KAAK,CAAC;QAE3B,gBAAW,GAAa,EAAE,CAAC;QAE3B,aAAQ,GAAY,KAAK,CAAC;QAWjC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;QAExD,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,kDAAkD,CAAC,CAAC;YACvE,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW;gBAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEM,SAAS,CAAC,MAAmB;QACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEM,WAAW;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEM,KAAK;QACX,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;YAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;SACxB;IACF,CAAC;IAEM,MAAM;QACZ,sBAAsB,CAAC,uBAAuB,CAAC,CAAC;QAEhD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAEtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;YAC1C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;SACzB;IACF,CAAC;IAEO,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;IAChD,CAAC;IAEY,SAAS;;YACrB,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;KAAA;IAEO,UAAU,CAAC,MAAc;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,kBAAkB,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;QAEpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QACjC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,sBAAsB,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAEY,KAAK;;YACjB,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAElC,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAEjD,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;oBACnB,MAAM,QAAQ,GAAa,CAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAE,CAAC;oBAEvD,MAAM,KAAK,GAAW,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;oBAErE,IAAI,KAAK,GAAG,CAAC,EAAE;wBACd,kBAAkB,CAAC,aAAa,KAAK,uBAAuB,CAAC,CAAC;wBAE9D,MAAM,sBAAsB,GAAa,QAAQ;6BAC9C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,OAAO,CAAC,CAAC;wBAEnG,MAAM,uBAAuB,GAAa,QAAQ;6BAC/C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,QAAQ,CAAC,CAAC;wBAEpG,kBAAkB,CAAC,6BAA6B,sBAAsB,CAAC,MAAM,cAAc,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAAC;wBAE7H,IAAI,sBAAsB,GAAW,CAAC,CAAC;wBACvC,IAAI,uBAAuB,GAAW,KAAK,CAAC;wBAE5C,IAAI,sBAAsB,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE;4BAClF,0EAA0E;yBAC1E;6BAAM;4BACN,2CAA2C;4BAC3C,IAAI,KAAK,GAAG,CAAC,EAAE;gCACd,sBAAsB,GAAG,CAAC,CAAC;gCAC3B,uBAAuB,EAAE,CAAC;6BAC1B;yBACD;wBAED,kBAAkB,CAAC,qBAAqB,sBAAsB,cAAc,uBAAuB,EAAE,CAAC,CAAC;wBAEvG,MAAM,iBAAiB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAC/D,kBAAkB,CAAC,OAAO,EAC1B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,sBAAsB,EACtB,QAAQ,CACT,CAAC;wBAEF,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAC;wBAEpC,MAAM,kBAAkB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAChE,kBAAkB,CAAC,QAAQ,EAC3B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,uBAAuB,EACvB,QAAQ,CACT,CAAC;wBAEF,MAAM,UAAU,GAAa;4BAC3B,GAAG,kBAAkB;4BACrB,GAAG,iBAAiB;yBACrB,CAAC;wBAEF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;4BACpB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gCAChC,qBAAqB,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;gCAE/D,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,EACN,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;gCAEF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gCAEnC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;oCACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC;iCACxD;qCAAM;oCACN,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;iCACzD;gCAED,sDAAsD;gCACtD,KAAK,CAAC,GAAwB,EAAE;oCAC/B,IAAI;wCACH,0DAA0D;wCAC1D,MAAM,UAAU,GAAW,YAAY,MAAM,EAAE,CAAC;wCAChD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wCAClC,IAAI;4CACH,MAAM,mBAAmB,CACvB,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,GAAG,IAAI,CAAC,EAC/D,UAAU,CACX,CAAC;yCACF;wCAAC,OAAO,CAAC,EAAE;4CACX,IAAK,CAAW,CAAC,OAAO,KAAK,qBAAqB;gDAAE,OAAO;4CAC3D,MAAM,CAAC,CAAC;yCACR;wCAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;6CAChC,MAAM,CAAC,CAAC,GAAW,EAAW,EAAE,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;wCAExD,IAAI,IAAI,CAAC,SAAS;4CAAE,OAAO;wCAC3B,IAAI,IAAI,CAAC,QAAQ;4CAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;wCAE3C,qBAAqB,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;wCACxD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;qCACtB;oCAAC,OAAO,EAAE,EAAE;wCACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;wCACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qCAChB;4CAAS;wCACT,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,uFAAuF;wCACrH,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;qCAC/B;gCACF,CAAC,CAAA,CAAC,EAAE,CAAC;6BACL;yBACD;qBACD;iBACD;gBAED,IAAI;oBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;iBACxF;gBAAC,OAAO,EAAE,EAAE;oBACZ,SAAS;iBACT;gBAED,IAAI,IAAI,CAAC,SAAS,EAAE;oBACnB,kBAAkB,CAAC,8CAA8C,CAAC,CAAC;oBACnE,MAAM;iBACN;aACD;QACF,CAAC;KAAA;IAEY,QAAQ;;YACpB,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE;gBAC9B,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAE1C,IAAI;oBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE;oBACZ,SAAS;iBACT;aACD;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/servers/express.server.d.ts

@@ -0,0 +1,8 @@
+/// <reference types="node" />
+import * as http from 'http';
+import * as express from 'express';
+import { CommonsStrictExpressServer } from 'nodecommons-es-express';
+import { ICommonsExpressConfig } from 'nodecommons-es-express';
+export declare class ExpressServer extends CommonsStrictExpressServer {
+    constructor(ex: express.Express, server: http.Server, config: ICommonsExpressConfig);
+}

package/dist/servers/express.server.js

@@ -1,16 +1,13 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExpressServer = void 0;
-const nodecommons_express_1 = require("nodecommons-express");
-const nodecommons_process_1 = require("nodecommons-process");
-const nodecommons_cli_1 = require("nodecommons-cli");
-class ExpressServer extends nodecommons_express_1.CommonsExpressServer {
+import { CommonsStrictExpressServer } from 'nodecommons-es-express';
+import { commonsGracefulAbortAddCallback } from 'nodecommons-es-process';
+import { commonsOutputAlert } from 'nodecommons-es-cli';
+export class ExpressServer extends CommonsStrictExpressServer {
     constructor(ex, server, config) {
         super(ex, server, config.port);
-        nodecommons_process_1.CommonsGracefulAbort.addCallback(() => {
-            nodecommons_cli_1.CommonsOutput.alert(`SIGINT abort flag is set. Aborting Express server.`);
+        commonsGracefulAbortAddCallback(() => {
+            commonsOutputAlert('SIGINT abort flag is set. Aborting Express server.');
             super.close();
         });
     }
 }
-exports.ExpressServer = ExpressServer;
+//# sourceMappingURL=express.server.js.map

package/dist/servers/express.server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.server.js","sourceRoot":"","sources":["../../src/servers/express.server.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAEpE,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExD,MAAM,OAAO,aAAc,SAAQ,0BAA0B;IAC5D,YACE,EAAmB,EACnB,MAAmB,EACnB,MAA6B;QAE9B,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAE/B,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,oDAAoD,CAAC,CAAC;YACzE,KAAK,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}

package/dist/servers/maintenance.server.d.ts

@@ -0,0 +1,22 @@
+import { TCommonsScheduleTime } from 'tscommons-es-async';
+import { Expiry } from '../classes/expiry';
+import { Lists } from '../classes/lists';
+import { CrawlServer } from '../servers/crawl.server';
+import { DatabaseService } from '../services/database.service';
+declare enum EAction {
+    EXPIRE = "expire",
+    PURGE_EMPTY_DOMAINS = "purge-empty-domains",
+    PURGE_ORPHAN_URLS = "purge-orphan-urls"
+}
+export declare function toEAction(value: string): EAction | undefined;
+export declare class MaintenanceServer {
+    private crawl;
+    private schedule;
+    private expirer;
+    private cleaner;
+    private isPaused;
+    constructor(times: TCommonsScheduleTime[], expiry: Expiry, lists: Lists, database: DatabaseService, crawl: CrawlServer);
+    private perform;
+    start(): void;
+}
+export {};