hydra-crawler 2.8.3 → 3.0.1

package/dist/queries/flash-content.query.js

@@ -1,82 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { ECommonsCsvColumnType } from 'tscommons-es-format';
-import { isIUrl } from 'hydra-crawler-ts-assets';
-import { isTLink } from 'hydra-crawler-ts-assets';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsCsv } from 'nodecommons-es-file';
-export const QUERY = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const filename = args.getString('filename');
-    commonsOutputDoing('Searching for FLV and SWF URLs');
-    const results = databaseService.getUrls().find({
-        url: /\.(flv|swf)$/i,
-        status: { $in: [
-                EStatus.ACTIVE,
-                EStatus.DENY,
-                EStatus.DISALLOWED,
-                EStatus.DONE,
-                EStatus.FAILED,
-                EStatus.QUEUED
-            ] }
-    }, {});
-    const urls = yield databaseService.listQueryResults(results, isIUrl);
-    commonsOutputResult(urls.length);
-    commonsOutputDoing('Searching for outgoing links to URLs');
-    let tally = 0;
-    const urlLinks = new Map();
-    for (const url of urls) {
-        const results3 = databaseService.getLinks().find({
-            outgoing: url.url
-        }, {});
-        const links = yield databaseService.listQueryResults(results3, isTLink);
-        if (links.length === 0)
-            continue;
-        tally += links.length;
-        commonsOutputProgress(tally);
-        urlLinks.set(url, links);
-    }
-    commonsOutputResult(tally);
-    commonsOutputDoing('Building CSV array');
-    const rows = [];
-    for (const url of urls) {
-        for (const link of (urlLinks.get(url) || [])) {
-            rows.push({
-                src: link.url,
-                dest: url.url
-            });
-        }
-    }
-    rows
-        .sort((a, b) => {
-        if (a.src < b.src)
-            return -1;
-        if (a.src > b.src)
-            return 1;
-        if (a.dest < b.dest)
-            return -1;
-        if (a.dest > b.dest)
-            return 1;
-        return 0;
-    });
-    commonsOutputSuccess();
-    const csv = new CommonsCsv([
-        {
-            name: 'src',
-            type: ECommonsCsvColumnType.STRING
-        },
-        {
-            name: 'dest',
-            type: ECommonsCsvColumnType.STRING
-        }
-    ]);
-    csv.save(rows, filename, true);
-});
-//# sourceMappingURL=flash-content.query.js.map

package/dist/queries/flash-content.query.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"flash-content.query.js","sourceRoot":"","sources":["../../src/queries/flash-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACvI,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAgBjD,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACtE;QACE,GAAG,EAAE,eAAe;QACpB,MAAM,EAAE,EAAE,GAAG,EAAE;gBACb,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,MAAM;aACf,EAAE;KACJ,EACD,EAAE,CACH,CAAC;IACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,OAAO,EACP,MAAM,CACP,CAAC;IACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjC,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;YACE,QAAQ,EAAE,GAAG,CAAC,GAAG;SAClB,EACD,EAAE,CACH,CAAC;QACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;QAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;QACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAE7B,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;KACzB;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC;gBACR,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,IAAI,EAAE,GAAG,CAAC,GAAG;aACd,CAAC,CAAC;SACH;KACD;IACD,IAAI;SACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;QAClC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEL,oBAAoB,EAAE,CAAC;IAEvB,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAA,CAAC"}

package/dist/queries/linking-to-domains.query.d.ts

@@ -1,2 +0,0 @@
-import { TQuery } from '../types/tquery';
-export declare const QUERY: TQuery;

package/dist/queries/linking-to-domains.query.js

@@ -1,130 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { isIUrl } from 'hydra-crawler-ts-assets';
-import { isTLink } from 'hydra-crawler-ts-assets';
-import { isTDomain } from 'hydra-crawler-ts-assets';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-const domainSort = (a, b) => {
-    if (a.startsWith('http://'))
-        a = a.substr(7);
-    if (a.startsWith('https://'))
-        a = a.substr(8);
-    if (a.startsWith('www.'))
-        a = a.substr(4);
-    if (b.startsWith('http://'))
-        b = b.substr(7);
-    if (b.startsWith('https://'))
-        b = b.substr(8);
-    if (b.startsWith('www.'))
-        b = b.substr(4);
-    if (a < b)
-        return -1;
-    if (a > b)
-        return 1;
-    return 0;
-};
-export const QUERY = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const regexStr = args.getString('regex');
-    if (!regexStr) {
-        commonsOutputError('No domain regex specified');
-        return;
-    }
-    const regex = new RegExp(regexStr);
-    commonsOutputDoing('Searching for matching domains');
-    const results = databaseService.getDomains().find({
-        domain: regex
-    }, {});
-    const domains = yield databaseService.listQueryResults(results, isTDomain);
-    commonsOutputResult(domains.length);
-    const domainUrls = new Map();
-    for (const domain of domains) {
-        commonsOutputDoing(`Searching for URLs for ${domain.domain}`);
-        const results2 = databaseService.getUrls().find({
-            domain: domain.domain,
-            status: { $ne: EStatus.ARCHIVED }
-        }, {});
-        const urls = yield databaseService.listQueryResults(results2, isIUrl);
-        commonsOutputResult(urls.length);
-        domainUrls.set(domain, urls);
-    }
-    commonsOutputDoing('Searching for outgoing links to URLs');
-    let tally = 0;
-    const domainUrlLinks = new Map();
-    for (const domain of domains) {
-        const urls = domainUrls.get(domain);
-        if (!urls || !urls.length)
-            continue;
-        const map = new Map();
-        for (const url of urls) {
-            const results3 = databaseService.getLinks().find({
-                outgoing: url.url
-            }, {});
-            const links = yield databaseService.listQueryResults(results3, isTLink);
-            if (links.length === 0)
-                continue;
-            tally += links.length;
-            commonsOutputProgress(tally);
-            map.set(url, links);
-        }
-        if (map.size === 0)
-            continue;
-        domainUrlLinks.set(domain, map);
-    }
-    commonsOutputResult(tally);
-    commonsOutputDoing('Constructing JSON object');
-    const outcome = {};
-    for (const domain of domainUrlLinks.keys()) {
-        outcome[domain.domain] = {};
-        for (const url of domainUrlLinks.get(domain).keys()) {
-            outcome[domain.domain][url.url] = domainUrlLinks.get(domain).get(url)
-                .map((link) => link.url);
-        }
-    }
-    commonsOutputSuccess();
-    if (args.hasAttribute('domains-only')) {
-        Object.keys(outcome)
-            .sort(domainSort)
-            .forEach((domain) => {
-            console.log(domain);
-        });
-    }
-    else if (args.hasAttribute('url-list')) {
-        const reverseMap = new Map();
-        for (const domain of Object.keys(outcome)) {
-            for (const out of Object.keys(outcome[domain])) {
-                for (const url of outcome[domain][out]) {
-                    if (!reverseMap.has(url))
-                        reverseMap.set(url, []);
-                    if (!reverseMap.get(url).includes(out))
-                        reverseMap.get(url).push(out);
-                }
-            }
-        }
-        const urls = Array.from(reverseMap.keys())
-            .sort(domainSort);
-        for (const url of urls) {
-            reverseMap.get(url).sort(domainSort);
-            let first = true;
-            for (const out of reverseMap.get(url)) {
-                const tsv = [
-                    first ? url : '',
-                    out
-                ];
-                console.log(tsv.join('\t'));
-                first = false;
-            }
-        }
-    }
-    else if (args.hasAttribute('json-dump')) {
-        console.log(JSON.stringify(outcome));
-    }
-});
-//# sourceMappingURL=linking-to-domains.query.js.map

package/dist/queries/linking-to-domains.query.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"linking-to-domains.query.js","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAS,OAAO,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAiB3J,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,QAAQ,EAAE;QACd,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,OAAO;KACP;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;IACrD,MAAM,OAAO,GAAgC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAC5E;QACE,MAAM,EAAE,KAAK;KACd,EACD,EAAE,CACH,CAAC;IACF,MAAM,OAAO,GAAc,MAAM,eAAe,CAAC,gBAAgB,CAC/D,OAAO,EACP,SAAS,CACV,CAAC;IACF,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC,MAAM,UAAU,GAAyB,IAAI,GAAG,EAAmB,CAAC;IACpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,kBAAkB,CAAC,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACvE;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;SAClC,EACD,EAAE,CACH,CAAC;QACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,QAAQ,EACR,MAAM,CACP,CAAC;QACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEjC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;KAC7B;IAED,kBAAkB,CAAC,sCAAsC,CAAC,CAAC;IAC3D,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,MAAM,cAAc,GAAqC,IAAI,GAAG,EAA+B,CAAC;IAChG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC7B,MAAM,IAAI,GAAqB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAEpC,MAAM,GAAG,GAAuB,IAAI,GAAG,EAAiB,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;gBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;aAClB,EACD,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,qBAAqB,CAAC,KAAK,CAAC,CAAC;YAE7B,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;SACpB;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAE7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE;QAC3C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE5B,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,EAAE;YACrD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAE;iBACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAC1C;KACD;IACD,oBAAoB,EAAE,CAAC;IAEvB,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE;QACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,CAAC,CAAC,MAAc,EAAQ,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;KACL;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,UAAU,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEtE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE;gBAC/C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACxE;aACD;SACD;QAED,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aACjD,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEtC,IAAI,KAAK,GAAY,IAAI,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE;gBACvC,MAAM,GAAG,GAAa;oBACpB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChB,GAAG;iBACJ,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE5B,KAAK,GAAG,KAAK,CAAC;aACd;SACD;KACD;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;KACrC;AACF,CAAC,CAAA,CAAC"}

package/dist/queries/llamaguard-unsafe-content.query.d.ts

@@ -1,2 +0,0 @@
-import { TQuery } from '../types/tquery';
-export declare const QUERY: TQuery;

package/dist/queries/llamaguard-unsafe-content.query.js

@@ -1,92 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { ECommonsCsvColumnType } from 'tscommons-es-format';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsCsv } from 'nodecommons-es-file';
-import { ELLAMA_GUARD_CATEGORYS, prettyLlamaGuardCategory } from '../services/rig-llama-guard.service';
-export const QUERY = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const filename = args.getString('filename');
-    commonsOutputDoing('Searching for URLs marked unsafe');
-    const results = databaseService.getUrls().find({
-        'llamaguard.classification': 'unsafe',
-        status: { $in: [
-                EStatus.ACTIVE,
-                EStatus.DENY,
-                EStatus.DISALLOWED,
-                EStatus.DONE,
-                EStatus.FAILED,
-                EStatus.QUEUED
-            ] }
-    }, {});
-    const urls = yield databaseService.listQueryResults(results, (_t) => true);
-    commonsOutputResult(urls.length);
-    commonsOutputDoing('Inverting categorisation');
-    const map = new Map();
-    for (const category of ELLAMA_GUARD_CATEGORYS)
-        map.set(category, []);
-    for (const url of urls) {
-        if (url.llamaguard.classification === 'safe')
-            continue;
-        for (const category of url.llamaguard.categories)
-            map.get(category).push(url);
-    }
-    commonsOutputSuccess();
-    commonsOutputDoing('Building CSV array');
-    const rows = [];
-    for (const category of ELLAMA_GUARD_CATEGORYS) {
-        const meaning = prettyLlamaGuardCategory(category);
-        for (const url of map.get(category)) {
-            rows.push({
-                category: category,
-                meaning: meaning,
-                domain: url.domain,
-                url: url.url
-            });
-        }
-    }
-    rows
-        .sort((a, b) => {
-        if (a.category < b.category)
-            return -1;
-        if (a.category > b.category)
-            return 1;
-        if (a.domain < b.domain)
-            return -1;
-        if (a.domain > b.domain)
-            return 1;
-        if (a.url < b.url)
-            return -1;
-        if (a.url > b.url)
-            return 1;
-        return 0;
-    });
-    commonsOutputSuccess();
-    const csv = new CommonsCsv([
-        {
-            name: 'category',
-            type: ECommonsCsvColumnType.STRING
-        },
-        {
-            name: 'meaning',
-            type: ECommonsCsvColumnType.STRING
-        },
-        {
-            name: 'domain',
-            type: ECommonsCsvColumnType.STRING
-        },
-        {
-            name: 'url',
-            type: ECommonsCsvColumnType.STRING
-        }
-    ]);
-    csv.save(rows, filename, true);
-});
-//# sourceMappingURL=llamaguard-unsafe-content.query.js.map

package/dist/queries/llamaguard-unsafe-content.query.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"llamaguard-unsafe-content.query.js","sourceRoot":"","sources":["../../src/queries/llamaguard-unsafe-content.query.ts"],"names":[],"mappings":";;;;;;;;;AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAG5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAChH,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,OAAO,EAAE,sBAAsB,EAAuB,wBAAwB,EAAqB,MAAM,qCAAqC,CAAC;AAe/I,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAEpD,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;IACvD,MAAM,OAAO,GAA2C,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACpF;QACE,2BAA2B,EAAE,QAAQ;QACrC,MAAM,EAAE,EAAE,GAAG,EAAE;gBACb,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,UAAU;gBAClB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;gBACd,OAAO,CAAC,MAAM;aACf,EAAE;KACJ,EACD,EAAE,CACH,CAAC;IACF,MAAM,IAAI,GAAyB,MAAM,eAAe,CAAC,gBAAgB,CACvE,OAAO,EACP,CAAC,EAAW,EAA4B,EAAE,CAAC,IAAI,CAChD,CAAC;IACF,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEjC,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAqC,IAAI,GAAG,EAA+B,CAAC;IACrF,KAAK,MAAM,QAAQ,IAAI,sBAAsB;QAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAErE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,IAAI,GAAG,CAAC,UAAU,CAAC,cAAc,KAAK,MAAM;YAAE,SAAS;QAEvD,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU;YAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KAC/E;IACD,oBAAoB,EAAE,CAAC;IAEvB,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,KAAK,MAAM,QAAQ,IAAI,sBAAsB,EAAE;QAC9C,MAAM,OAAO,GAAW,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QAE3D,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,EAAE;YACrC,IAAI,CAAC,IAAI,CAAC;gBACR,QAAQ,EAAE,QAAQ;gBAClB,OAAO,EAAE,OAAO;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;aACb,CAAC,CAAC;SACH;KACD;IAED,IAAI;SACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;QAClC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC;QAEtC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC;QAElC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;YAAE,OAAO,CAAC,CAAC;QAE5B,OAAO,CAAC,CAAC;IACV,CAAC,CAAC,CAAC;IAEL,oBAAoB,EAAE,CAAC;IAEvB,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAA,CAAC"}

package/dist/queries/readability-score.query.d.ts

@@ -1,2 +0,0 @@
-import { TQuery } from '../types/tquery';
-export declare const QUERY: TQuery;

package/dist/queries/readability-score.query.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"readability-score.query.js","sourceRoot":"","sources":["../../src/queries/readability-score.query.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAe,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAuBrI,MAAM,CAAC,MAAM,KAAK,GAAW,CAC3B,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAAyC,EAC1B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1D,IAAI,CAAC,MAAM,EAAE;QACZ,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO;KACP;IAED,kBAAkB,CAAC,4DAA4D,MAAM,EAAE,CAAC,CAAC;IAEzF,MAAM,OAAO,GAA+B,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAU;QACvF,EAAE,MAAM,EAAE;gBACR,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,sBAAsB,EAAE,WAAW;aACpC,EAAE;QACH,EAAE,MAAM,EAAE;gBACR,2BAA2B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;aAC3C,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE;oBACL,UAAU,EAAE,kCAAkC;oBAC9C,SAAS,EAAE,iCAAiC;oBAC5C,KAAK,EAAE,6BAA6B;oBACpC,IAAI,EAAE,4BAA4B;oBAClC,GAAG,EAAE,2BAA2B;iBACjC;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,CAAE,EAAE;oBACtD,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,EAAE,EAAE,YAAY,CAAE,EAAE,CAAE,EAAE;iBACpD;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE;oBACR,IAAI,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,EAAE,GAAG,CAAE,EAAE;oBACjE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,YAAY,EAAE,CAAC,CAAE,EAAE,EAAE,EAAE,CAAE,EAAE;iBAC7D;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,IAAI,EAAE,CAAE,gBAAgB,EAAE,eAAe,CAAE,EAAE,EAAE,CAAC,CAAE,EAAE;aAC3E,EAAE;QACH,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE;QACZ,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAE3D,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KAClB;IAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAE3B,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;KAChK;AACF,CAAC,CAAA,CAAC"}

package/dist/servers/crawl.server.js

@@ -1,198 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-es-async';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputAlert, commonsOutputCompleted, commonsOutputDebug, commonsOutputDoing, commonsOutputProgress, commonsOutputStarting, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { commonsGracefulAbortAddCallback } from 'nodecommons-es-process';
-import { Crawler } from '../classes/crawler';
-import { EAvailableStrategy } from '../enums/eavailable-strategy';
-export class CrawlServer {
-    set restrictTo(domains) {
-        this.domainRestrictTo = domains;
-    }
-    constructor(database, hydraConfig, crawlConfig, parsersConfig, robotsConfig, lists, tracker) {
-        this.database = database;
-        this.hydraConfig = hydraConfig;
-        this.crawlConfig = crawlConfig;
-        this.parsersConfig = parsersConfig;
-        this.robotsConfig = robotsConfig;
-        this.lists = lists;
-        this.tracker = tracker;
-        this.parsers = [];
-        this.isAborted = false;
-        this.preDelayIds = [];
-        this.isPaused = false;
-        this.domainRestrictTo = [];
-        this.crawlers = new Map();
-        this.strategies = new Map();
-        commonsGracefulAbortAddCallback(() => {
-            commonsOutputAlert('SIGINT abort flag is set. Aborting crawl server.');
-            this.abort();
-            for (const preDelayId of this.preDelayIds)
-                commonsAsyncAbortTimeout(preDelayId);
-        });
-    }
-    addParser(parser) {
-        this.parsers.push(parser);
-    }
-    listParsers() {
-        return this.parsers.slice();
-    }
-    pause() {
-        commonsOutputAlert('Pausing crawl server');
-        this.isPaused = true;
-        for (const domain of this.crawlers.keys()) {
-            this.pauseCrawl(domain);
-        }
-    }
-    resume() {
-        commonsOutputCompleted('Resuming crawl server');
-        this.isPaused = false;
-        for (const domain of this.crawlers.keys()) {
-            this.resumeCrawl(domain);
-        }
-    }
-    abort() {
-        this.isAborted = true;
-        commonsAsyncAbortTimeout('find-new-available');
-    }
-    terminate() {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.abort();
-            yield this.database.close();
-            process.exit(0);
-        });
-    }
-    pauseCrawl(domain) {
-        if (!this.crawlers.has(domain))
-            return;
-        commonsOutputAlert(`Pausing crawler for ${domain}`);
-        this.crawlers.get(domain).pause();
-    }
-    resumeCrawl(domain) {
-        if (!this.crawlers.has(domain))
-            return;
-        commonsOutputCompleted(`Resuming crawler for ${domain}`);
-        this.crawlers.get(domain).resume();
-    }
-    start() {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.database.resetActive();
-            const added = yield this.database.queue(this.hydraConfig.startUrl);
-            if (added)
-                this.tracker.delta(EStatus.QUEUED, 1);
-            while (!this.isAborted) {
-                if (!this.isPaused) {
-                    const existing = [...this.crawlers.keys()];
-                    const space = this.crawlConfig.maxCrawlers - existing.length;
-                    if (space > 0) {
-                        commonsOutputDebug(`Space for ${space} new crawls available`);
-                        const largestStrategyDomains = existing
-                            .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.LARGEST);
-                        const smallestStrategyDomains = existing
-                            .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.SMALLEST);
-                        commonsOutputDebug(`Current crawl has LARGEST=${largestStrategyDomains.length}; SMALLEST=${smallestStrategyDomains.length}`);
-                        let spaceForLargestDomains = 0;
-                        let spaceForSmallestDomains = space;
-                        if (largestStrategyDomains.length >= Math.floor(this.crawlConfig.maxCrawlers / 2)) {
-                            // already 50% large crawling, so all remaining space goes to small crawls
-                        }
-                        else {
-                            // allow a new large crawl, if enough space
-                            if (space > 1) {
-                                spaceForLargestDomains = 1;
-                                spaceForSmallestDomains--;
-                            }
-                        }
-                        commonsOutputDebug(`Space for LARGEST=${spaceForLargestDomains}; SMALLEST=${spaceForSmallestDomains}`);
-                        const availablesLargest = yield this.database.available(EAvailableStrategy.LARGEST, this.crawlConfig.availableStrategyThreshold, spaceForLargestDomains, existing, this.domainRestrictTo);
-                        existing.push(...availablesLargest);
-                        const availablesSmallest = yield this.database.available(EAvailableStrategy.SMALLEST, this.crawlConfig.availableStrategyThreshold, spaceForSmallestDomains, existing, this.domainRestrictTo);
-                        const availables = [
-                            ...availablesSmallest,
-                            ...availablesLargest
-                        ];
-                        if (!this.isAborted) {
-                            for (const domain of availables) {
-                                commonsOutputStarting(`Creating new crawl head for ${domain}`);
-                                const crawler = new Crawler(domain, this.database, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.parsers, this.lists, this.tracker);
-                                this.crawlers.set(domain, crawler);
-                                if (availablesLargest.includes(domain)) {
-                                    this.strategies.set(domain, EAvailableStrategy.LARGEST);
-                                }
-                                else {
-                                    this.strategies.set(domain, EAvailableStrategy.SMALLEST);
-                                }
-                                // called without await in order to do parallel crawls
-                                void (() => __awaiter(this, void 0, void 0, function* () {
-                                    try {
-                                        // delay randomly to prevent network socket request spikes
-                                        const preDelayId = `predelay_${domain}`;
-                                        this.preDelayIds.push(preDelayId);
-                                        try {
-                                            yield commonsAsyncTimeout(Math.random() * (this.crawlConfig.findNewAvailableDelay - 1000), preDelayId);
-                                        }
-                                        catch (e) {
-                                            if (e.message === 'abortTimeout called')
-                                                return;
-                                            throw e;
-                                        }
-                                        this.preDelayIds = this.preDelayIds
-                                            .filter((pid) => pid !== preDelayId);
-                                        if (this.isAborted)
-                                            return;
-                                        if (this.isPaused)
-                                            this.pauseCrawl(domain);
-                                        commonsOutputStarting(`Starting crawler for ${domain}`);
-                                        yield crawler.crawl();
-                                    }
-                                    catch (ex) {
-                                        commonsOutputDebug('debug position 6');
-                                        console.log(ex);
-                                    }
-                                    finally {
-                                        this.crawlers.delete(domain); // doesn't get called until after doCrawl() is called, so have to do it here if aborted
-                                        this.strategies.delete(domain);
-                                    }
-                                }))();
-                            }
-                        }
-                    }
-                }
-                try {
-                    yield commonsAsyncTimeout(this.crawlConfig.findNewAvailableDelay, 'find-new-available');
-                }
-                catch (ex) {
-                    // ignore
-                }
-                if (this.isAborted) {
-                    commonsOutputAlert('SIGINT abort is set. Aborting new head loop.');
-                    break;
-                }
-            }
-        });
-    }
-    shutdown() {
-        return __awaiter(this, void 0, void 0, function* () {
-            commonsOutputDoing('Waiting for all crawlers to abort');
-            while (this.crawlers.size > 0) {
-                commonsOutputProgress(this.crawlers.size);
-                try {
-                    yield commonsAsyncTimeout(1000);
-                }
-                catch (ex) {
-                    // ignore
-                }
-            }
-            commonsOutputSuccess();
-        });
-    }
-}
-//# sourceMappingURL=crawl.server.js.map

package/dist/servers/crawl.server.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawl.server.js","sourceRoot":"","sources":["../../src/servers/crawl.server.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEnF,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC5L,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAEzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAa7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,MAAM,OAAO,WAAW;IAWvB,IAAW,UAAU,CAAC,OAAiB;QACtC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;IACjC,CAAC;IAED,YACU,QAAyB,EACzB,WAAyB,EACzB,WAAyB,EACzB,aAAwC,EACxC,YAA2B,EAC3B,KAAY,EACZ,OAAgB;QANhB,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAA2B;QACxC,iBAAY,GAAZ,YAAY,CAAe;QAC3B,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAS;QArBlB,YAAO,GAAkB,EAAE,CAAC;QAG5B,cAAS,GAAY,KAAK,CAAC;QAE3B,gBAAW,GAAa,EAAE,CAAC;QAE3B,aAAQ,GAAY,KAAK,CAAC;QAE1B,qBAAgB,GAAa,EAAE,CAAC;QAcvC,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;QAExD,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,kDAAkD,CAAC,CAAC;YACvE,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW;gBAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEM,SAAS,CAAC,MAAmB;QACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEM,WAAW;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEM,KAAK;QACX,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;QAE3C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;YAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;SACxB;IACF,CAAC;IAEM,MAAM;QACZ,sBAAsB,CAAC,uBAAuB,CAAC,CAAC;QAEhD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAEtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;YAC1C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;SACzB;IACF,CAAC;IAEO,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;IAChD,CAAC;IAEY,SAAS;;YACrB,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;KAAA;IAEO,UAAU,CAAC,MAAc;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,kBAAkB,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;QAEpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QACjC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,sBAAsB,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAEY,KAAK;;YACjB,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAElC,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC5E,IAAI,KAAK;gBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAEjD,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE;gBACvB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;oBACnB,MAAM,QAAQ,GAAa,CAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAE,CAAC;oBAEvD,MAAM,KAAK,GAAW,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;oBAErE,IAAI,KAAK,GAAG,CAAC,EAAE;wBACd,kBAAkB,CAAC,aAAa,KAAK,uBAAuB,CAAC,CAAC;wBAE9D,MAAM,sBAAsB,GAAa,QAAQ;6BAC9C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,OAAO,CAAC,CAAC;wBAEnG,MAAM,uBAAuB,GAAa,QAAQ;6BAC/C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,QAAQ,CAAC,CAAC;wBAEpG,kBAAkB,CAAC,6BAA6B,sBAAsB,CAAC,MAAM,cAAc,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAAC;wBAE7H,IAAI,sBAAsB,GAAW,CAAC,CAAC;wBACvC,IAAI,uBAAuB,GAAW,KAAK,CAAC;wBAE5C,IAAI,sBAAsB,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE;4BAClF,0EAA0E;yBAC1E;6BAAM;4BACN,2CAA2C;4BAC3C,IAAI,KAAK,GAAG,CAAC,EAAE;gCACd,sBAAsB,GAAG,CAAC,CAAC;gCAC3B,uBAAuB,EAAE,CAAC;6BAC1B;yBACD;wBAED,kBAAkB,CAAC,qBAAqB,sBAAsB,cAAc,uBAAuB,EAAE,CAAC,CAAC;wBAEvG,MAAM,iBAAiB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAC/D,kBAAkB,CAAC,OAAO,EAC1B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,sBAAsB,EACtB,QAAQ,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;wBAEF,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAC;wBAEpC,MAAM,kBAAkB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAChE,kBAAkB,CAAC,QAAQ,EAC3B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,uBAAuB,EACvB,QAAQ,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;wBAEF,MAAM,UAAU,GAAa;4BAC3B,GAAG,kBAAkB;4BACrB,GAAG,iBAAiB;yBACrB,CAAC;wBAEF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;4BACpB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gCAChC,qBAAqB,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;gCAE/D,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,EACN,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;gCAEF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gCAEnC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;oCACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC;iCACxD;qCAAM;oCACN,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;iCACzD;gCAED,sDAAsD;gCACtD,KAAK,CAAC,GAAwB,EAAE;oCAC/B,IAAI;wCACH,0DAA0D;wCAC1D,MAAM,UAAU,GAAW,YAAY,MAAM,EAAE,CAAC;wCAChD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wCAClC,IAAI;4CACH,MAAM,mBAAmB,CACvB,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,GAAG,IAAI,CAAC,EAC/D,UAAU,CACX,CAAC;yCACF;wCAAC,OAAO,CAAC,EAAE;4CACX,IAAK,CAAW,CAAC,OAAO,KAAK,qBAAqB;gDAAE,OAAO;4CAC3D,MAAM,CAAC,CAAC;yCACR;wCAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;6CAChC,MAAM,CAAC,CAAC,GAAW,EAAW,EAAE,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;wCAExD,IAAI,IAAI,CAAC,SAAS;4CAAE,OAAO;wCAC3B,IAAI,IAAI,CAAC,QAAQ;4CAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;wCAE3C,qBAAqB,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;wCACxD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;qCACtB;oCAAC,OAAO,EAAE,EAAE;wCACZ,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;wCACvC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;qCAChB;4CAAS;wCACT,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,uFAAuF;wCACrH,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;qCAC/B;gCACF,CAAC,CAAA,CAAC,EAAE,CAAC;6BACL;yBACD;qBACD;iBACD;gBAED,IAAI;oBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;iBACxF;gBAAC,OAAO,EAAE,EAAE;oBACZ,SAAS;iBACT;gBAED,IAAI,IAAI,CAAC,SAAS,EAAE;oBACnB,kBAAkB,CAAC,8CAA8C,CAAC,CAAC;oBACnE,MAAM;iBACN;aACD;QACF,CAAC;KAAA;IAEY,QAAQ;;YACpB,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE;gBAC9B,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAE1C,IAAI;oBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAAC,OAAO,EAAE,EAAE;oBACZ,SAAS;iBACT;aACD;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/servers/express.server.js

@@ -1,13 +0,0 @@
-import { CommonsStrictExpressServer } from 'nodecommons-es-express';
-import { commonsGracefulAbortAddCallback } from 'nodecommons-es-process';
-import { commonsOutputAlert } from 'nodecommons-es-cli';
-export class ExpressServer extends CommonsStrictExpressServer {
-    constructor(ex, server, config) {
-        super(ex, server, config.port);
-        commonsGracefulAbortAddCallback(() => {
-            commonsOutputAlert('SIGINT abort flag is set. Aborting Express server.');
-            super.close();
-        });
-    }
-}
-//# sourceMappingURL=express.server.js.map

package/dist/servers/express.server.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"express.server.js","sourceRoot":"","sources":["../../src/servers/express.server.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAEpE,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExD,MAAM,OAAO,aAAc,SAAQ,0BAA0B;IAC5D,YACE,EAAmB,EACnB,MAAmB,EACnB,MAA6B;QAE9B,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAE/B,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,oDAAoD,CAAC,CAAC;YACzE,KAAK,CAAC,KAAK,EAAE,CAAC;QACf,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}