hydra-crawler 2.8.4 → 3.0.2

package/dist/queries/flash-content.query.mjs

@@ -0,0 +1,80 @@
+import { ECommonsCsvColumnType } from 'tscommons-esm-format';
+import { isIUrl } from 'hydra-crawler-esm-assets';
+import { isTLink } from 'hydra-crawler-esm-assets';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsCsv } from 'nodecommons-esm-file';
+export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+    const filename = args.getFirstValue('filename');
+    let urls;
+    { // scope
+        const log = commonsLogDoing('flash-content-query', 'Searching for FLV and SWF URLs');
+        const results = databaseService.getUrls().find({
+            url: /\.(flv|swf)$/i,
+            status: { $in: [
+                    EStatus.ACTIVE,
+                    EStatus.DENY,
+                    EStatus.DISALLOWED,
+                    EStatus.DONE,
+                    EStatus.FAILED,
+                    EStatus.QUEUED
+                ] }
+        }, {});
+        urls = await databaseService.listQueryResults(results, isIUrl);
+        log.result(urls.length);
+    }
+    const urlLinks = new Map();
+    { // scope
+        const log = commonsLogDoing('flash-content-query', 'Searching for outgoing links to URLs');
+        let tally = 0;
+        for (const url of urls) {
+            const results3 = databaseService.getLinks().find({
+                outgoing: url.url
+            }, {});
+            const links = await databaseService.listQueryResults(results3, isTLink);
+            if (links.length === 0)
+                continue;
+            tally += links.length;
+            log.progress(tally);
+            urlLinks.set(url, links);
+        }
+        log.result(tally);
+    }
+    const rows = [];
+    { // scope
+        const log = commonsLogDoing('flash-content-query', 'Building CSV array');
+        for (const url of urls) {
+            for (const link of (urlLinks.get(url) || [])) {
+                rows.push({
+                    src: link.url,
+                    dest: url.url
+                });
+            }
+        }
+        rows
+            .sort((a, b) => {
+            if (a.src < b.src)
+                return -1;
+            if (a.src > b.src)
+                return 1;
+            if (a.dest < b.dest)
+                return -1;
+            if (a.dest > b.dest)
+                return 1;
+            return 0;
+        });
+        log.success();
+    }
+    const csv = new CommonsCsv([
+        {
+            name: 'src',
+            type: ECommonsCsvColumnType.STRING
+        },
+        {
+            name: 'dest',
+            type: ECommonsCsvColumnType.STRING
+        }
+    ]);
+    csv.save(rows, filename, true);
+};
+//# sourceMappingURL=flash-content.query.mjs.map

package/dist/queries/flash-content.query.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"flash-content.query.mjs","sourceRoot":"","sources":["../../src/queries/flash-content.query.mts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAS,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAgBlD,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAExD,IAAI,IAAY,CAAC;IACjB,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,qBAAqB,EAAE,gCAAgC,CAAC,CAAC;QACzG,MAAM,OAAO,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACtE;YACE,GAAG,EAAE,eAAe;YACpB,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,UAAU;oBAClB,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EACD,EAAE,CACH,CAAC;QACF,IAAI,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAC3C,OAAO,EACP,MAAM,CACP,CAAC;QACF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;IAC9D,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,qBAAqB,EAAE,sCAAsC,CAAC,CAAC;QAC/G,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;gBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;aAClB,EACD,EAAE,CACH,CAAC;YACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAEpB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAED,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;QAE7F,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC;oBACR,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,IAAI,EAAE,GAAG,CAAC,GAAG;iBACd,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QACD,IAAI;aACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;YAClC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;gBAAE,OAAO,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC/B,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI;gBAAE,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAC"}

package/dist/queries/linking-to-domains.query.d.mts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery.mjs';
+export declare const QUERY: TQuery;

package/dist/queries/linking-to-domains.query.mjs

@@ -0,0 +1,128 @@
+import { isIUrl } from 'hydra-crawler-esm-assets';
+import { isTLink } from 'hydra-crawler-esm-assets';
+import { isTDomain } from 'hydra-crawler-esm-assets';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
+const domainSort = (a, b) => {
+    if (a.startsWith('http://'))
+        a = a.substr(7);
+    if (a.startsWith('https://'))
+        a = a.substr(8);
+    if (a.startsWith('www.'))
+        a = a.substr(4);
+    if (b.startsWith('http://'))
+        b = b.substr(7);
+    if (b.startsWith('https://'))
+        b = b.substr(8);
+    if (b.startsWith('www.'))
+        b = b.substr(4);
+    if (a < b)
+        return -1;
+    if (a > b)
+        return 1;
+    return 0;
+};
+export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+    const regexStr = args.getFirstValue('regex');
+    if (!regexStr) {
+        commonsLogError('No domain regex specified');
+        return;
+    }
+    const regex = new RegExp(regexStr);
+    let domains;
+    { // scope
+        const log = commonsLogDoing('linking-to-domains-query', 'Searching for matching domains');
+        const results = databaseService.getDomains().find({
+            domain: regex
+        }, {});
+        domains = await databaseService.listQueryResults(results, isTDomain);
+        log.result(domains.length);
+    }
+    const domainUrls = new Map();
+    for (const domain of domains) {
+        const log = commonsLogDoing('linking-to-domains-query', `Searching for URLs for ${domain.domain}`);
+        const results2 = databaseService.getUrls().find({
+            domain: domain.domain,
+            status: { $ne: EStatus.ARCHIVED }
+        }, {});
+        const urls = await databaseService.listQueryResults(results2, isIUrl);
+        log.result(urls.length);
+        domainUrls.set(domain, urls);
+    }
+    const domainUrlLinks = new Map();
+    { // scope
+        const log = commonsLogDoing('linking-to-domains-query', 'Searching for outgoing links to URLs');
+        let tally = 0;
+        for (const domain of domains) {
+            const urls = domainUrls.get(domain);
+            if (!urls || !urls.length)
+                continue;
+            const map = new Map();
+            for (const url of urls) {
+                const results3 = databaseService.getLinks().find({
+                    outgoing: url.url
+                }, {});
+                const links = await databaseService.listQueryResults(results3, isTLink);
+                if (links.length === 0)
+                    continue;
+                tally += links.length;
+                log.progress(tally);
+                map.set(url, links);
+            }
+            if (map.size === 0)
+                continue;
+            domainUrlLinks.set(domain, map);
+        }
+        log.result(tally);
+    }
+    const outcome = {};
+    { // scope
+        const log = commonsLogDoing('linking-to-domains-query', 'Constructing JSON object');
+        for (const domain of domainUrlLinks.keys()) {
+            outcome[domain.domain] = {};
+            for (const url of domainUrlLinks.get(domain).keys()) {
+                outcome[domain.domain][url.url] = domainUrlLinks.get(domain).get(url)
+                    .map((link) => link.url);
+            }
+        }
+        log.success();
+    }
+    if (args.hasAttribute('domains-only')) {
+        Object.keys(outcome)
+            .sort(domainSort)
+            .forEach((domain) => {
+            console.log(domain);
+        });
+    }
+    else if (args.hasAttribute('url-list')) {
+        const reverseMap = new Map();
+        for (const domain of Object.keys(outcome)) {
+            for (const out of Object.keys(outcome[domain])) {
+                for (const url of outcome[domain][out]) {
+                    if (!reverseMap.has(url))
+                        reverseMap.set(url, []);
+                    if (!reverseMap.get(url).includes(out))
+                        reverseMap.get(url).push(out);
+                }
+            }
+        }
+        const urls = Array.from(reverseMap.keys())
+            .sort(domainSort);
+        for (const url of urls) {
+            reverseMap.get(url).sort(domainSort);
+            let first = true;
+            for (const out of reverseMap.get(url)) {
+                const tsv = [
+                    first ? url : '',
+                    out
+                ];
+                console.log(tsv.join('\t'));
+                first = false;
+            }
+        }
+    }
+    else if (args.hasAttribute('json-dump')) {
+        console.log(JSON.stringify(outcome));
+    }
+};
+//# sourceMappingURL=linking-to-domains.query.mjs.map

package/dist/queries/linking-to-domains.query.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"linking-to-domains.query.mjs","sourceRoot":"","sources":["../../src/queries/linking-to-domains.query.mts"],"names":[],"mappings":"AAIA,OAAO,EAAQ,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAS,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAW,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAiB3F,MAAM,UAAU,GAAqC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;IACrF,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE1C,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,QAAQ,GAAqB,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,eAAe,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;IACR,CAAC;IACD,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC;IAE3C,IAAI,OAAkB,CAAC;IACvB,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,0BAA0B,EAAE,gCAAgC,CAAC,CAAC;QAC9G,MAAM,OAAO,GAAgC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAC5E;YACE,MAAM,EAAE,KAAK;SACd,EACD,EAAE,CACH,CAAC;QACF,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAC9C,OAAO,EACP,SAAS,CACV,CAAC;QACF,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,UAAU,GAAyB,IAAI,GAAG,EAAmB,CAAC;IACpE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAuB,eAAe,CAAC,0BAA0B,EAAE,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACvH,MAAM,QAAQ,GAA6B,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACvE;YACE,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;SAClC,EACD,EAAE,CACH,CAAC;QACF,MAAM,IAAI,GAAW,MAAM,eAAe,CAAC,gBAAgB,CACzD,QAAQ,EACR,MAAM,CACP,CAAC;QACF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExB,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,cAAc,GAAqC,IAAI,GAAG,EAA+B,CAAC;IAChG,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,0BAA0B,EAAE,sCAAsC,CAAC,CAAC;QACpH,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAqB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACtD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEpC,MAAM,GAAG,GAAuB,IAAI,GAAG,EAAiB,CAAC;YAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,GAA8B,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CACzE;oBACE,QAAQ,EAAE,GAAG,CAAC,GAAG;iBAClB,EACD,EAAE,CACH,CAAC;gBACF,MAAM,KAAK,GAAY,MAAM,eAAe,CAAC,gBAAgB,CAC3D,QAAQ,EACR,OAAO,CACR,CAAC;gBAEF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;gBACtB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAEpB,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YAE7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACjC,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,0BAA0B,EAAE,0BAA0B,CAAC,CAAC;QACxG,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YAE5B,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBACtD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,GAAG,CAAE;qBACpE,GAAG,CAAC,CAAC,IAAW,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC;QACF,CAAC;QACD,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;aACjB,IAAI,CAAC,UAAU,CAAC;aAChB,OAAO,CAAC,CAAC,MAAc,EAAQ,EAAE;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,MAAM,UAAU,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEtE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAChD,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;wBAAE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzE,CAAC;YACF,CAAC;QACF,CAAC;QAED,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;aACjD,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAEtC,IAAI,KAAK,GAAY,IAAI,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAa;oBACpB,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChB,GAAG;iBACJ,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE5B,KAAK,GAAG,KAAK,CAAC;YACf,CAAC;QACF,CAAC;IACF,CAAC;SAAM,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACtC,CAAC;AACF,CAAC,CAAC"}

package/dist/queries/llamaguard-unsafe-content.query.d.mts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery.mjs';
+export declare const QUERY: TQuery;

package/dist/queries/llamaguard-unsafe-content.query.mjs

@@ -0,0 +1,90 @@
+import { ECommonsCsvColumnType } from 'tscommons-esm-format';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsCsv } from 'nodecommons-esm-file';
+import { ELLAMA_GUARD_CATEGORYS, ELlamaGuardClassification, prettyLlamaGuardCategory } from '../services/rig-llama-guard.service.mjs';
+export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+    const filename = args.getFirstValue('filename');
+    let urls;
+    { // scope
+        const log = commonsLogDoing('llamaguard-unsafe-content-query', 'Searching for URLs marked unsafe');
+        const results = databaseService.getUrls().find({
+            'llamaguard.classification': 'unsafe',
+            status: { $in: [
+                    EStatus.ACTIVE,
+                    EStatus.DENY,
+                    EStatus.DISALLOWED,
+                    EStatus.DONE,
+                    EStatus.FAILED,
+                    EStatus.QUEUED
+                ] }
+        }, {});
+        urls = await databaseService.listQueryResults(results, (_t) => true);
+        log.result(urls.length);
+    }
+    const map = new Map();
+    { // scope
+        const log = commonsLogDoing('llamaguard-unsafe-content-query', 'Inverting categorisation');
+        for (const category of ELLAMA_GUARD_CATEGORYS)
+            map.set(category, []);
+        for (const url of urls) {
+            if (url.llamaguard.classification === ELlamaGuardClassification.SAFE)
+                continue;
+            for (const category of url.llamaguard.categories)
+                map.get(category).push(url);
+        }
+        log.success();
+    }
+    const rows = [];
+    { // scope
+        const log = commonsLogDoing('llamaguard-unsafe-content-query', 'Building CSV array');
+        for (const category of ELLAMA_GUARD_CATEGORYS) {
+            const meaning = prettyLlamaGuardCategory(category);
+            for (const url of map.get(category)) {
+                rows.push({
+                    category: category,
+                    meaning: meaning,
+                    domain: url.domain,
+                    url: url.url
+                });
+            }
+        }
+        rows
+            .sort((a, b) => {
+            if (a.category < b.category)
+                return -1;
+            if (a.category > b.category)
+                return 1;
+            if (a.domain < b.domain)
+                return -1;
+            if (a.domain > b.domain)
+                return 1;
+            if (a.url < b.url)
+                return -1;
+            if (a.url > b.url)
+                return 1;
+            return 0;
+        });
+        log.success();
+    }
+    const csv = new CommonsCsv([
+        {
+            name: 'category',
+            type: ECommonsCsvColumnType.STRING
+        },
+        {
+            name: 'meaning',
+            type: ECommonsCsvColumnType.STRING
+        },
+        {
+            name: 'domain',
+            type: ECommonsCsvColumnType.STRING
+        },
+        {
+            name: 'url',
+            type: ECommonsCsvColumnType.STRING
+        }
+    ]);
+    csv.save(rows, filename, true);
+};
+//# sourceMappingURL=llamaguard-unsafe-content.query.mjs.map

package/dist/queries/llamaguard-unsafe-content.query.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"llamaguard-unsafe-content.query.mjs","sourceRoot":"","sources":["../../src/queries/llamaguard-unsafe-content.query.mts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAG7D,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAMlD,OAAO,EAAE,sBAAsB,EAAuB,yBAAyB,EAAE,wBAAwB,EAAqB,MAAM,yCAAyC,CAAC;AAe9K,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,QAAQ,GAAW,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAExD,IAAI,IAA0B,CAAC;IAC/B,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,iCAAiC,EAAE,kCAAkC,CAAC,CAAC;QACvH,MAAM,OAAO,GAA2C,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CACpF;YACE,2BAA2B,EAAE,QAAQ;YACrC,MAAM,EAAE,EAAE,GAAG,EAAE;oBACb,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,UAAU;oBAClB,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;oBACd,OAAO,CAAC,MAAM;iBACf,EAAE;SACJ,EACD,EAAE,CACH,CAAC;QACF,IAAI,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAC3C,OAAO,EACP,CAAC,EAAW,EAA4B,EAAE,CAAC,IAAI,CAChD,CAAC;QACF,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,GAAG,GAAqC,IAAI,GAAG,EAA+B,CAAC;IACrF,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,iCAAiC,EAAE,0BAA0B,CAAC,CAAC;QAC/G,KAAK,MAAM,QAAQ,IAAI,sBAAsB;YAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAErE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,UAAU,CAAC,cAAc,KAAK,yBAAyB,CAAC,IAAI;gBAAE,SAAS;YAE/E,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU;gBAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;IAED,MAAM,IAAI,GAAW,EAAE,CAAC;IACxB,CAAC,CAAC,QAAQ;QACT,MAAM,GAAG,GAAuB,eAAe,CAAC,iCAAiC,EAAE,oBAAoB,CAAC,CAAC;QAEzG,KAAK,MAAM,QAAQ,IAAI,sBAAsB,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAW,wBAAwB,CAAC,QAAQ,CAAC,CAAC;YAE3D,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,QAAQ;oBAClB,OAAO,EAAE,OAAO;oBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;iBACb,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;QAED,IAAI;aACD,IAAI,CAAC,CAAC,CAAO,EAAE,CAAO,EAAU,EAAE;YAClC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;gBAAE,OAAO,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;gBAAE,OAAO,CAAC,CAAC;YAEtC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;gBAAE,OAAO,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;gBAAE,OAAO,CAAC,CAAC;YAElC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;gBAAE,OAAO,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;gBAAE,OAAO,CAAC,CAAC;YAE5B,OAAO,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEL,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAe,IAAI,UAAU,CAAC;QACrC;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;QACD;YACE,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,qBAAqB,CAAC,MAAM;SACnC;KACF,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,CACN,IAAI,EACJ,QAAQ,EACR,IAAI,CACL,CAAC;AACH,CAAC,CAAC"}

package/dist/queries/readability-score.query.d.mts

@@ -0,0 +1,2 @@
+import { TQuery } from '../types/tquery.mjs';
+export declare const QUERY: TQuery;

package/dist/queries/{readability-score.query.js → readability-score.query.mjs}

@@ -1,21 +1,12 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult } from 'nodecommons-es-cli';
-export const QUERY = (args, databaseService, _lists, _expiry, _parsersConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const domain = args.getString('domain');
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
+export const QUERY = async (args, databaseService, _lists, _expiry, _parsersConfig) => {
+    const domain = args.getFirstValue('domain');
     if (!domain) {
-        commonsOutputError('No domain specified');
+        commonsLogError('No domain specified');
         return;
     }
-    commonsOutputDoing(`Scoring and ordering readability of DONE URLs for domain ${domain}`);
+    const log = commonsLogDoing('readability-core', `Scoring and ordering readability of DONE URLs for domain ${domain}`);
     const results = databaseService.getUrls().aggregate([
         { $match: {
                 status: { $ne: EStatus.ARCHIVED },
@@ -64,16 +55,16 @@ export const QUERY = (args, databaseService, _lists, _expiry, _parsersConfig) =>
     while (true) {
         tally++;
         if ((tally % 100) === 0)
-            commonsOutputProgress(`${tally}`);
-        const row = yield results.next();
+            log.progress(`${tally}`);
+        const row = await results.next();
         if (row === null)
             break;
         matches.push(row);
     }
-    commonsOutputResult(tally);
+    log.result(tally);
     console.log('url\tparagraphs\tsentences\twords\tfkre\tari\tscore');
     for (const match of matches) {
         console.log(`${match.url}\t${match.stats.paragraphs}\t${match.stats.sentences}\t${match.stats.words}\t${match.stats.fkre}\t${match.stats.ari}\t${match.score}`);
     }
-});
-//# sourceMappingURL=readability-score.query.js.map
+};
+//# sourceMappingURL=readability-score.query.mjs.map

package/dist/queries/readability-score.query.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability-score.query.mjs","sourceRoot":"","sources":["../../src/queries/readability-score.query.mts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAuB3F,MAAM,CAAC,MAAM,KAAK,GAAW,KAAK,EAChC,IAAiB,EACjB,eAAgC,EAChC,MAAa,EACb,OAAe,EACf,cAA8C,EAC/B,EAAE;IAClB,MAAM,MAAM,GAAqB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACvC,OAAO;IACR,CAAC;IAED,MAAM,GAAG,GAAuB,eAAe,CAAC,kBAAkB,EAAE,4DAA4D,MAAM,EAAE,CAAC,CAAC;IAE1I,MAAM,OAAO,GAA+B,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAU;QACvF,EAAE,MAAM,EAAE;gBACR,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,sBAAsB,EAAE,WAAW;aACpC,EAAE;QACH,EAAE,MAAM,EAAE;gBACR,2BAA2B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,0BAA0B,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;aAC3C,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE;oBACL,UAAU,EAAE,kCAAkC;oBAC9C,SAAS,EAAE,iCAAiC;oBAC5C,KAAK,EAAE,6BAA6B;oBACpC,IAAI,EAAE,4BAA4B;oBAClC,GAAG,EAAE,2BAA2B;iBACjC;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACL,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,CAAE,EAAE;oBACtD,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,CAAE,EAAE,EAAE,YAAY,CAAE,EAAE,CAAE,EAAE;iBACpD;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE;oBACR,IAAI,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,GAAG,EAAE,aAAa,CAAE,EAAE,EAAE,GAAG,CAAE,EAAE;oBACjE,GAAG,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,SAAS,EAAE,CAAE,YAAY,EAAE,CAAC,CAAE,EAAE,EAAE,EAAE,CAAE,EAAE;iBAC7D;aACF,EAAE;QACH,EAAE,QAAQ,EAAE;gBACV,GAAG,EAAE,IAAI;gBACT,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,CAAE,EAAE,IAAI,EAAE,CAAE,gBAAgB,EAAE,eAAe,CAAE,EAAE,EAAE,CAAC,CAAE,EAAE;aAC3E,EAAE;QACH,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAc,EAAE,CAAC;IAC9B,IAAI,KAAK,GAAW,CAAC,CAAC;IACtB,OAAO,IAAI,EAAE,CAAC;QACb,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;YAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;QAElD,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,IAAI;YAAE,MAAM;QAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElB,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACjK,CAAC;AACF,CAAC,CAAC"}

package/dist/servers/{crawl.server.d.ts → crawl.server.d.mts}

@@ -1,12 +1,12 @@
-import { TKeyObject } from 'tscommons-es-core';
-import { Lists } from '../classes/lists';
-import { Tracker } from '../classes/tracker';
-import { DatabaseService } from '../services/database.service';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { THydraConfig } from '../types/thydra-config';
-import { TCrawlConfig } from '../types/tcrawl-config';
-import { TRobotsConfig } from '../types/trobots-config';
-import { TParserCtor } from '../types/tparser-ctor';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { Lists } from '../classes/lists.mjs';
+import { Tracker } from '../classes/tracker.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { THydraConfig } from '../types/thydra-config.mjs';
+import { TCrawlConfig } from '../types/tcrawl-config.mjs';
+import { TRobotsConfig } from '../types/trobots-config.mjs';
+import { TParserCtor } from '../types/tparser-ctor.mjs';
 export declare class CrawlServer {
     private database;
     private hydraConfig;
@@ -23,7 +23,7 @@ export declare class CrawlServer {
     private isPaused;
     private domainRestrictTo;
     set restrictTo(domains: string[]);
-    constructor(database: DatabaseService, hydraConfig: THydraConfig, crawlConfig: TCrawlConfig, parsersConfig: TKeyObject<IParserConfig>, robotsConfig: TRobotsConfig, lists: Lists, tracker: Tracker);
+    constructor(database: DatabaseService, hydraConfig: THydraConfig, crawlConfig: TCrawlConfig, parsersConfig: TPropertyObject<IParserConfig>, robotsConfig: TRobotsConfig, lists: Lists, tracker: Tracker);
     addParser(parser: TParserCtor): void;
     listParsers(): TParserCtor[];
     pause(): void;

package/dist/servers/crawl.server.mjs

@@ -0,0 +1,192 @@
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-esm-async';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogAlert, commonsLogDebug, commonsLogDoing, commonsLogInfo } from 'nodecommons-esm-log';
+import { commonsGracefulAbortAddCallback } from 'nodecommons-esm-process';
+import { Crawler } from '../classes/crawler.mjs';
+import { EAvailableStrategy } from '../enums/eavailable-strategy.mjs';
+export class CrawlServer {
+    database;
+    hydraConfig;
+    crawlConfig;
+    parsersConfig;
+    robotsConfig;
+    lists;
+    tracker;
+    parsers = [];
+    crawlers;
+    strategies;
+    isAborted = false;
+    preDelayIds = [];
+    isPaused = false;
+    domainRestrictTo = [];
+    set restrictTo(domains) {
+        this.domainRestrictTo = domains;
+    }
+    constructor(database, hydraConfig, crawlConfig, parsersConfig, robotsConfig, lists, tracker) {
+        this.database = database;
+        this.hydraConfig = hydraConfig;
+        this.crawlConfig = crawlConfig;
+        this.parsersConfig = parsersConfig;
+        this.robotsConfig = robotsConfig;
+        this.lists = lists;
+        this.tracker = tracker;
+        this.crawlers = new Map();
+        this.strategies = new Map();
+        commonsGracefulAbortAddCallback(() => {
+            commonsLogAlert('SIGINT abort flag is set. Aborting crawl server.');
+            this.abort();
+            for (const preDelayId of this.preDelayIds)
+                commonsAsyncAbortTimeout(preDelayId);
+        });
+    }
+    addParser(parser) {
+        this.parsers.push(parser);
+    }
+    listParsers() {
+        return this.parsers.slice();
+    }
+    pause() {
+        commonsLogAlert('Pausing crawl server');
+        this.isPaused = true;
+        for (const domain of this.crawlers.keys()) {
+            this.pauseCrawl(domain);
+        }
+    }
+    resume() {
+        commonsLogInfo('Resuming crawl server');
+        this.isPaused = false;
+        for (const domain of this.crawlers.keys()) {
+            this.resumeCrawl(domain);
+        }
+    }
+    abort() {
+        this.isAborted = true;
+        commonsAsyncAbortTimeout('find-new-available');
+    }
+    async terminate() {
+        this.abort();
+        await this.database.close();
+        process.exit(0);
+    }
+    pauseCrawl(domain) {
+        if (!this.crawlers.has(domain))
+            return;
+        commonsLogAlert(`Pausing crawler for ${domain}`);
+        this.crawlers.get(domain).pause();
+    }
+    resumeCrawl(domain) {
+        if (!this.crawlers.has(domain))
+            return;
+        commonsLogInfo(`Resuming crawler for ${domain}`);
+        this.crawlers.get(domain).resume();
+    }
+    async start() {
+        await this.database.resetActive();
+        const added = await this.database.queue(this.hydraConfig.startUrl);
+        if (added)
+            this.tracker.delta(EStatus.QUEUED, 1);
+        while (!this.isAborted) {
+            if (!this.isPaused) {
+                const existing = [...this.crawlers.keys()];
+                const space = this.crawlConfig.maxCrawlers - existing.length;
+                if (space > 0) {
+                    commonsLogDebug(`Space for ${space} new crawls available`);
+                    const largestStrategyDomains = existing
+                        .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.LARGEST);
+                    const smallestStrategyDomains = existing
+                        .filter((domain) => this.strategies.get(domain) === EAvailableStrategy.SMALLEST);
+                    commonsLogDebug(`Current crawl has LARGEST=${largestStrategyDomains.length}; SMALLEST=${smallestStrategyDomains.length}`);
+                    let spaceForLargestDomains = 0;
+                    let spaceForSmallestDomains = space;
+                    if (largestStrategyDomains.length >= Math.floor(this.crawlConfig.maxCrawlers / 2)) {
+                        // already 50% large crawling, so all remaining space goes to small crawls
+                    }
+                    else {
+                        // allow a new large crawl, if enough space
+                        if (space > 1) {
+                            spaceForLargestDomains = 1;
+                            spaceForSmallestDomains--;
+                        }
+                    }
+                    commonsLogDebug(`Space for LARGEST=${spaceForLargestDomains}; SMALLEST=${spaceForSmallestDomains}`);
+                    const availablesLargest = await this.database.available(EAvailableStrategy.LARGEST, this.crawlConfig.availableStrategyThreshold, spaceForLargestDomains, existing, this.domainRestrictTo);
+                    existing.push(...availablesLargest);
+                    const availablesSmallest = await this.database.available(EAvailableStrategy.SMALLEST, this.crawlConfig.availableStrategyThreshold, spaceForSmallestDomains, existing, this.domainRestrictTo);
+                    const availables = [
+                        ...availablesSmallest,
+                        ...availablesLargest
+                    ];
+                    if (!this.isAborted) {
+                        for (const domain of availables) {
+                            commonsLogInfo(`Creating new crawl head for ${domain}`);
+                            const crawler = new Crawler(domain, this.database, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.parsers, this.lists, this.tracker);
+                            this.crawlers.set(domain, crawler);
+                            if (availablesLargest.includes(domain)) {
+                                this.strategies.set(domain, EAvailableStrategy.LARGEST);
+                            }
+                            else {
+                                this.strategies.set(domain, EAvailableStrategy.SMALLEST);
+                            }
+                            // called without await in order to do parallel crawls
+                            void (async () => {
+                                try {
+                                    // delay randomly to prevent network socket request spikes
+                                    const preDelayId = `predelay_${domain}`;
+                                    this.preDelayIds.push(preDelayId);
+                                    try {
+                                        await commonsAsyncTimeout(Math.random() * (this.crawlConfig.findNewAvailableDelay - 1000), preDelayId);
+                                    }
+                                    catch (e) {
+                                        if (e.message === 'abortTimeout called')
+                                            return;
+                                        throw e;
+                                    }
+                                    this.preDelayIds = this.preDelayIds
+                                        .filter((pid) => pid !== preDelayId);
+                                    if (this.isAborted)
+                                        return;
+                                    if (this.isPaused)
+                                        this.pauseCrawl(domain);
+                                    commonsLogInfo(`Starting crawler for ${domain}`);
+                                    await crawler.crawl();
+                                }
+                                catch (ex) {
+                                    commonsLogDebug('debug position 6');
+                                    console.log(ex);
+                                }
+                                finally {
+                                    this.crawlers.delete(domain); // doesn't get called until after doCrawl() is called, so have to do it here if aborted
+                                    this.strategies.delete(domain);
+                                }
+                            })();
+                        }
+                    }
+                }
+            }
+            try {
+                await commonsAsyncTimeout(this.crawlConfig.findNewAvailableDelay, 'find-new-available');
+            }
+            catch (_ex) {
+                // ignore
+            }
+            if (this.isAborted) {
+                commonsLogAlert('SIGINT abort is set. Aborting new head loop.');
+                break;
+            }
+        }
+    }
+    async shutdown() {
+        const log = commonsLogDoing('crawler', 'Waiting for all crawlers to abort');
+        while (this.crawlers.size > 0) {
+            log.progress(this.crawlers.size);
+            try {
+                await commonsAsyncTimeout(1000);
+            }
+            catch (_ex) {
+                // ignore
+            }
+        }
+        log.success();
+    }
+}
+//# sourceMappingURL=crawl.server.mjs.map

package/dist/servers/crawl.server.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.server.mjs","sourceRoot":"","sources":["../../src/servers/crawl.server.mts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,cAAc,EAAsB,MAAM,qBAAqB,CAAC;AAC5H,OAAO,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAC;AAE1E,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAajD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AAEtE,MAAM,OAAO,WAAW;IAgBb;IACA;IACA;IACA;IACA;IACA;IACA;IArBF,OAAO,GAAkB,EAAE,CAAC;IAC5B,QAAQ,CAAuB;IAC/B,UAAU,CAAkC;IAC5C,SAAS,GAAY,KAAK,CAAC;IAE3B,WAAW,GAAa,EAAE,CAAC;IAE3B,QAAQ,GAAY,KAAK,CAAC;IAE1B,gBAAgB,GAAa,EAAE,CAAC;IACxC,IAAW,UAAU,CAAC,OAAiB;QACtC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;IACjC,CAAC;IAED,YACU,QAAyB,EACzB,WAAyB,EACzB,WAAyB,EACzB,aAA6C,EAC7C,YAA2B,EAC3B,KAAY,EACZ,OAAgB;QANhB,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAAgC;QAC7C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAS;QAEzB,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;QAExD,+BAA+B,CAAC,GAAS,EAAE;YAC1C,eAAe,CAAC,kDAAkD,CAAC,CAAC;YACpE,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW;gBAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEM,SAAS,CAAC,MAAmB;QACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAEM,WAAW;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;IAEM,KAAK;QACX,eAAe,CAAC,sBAAsB,CAAC,CAAC;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAErB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;IAEM,MAAM;QACZ,cAAc,CAAC,uBAAuB,CAAC,CAAC;QAExC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAEtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACF,CAAC;IAEO,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;IAChD,CAAC;IAEM,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAEO,UAAU,CAAC,MAAc;QAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,eAAe,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC;IAEO,WAAW,CAAC,MAAc;QACjC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAEvC,cAAc,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;QAEjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,MAAM,EAAE,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,KAAK;QACjB,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAElC,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC5E,IAAI,KAAK;YAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEjD,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAa,CAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAE,CAAC;gBAEvD,MAAM,KAAK,GAAW,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAErE,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACf,eAAe,CAAC,aAAa,KAAK,uBAAuB,CAAC,CAAC;oBAE3D,MAAM,sBAAsB,GAAa,QAAQ;yBAC9C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,OAAO,CAAC,CAAC;oBAEnG,MAAM,uBAAuB,GAAa,QAAQ;yBAC/C,MAAM,CAAC,CAAC,MAAc,EAAW,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,kBAAkB,CAAC,QAAQ,CAAC,CAAC;oBAEpG,eAAe,CAAC,6BAA6B,sBAAsB,CAAC,MAAM,cAAc,uBAAuB,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE1H,IAAI,sBAAsB,GAAW,CAAC,CAAC;oBACvC,IAAI,uBAAuB,GAAW,KAAK,CAAC;oBAE5C,IAAI,sBAAsB,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;wBACnF,0EAA0E;oBAC3E,CAAC;yBAAM,CAAC;wBACP,2CAA2C;wBAC3C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;4BACf,sBAAsB,GAAG,CAAC,CAAC;4BAC3B,uBAAuB,EAAE,CAAC;wBAC3B,CAAC;oBACF,CAAC;oBAED,eAAe,CAAC,qBAAqB,sBAAsB,cAAc,uBAAuB,EAAE,CAAC,CAAC;oBAEpG,MAAM,iBAAiB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAC/D,kBAAkB,CAAC,OAAO,EAC1B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,sBAAsB,EACtB,QAAQ,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;oBAEF,QAAQ,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,CAAC;oBAEpC,MAAM,kBAAkB,GAAa,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAChE,kBAAkB,CAAC,QAAQ,EAC3B,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAC3C,uBAAuB,EACvB,QAAQ,EACR,IAAI,CAAC,gBAAgB,CACtB,CAAC;oBAEF,MAAM,UAAU,GAAa;wBAC3B,GAAG,kBAAkB;wBACrB,GAAG,iBAAiB;qBACrB,CAAC;oBAEF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;wBACrB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;4BACjC,cAAc,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;4BAExD,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,EACN,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;4BAEF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;4BAEnC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gCACxC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC;4BACzD,CAAC;iCAAM,CAAC;gCACP,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC;4BAC1D,CAAC;4BAED,sDAAsD;4BACtD,KAAK,CAAC,KAAK,IAAmB,EAAE;gCAC/B,IAAI,CAAC;oCACJ,0DAA0D;oCAC1D,MAAM,UAAU,GAAW,YAAY,MAAM,EAAE,CAAC;oCAChD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oCAClC,IAAI,CAAC;wCACJ,MAAM,mBAAmB,CACvB,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,GAAG,IAAI,CAAC,EAC/D,UAAU,CACX,CAAC;oCACH,CAAC;oCAAC,OAAO,CAAC,EAAE,CAAC;wCACZ,IAAK,CAAW,CAAC,OAAO,KAAK,qBAAqB;4CAAE,OAAO;wCAC3D,MAAM,CAAC,CAAC;oCACT,CAAC;oCAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;yCAChC,MAAM,CAAC,CAAC,GAAW,EAAW,EAAE,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;oCAExD,IAAI,IAAI,CAAC,SAAS;wCAAE,OAAO;oCAC3B,IAAI,IAAI,CAAC,QAAQ;wCAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oCAE3C,cAAc,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;oCACjD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;gCACvB,CAAC;gCAAC,OAAO,EAAE,EAAE,CAAC;oCACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;oCACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gCACjB,CAAC;wCAAS,CAAC;oCACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,uFAAuF;oCACrH,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gCAChC,CAAC;4BACF,CAAC,CAAC,EAAE,CAAC;wBACN,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;YAED,IAAI,CAAC;gBACJ,MAAM,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,oBAAoB,CAAC,CAAC;YACzF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,SAAS;YACV,CAAC;YAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,eAAe,CAAC,8CAA8C,CAAC,CAAC;gBAChE,MAAM;YACP,CAAC;QACF,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,QAAQ;QACpB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mCAAmC,CAAC,CAAC;QAChG,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEjC,IAAI,CAAC;gBACJ,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,SAAS;YACV,CAAC;QACF,CAAC;QACD,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;CACD"}

package/dist/servers/{express.server.d.ts → express.server.d.mts}

@@ -1,8 +1,7 @@
-/// <reference types="node" />
 import * as http from 'http';
 import * as express from 'express';
-import { CommonsStrictExpressServer } from 'nodecommons-es-express';
-import { ICommonsExpressConfig } from 'nodecommons-es-express';
+import { CommonsStrictExpressServer } from 'nodecommons-esm-express';
+import { ICommonsExpressConfig } from 'nodecommons-esm-express';
 export declare class ExpressServer extends CommonsStrictExpressServer {
     constructor(ex: express.Express, server: http.Server, config: ICommonsExpressConfig);
 }