hydra-crawler 2.8.4 → 3.0.2

package/dist/apps/hydra.app.mjs

@@ -0,0 +1,222 @@
+import { commonsStringSplitString, commonsTypeIsTPropertyObjectT } from 'tscommons-esm-core';
+import { commonsLogDebug, commonsLogDoing, commonsLogInfo } from 'nodecommons-esm-log';
+import { commonsExpressBuildDefaultServer, isICommonsExpressConfig } from 'nodecommons-esm-express';
+import { CommonsSocketIoApp } from 'nodecommons-esm-app-socket-io';
+import { AutocompleteApi } from '../apis/autocomplete.api.mjs';
+import { BugsApi } from '../apis/bugs.api.mjs';
+import { CrawlApi } from '../apis/crawl.api.mjs';
+import { DomainsApi } from '../apis/domains.api.mjs';
+import { ImagesApi } from '../apis/images.api.mjs';
+import { StatisticsApi } from '../apis/statistics.api.mjs';
+import { TestApi } from '../apis/test.api.mjs';
+import { UrlsApi } from '../apis/urls.api.mjs';
+import { Lists } from '../classes/lists.mjs';
+import { Expiry } from '../classes/expiry.mjs';
+import { Tracker } from '../classes/tracker.mjs';
+import { Crawler } from '../classes/crawler.mjs';
+import { SocketIoServer } from '../servers/socket-io.server.mjs';
+import { CrawlServer } from '../servers/crawl.server.mjs';
+import { MaintenanceServer } from '../servers/maintenance.server.mjs';
+import { ServerParser } from '../parsers/server.parser.mjs';
+import { HyperlinksParser } from '../parsers/hyperlinks.parser.mjs';
+import { ImageTagsParser } from '../parsers/image-tags.parser.mjs';
+import { JpegParser } from '../parsers/jpeg.parser.mjs';
+import { SpellingParser } from '../parsers/spelling.parser.mjs';
+import { BadWordsParser } from '../parsers/bad-words.parser.mjs';
+import { PhpErrorParser } from '../parsers/php-error.parser.mjs';
+import { AspErrorParser } from '../parsers/asp-error.parser.mjs';
+import { ComplexEnglishParser } from '../parsers/complex-english.parser.mjs';
+import { AccessibilityMetricsParser } from '../parsers/accessibility-metrics.parser.mjs';
+// import { OffenceParser } from '../parsers/offence.parser.mjs';
+// import { InterestParser } from '../parsers/interest.parser.mjs';
+import { LlamaGuardParser } from '../parsers/llama-guard.parser.mjs';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+import { isTHydraConfig } from '../types/thydra-config.mjs';
+import { isTCrawlConfig } from '../types/tcrawl-config.mjs';
+import { isTRobotsConfig } from '../types/trobots-config.mjs';
+export class HydraApp extends CommonsSocketIoApp {
+    databaseService;
+    lists;
+    expiry;
+    hydraConfig;
+    crawlConfig;
+    parsersConfig;
+    robotsConfig;
+    crawlServer;
+    maintenanceTimes = [];
+    maintenanceServer;
+    tracker;
+    constructor() {
+        super('hydra-crawler');
+        this.lists = new Lists();
+        this.expiry = new Expiry();
+        const hydraConfig = this.getConfigArea('hydra');
+        if (!isTHydraConfig(hydraConfig))
+            throw new Error('Hydra config is not valid');
+        this.hydraConfig = hydraConfig;
+        const crawlConfig = this.getConfigArea('crawl');
+        if (!isTCrawlConfig(crawlConfig))
+            throw new Error('Crawl config is not valid');
+        this.crawlConfig = crawlConfig;
+        const parsersConfig = this.getConfigArea('parsers');
+        if (!commonsTypeIsTPropertyObjectT(parsersConfig, isIParserConfig))
+            throw new Error('Parsers config is not valid');
+        this.parsersConfig = parsersConfig;
+        const robotsConfig = this.getConfigArea('robots');
+        if (!isTRobotsConfig(robotsConfig))
+            throw new Error('Robots config is not valid');
+        this.robotsConfig = robotsConfig;
+    }
+    buildHttpServer() {
+        const expressConfig = this.getConfigArea('express');
+        if (!isICommonsExpressConfig(expressConfig))
+            throw new Error('Invalid express config');
+        return commonsExpressBuildDefaultServer(this.httpConfig.port, expressConfig.bodyParserLimit);
+    }
+    buildSocketIoServer(expressServer, expressConfig) {
+        return new SocketIoServer(expressServer, expressConfig);
+    }
+    getAppName() {
+        return 'Hydra';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    addToExpiry(expiries) {
+        this.expiry.add(expiries);
+    }
+    setMaintenanceSchedule(times) {
+        this.maintenanceTimes = times;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        this.install((restServer, path) => {
+            if (!this.databaseService)
+                throw new Error('Database service has not been defined yet');
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing auto-complete API');
+                new AutocompleteApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing bugs API');
+                new BugsApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing crawl API');
+                new CrawlApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing domains API');
+                new DomainsApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing images API');
+                new ImagesApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing statistics API');
+                new StatisticsApi(restServer, this.databaseService, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing test API');
+                new TestApi(restServer, path);
+                log.success();
+            }
+            { // scope
+                const log = commonsLogDoing('hydra', 'Installing urls API');
+                new UrlsApi(restServer, this.databaseService, path);
+                log.success();
+            }
+        });
+        await super.init();
+        const socketIoServer = this.socketIoServer;
+        if (!socketIoServer)
+            throw new Error('SocketIo server has not been set yet');
+        this.tracker = new Tracker(this.databaseService, socketIoServer);
+        this.crawlServer = new CrawlServer(this.databaseService, this.hydraConfig, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.lists, this.tracker);
+        { // scope
+            const log = commonsLogDoing('hydra', 'Connecting to database');
+            await this.databaseService.init();
+            log.success();
+        }
+        for (const parser of [
+            ServerParser,
+            HyperlinksParser,
+            ImageTagsParser,
+            JpegParser,
+            SpellingParser,
+            BadWordsParser,
+            PhpErrorParser,
+            AspErrorParser,
+            ComplexEnglishParser,
+            AccessibilityMetricsParser,
+            // OffenceParser,
+            // InterestParser,
+            LlamaGuardParser
+        ]) {
+            commonsLogDebug(`Installing parser ${parser.name}`);
+            this.crawlServer.addParser(parser);
+            await this.databaseService.initParser(parser);
+        }
+        const maintenanceHardLimit = this.args.getFirstNumberOrUndefined('maintenance-hard-limit');
+        if (maintenanceHardLimit)
+            commonsLogDebug(`Setting maintenance hard limit to ${maintenanceHardLimit}`);
+        this.maintenanceServer = new MaintenanceServer(this.maintenanceTimes, this.expiry, this.lists, this.databaseService, this.crawlServer, maintenanceHardLimit // to prevent running out of heap on very large collections
+        );
+    }
+    listening() {
+        void (async () => {
+            if (!this.crawlServer)
+                throw new Error('No crawl server instantiated! This should not be possible');
+            await this.crawlServer.start();
+            this.abort();
+        })();
+    }
+    async run() {
+        const singleUrl = this.args.getFirstValueOrUndefined('single-url');
+        if (singleUrl) {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            if (!this.crawlServer)
+                throw new Error('Crawl server has not been set yet');
+            const whatwg = new URL(singleUrl);
+            const crawler = new Crawler(whatwg.hostname, this.databaseService, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.crawlServer.listParsers(), this.lists);
+            commonsLogInfo(`Running single crawl of: ${singleUrl}`);
+            const outcome = await crawler.fetch(singleUrl, false);
+            commonsLogInfo(`HTTP status code outcome was ${outcome}`);
+            return;
+        }
+        const domainRestrictTo = this.args.getFirstValueOrUndefined('domain-restrict-to');
+        if (domainRestrictTo) {
+            const domains = commonsStringSplitString(domainRestrictTo, ',', true, true);
+            commonsLogDebug(`Restricting crawl domains to: ${domains.join(',')}`);
+            if (!this.crawlServer)
+                throw new Error('CrawlServer has not been defined! This should not be possible');
+            this.crawlServer.restrictTo = domains;
+        }
+        if (!this.tracker)
+            throw new Error('Tracker has not been defined! This should not be possible');
+        if (!this.maintenanceServer)
+            throw new Error('Maintenance server has not been defined! This should not be possible');
+        this.maintenanceServer.start();
+        void this.tracker.start();
+        await super.run();
+    }
+    async shutdown() {
+        if (!this.crawlServer)
+            throw new Error('No crawl server instantiated! This should not be possible');
+        await this.crawlServer.shutdown();
+        await super.shutdown();
+    }
+}
+//# sourceMappingURL=hydra.app.mjs.map

package/dist/apps/hydra.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"hydra.app.mjs","sourceRoot":"","sources":["../../src/apps/hydra.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,6BAA6B,EAAmB,MAAM,oBAAoB,CAAC;AAG9G,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,cAAc,EAAsB,MAAM,qBAAqB,CAAC;AAC3G,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAG5J,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAItE,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAE,0BAA0B,EAAE,MAAM,6CAA6C,CAAC;AACzF,iEAAiE;AACjE,mEAAmE;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAErE,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAIlF,OAAO,EAAgB,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAgB,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAiB,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAU7E,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IACvD,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IACb,MAAM,CAAS;IAEf,WAAW,CAAe;IAC1B,WAAW,CAAe;IAC1B,aAAa,CAAiC;IAC9C,YAAY,CAAgB;IAE5B,WAAW,CAAwB;IAEnC,gBAAgB,GAA2B,EAAE,CAAC;IAC9C,iBAAiB,CAA8B;IAE/C,OAAO,CAAoB;IAEnC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,6BAA6B,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAClI,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;YACxG,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAExF,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,8BAA8B,CAAC,CAAC;gBACzF,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;gBAChF,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;gBACjF,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;gBACnF,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;gBAClF,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;gBACtF,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;gBAChF,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;YAED,CAAC,CAAC,QAAQ;gBACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;gBAChF,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,GAAG,CAAC,OAAO,EAAE,CAAC;YACf,CAAC;QACF,CAAC,CACF,CAAC;QAEF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,cAAc,CAAC;QACrE,IAAI,CAAC,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;YACnF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,KAAK,MAAM,MAAM,IAAI;YACnB,YAAY;YACZ,gBAAgB;YAChB,eAAe;YACf,UAAU;YACV,cAAc;YACd,cAAc;YACd,cAAc;YACd,cAAc;YACd,oBAAoB;YACpB,0BAA0B;YAC1B,iBAAiB;YACjB,kBAAkB;YAClB,gBAAgB;SACjB,EAAE,CAAC;YACH,eAAe,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,wBAAwB,CAAC,CAAC;QAC7G,IAAI,oBAAoB;YAAE,eAAe,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;QAEvG,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;SACjF,CAAC;IACH,CAAC;IAEkB,SAAS;QAC3B,KAAK,CAAC,KAAK,IAAmB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,SAAS,GAAqB,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,YAAY,CAAC,CAAC;QACrF,IAAI,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACpF,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;YAEF,cAAc,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;YACxD,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC9D,cAAc,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;YAE1D,OAAO;QACR,CAAC;QAED,MAAM,gBAAgB,GAAqB,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;QACpG,IAAI,gBAAgB,EAAE,CAAC;YACtB,MAAM,OAAO,GAAa,wBAAwB,CAAC,gBAAgB,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACtF,eAAe,CAAC,iCAAiC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAEtE,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;YACxG,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;YAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE1B,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;IAEkB,KAAK,CAAC,QAAQ;QAChC,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QAElC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;CACD"}

package/dist/apps/{import.app.d.ts → import.app.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class ImportApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private databaseService;
     constructor();

package/dist/apps/import.app.mjs

@@ -0,0 +1,48 @@
+import { commonsInputReadAllStdIn } from 'nodecommons-esm-cli';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+export class ImportApp extends CommonsApp {
+    databaseService;
+    constructor() {
+        super('hydra-crawler');
+    }
+    getAppName() {
+        return 'Hydra - Import';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('import', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        let urls = [];
+        { // scope
+            const log = commonsLogDoing('import', 'Loading URLs from stdin');
+            urls = await commonsInputReadAllStdIn();
+            log.result(urls.length);
+        }
+        { // scope
+            const log = commonsLogDoing('import', 'Seeding into queue');
+            let tally = 0;
+            let done = 0;
+            for (const url of urls) {
+                if ((++tally % 100) === 0)
+                    log.progress(`${tally}, imported ${done}`);
+                const outcome = await this.databaseService.queue(url);
+                if (outcome) {
+                    done++;
+                }
+            }
+            log.result(done);
+        }
+    }
+}
+//# sourceMappingURL=import.app.mjs.map

package/dist/apps/import.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.app.mjs","sourceRoot":"","sources":["../../src/apps/import.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAChC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;YACrF,IAAI,GAAG,MAAM,wBAAwB,EAAE,CAAC;YACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;YAChF,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,IAAI,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,CAAC,CAAC;gBAEtE,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE,CAAC;oBACb,IAAI,EAAE,CAAC;gBACR,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{internal-hydra-common.app.d.ts → internal-hydra-common.app.d.mts}

@@ -1,9 +1,9 @@
-import { TCommonsScheduleTime } from 'tscommons-es-async';
-import { CommonsConfigFile } from 'nodecommons-es-config';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { IExpiry } from '../interfaces/iexpiry';
-import { EList } from '../enums/elist';
+import { TCommonsScheduleTime } from 'tscommons-esm-async';
+import { CommonsConfigFile } from 'nodecommons-esm-config';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { IExpiry } from '../interfaces/iexpiry.mjs';
+import { EList } from '../enums/elist.mjs';
 export interface IInternalHydraCommonApp {
     loadConfigFile(file: string): CommonsConfigFile;
     loadRawJsonConfigFile(file: string): unknown;

package/dist/apps/{internal-hydra-common.app.js → internal-hydra-common.app.mjs}

@@ -16,4 +16,4 @@ export function isIInternalHydraCommonMaintenanceApp(test) {
         return false;
     return 'setMaintenanceSchedule' in test;
 }
-//# sourceMappingURL=internal-hydra-common.app.js.map
+//# sourceMappingURL=internal-hydra-common.app.mjs.map

package/dist/apps/internal-hydra-common.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"internal-hydra-common.app.mjs","sourceRoot":"","sources":["../../src/apps/internal-hydra-common.app.mts"],"names":[],"mappings":"AAyBA,MAAM,UAAU,2BAA2B,CAAC,IAA6B;IACxE,OAAO,oBAAoB,IAAI,IAAI,CAAC;AACrC,CAAC;AAQD,MAAM,UAAU,6BAA6B,CAAC,IAA6B;IAC1E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,WAAW,IAAI,IAAI,CAAC;AAC5B,CAAC;AAKD,MAAM,UAAU,+BAA+B,CAAC,IAA6B;IAC5E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,aAAa,IAAI,IAAI,CAAC;AAC9B,CAAC;AAKD,MAAM,UAAU,oCAAoC,CAAC,IAA6B;IACjF,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,wBAAwB,IAAI,IAAI,CAAC;AACzC,CAAC"}

package/dist/apps/{move-to-archive.app.d.ts → move-to-archive.app.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class MoveToArchiveApp extends CommonsApp {
     private databaseService;
     constructor();

package/dist/apps/move-to-archive.app.mjs

@@ -0,0 +1,31 @@
+import { commonsLogDoing, commonsLogInfo } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+export class MoveToArchiveApp extends CommonsApp {
+    databaseService;
+    constructor() {
+        super('hydra-crawler');
+    }
+    getAppName() {
+        return 'Hydra - Move to archive';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('move-to-archive', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        const batchSize = this.args.getFirstNumber('batch-size');
+        const limit = this.args.getFirstNumberOrUndefined('limit');
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        commonsLogInfo('Moving ARCHIVED urls into archives collection.');
+        await this.databaseService.moveArchivedUrlsToArchive(batchSize, limit);
+    }
+}
+//# sourceMappingURL=move-to-archive.app.mjs.map

package/dist/apps/move-to-archive.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"move-to-archive.app.mjs","sourceRoot":"","sources":["../../src/apps/move-to-archive.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAIjD,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,yBAAyB,CAAC;IAClC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QAC7F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,SAAS,GAAW,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,KAAK,GAAqB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,cAAc,CAAC,gDAAgD,CAAC,CAAC;QACjE,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACxE,CAAC;CACD"}

package/dist/apps/{prune-archive.app.d.ts → prune-archive.app.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class PruneArchiveApp extends CommonsApp {
     private databaseService;
     constructor();

package/dist/apps/prune-archive.app.mjs

@@ -0,0 +1,40 @@
+import { commonsLogAlert, commonsLogDoing, commonsLogInfo } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+export class PruneArchiveApp extends CommonsApp {
+    databaseService;
+    constructor() {
+        super('hydra-crawler');
+    }
+    getAppName() {
+        return 'Hydra - Prune archive';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('prune-archive', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        commonsLogInfo('Searching for duplicate entries in archiveds table.');
+        const duplicates = await this.databaseService.listDuplicateArchivedUrls();
+        if (this.args.hasAttribute('purge')) {
+            const log = commonsLogDoing('prune-archive', 'Purging duplicates');
+            for (const duplicate of duplicates) {
+                log.progress(duplicate.substring(0, 90));
+                await this.databaseService.purgeArchiveDuplicates(duplicate);
+            }
+            log.success();
+        }
+        else {
+            commonsLogAlert('Specify --purge flag to perform the purge');
+        }
+    }
+}
+//# sourceMappingURL=prune-archive.app.mjs.map

package/dist/apps/prune-archive.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune-archive.app.mjs","sourceRoot":"","sources":["../../src/apps/prune-archive.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,cAAc,EAAsB,MAAM,qBAAqB,CAAC;AAC3G,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAIjD,MAAM,OAAO,eAAgB,SAAQ,UAAU;IACtC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,uBAAuB,CAAC;IAChC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC;QAC3F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,cAAc,CAAC,qDAAqD,CAAC,CAAC;QACtE,MAAM,UAAU,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,EAAE,CAAC;QAEpF,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,MAAM,GAAG,GAAuB,eAAe,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;YAEvF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACpC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAEzC,MAAM,IAAI,CAAC,eAAe,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;YAC9D,CAAC;YAED,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,2CAA2C,CAAC,CAAC;QAC9D,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{query.app.d.ts → query.app.d.mts}

@@ -1,9 +1,9 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { IExpiry } from '../interfaces/iexpiry';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { IExpiry } from '../interfaces/iexpiry.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class QueryApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private queryName;
     private databaseService;

package/dist/apps/query.app.mjs

@@ -0,0 +1,69 @@
+import { commonsTypeIsTPropertyObjectT } from 'tscommons-esm-core';
+import { commonsLogDie, commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { Expiry } from '../classes/expiry.mjs';
+import { isIParserConfig } from '../interfaces/iparser-config.mjs';
+import { QUERY as QUERY_COMPLEX_ENGLISH } from '../queries/complex-english.query.mjs';
+import { QUERY as QUERY_FLASH_CONTENT } from '../queries/flash-content.query.mjs';
+import { QUERY as QUERY_LINKING_TO_DOMAINS } from '../queries/linking-to-domains.query.mjs';
+import { QUERY as QUERY_READABILITY_SCORE } from '../queries/readability-score.query.mjs';
+import { QUERY as QUERY_LLAMAGUARD_UNSAFE_CONTENT } from '../queries/llamaguard-unsafe-content.query.mjs';
+export class QueryApp extends CommonsApp {
+    queryName;
+    databaseService;
+    lists;
+    expiry;
+    parsersConfig;
+    constructor(queryName) {
+        super('hydra-crawler');
+        this.queryName = queryName;
+        if (/[^-a-z0-9]/i.test(queryName))
+            throw new Error('Invalid query name');
+        this.lists = new Lists();
+        this.expiry = new Expiry();
+        const parsersConfig = this.getConfigArea('parsers');
+        if (!commonsTypeIsTPropertyObjectT(parsersConfig, isIParserConfig))
+            throw new Error('Parsers config is not valid');
+        this.parsersConfig = parsersConfig;
+    }
+    getAppName() {
+        return 'Hydra - Query';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    addToExpiry(expiries) {
+        this.expiry.add(expiries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('query', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        let query;
+        if (this.queryName === 'complex-english')
+            query = QUERY_COMPLEX_ENGLISH;
+        if (this.queryName === 'flash-content')
+            query = QUERY_FLASH_CONTENT;
+        if (this.queryName === 'linking-to-domains')
+            query = QUERY_LINKING_TO_DOMAINS;
+        if (this.queryName === 'readability-score')
+            query = QUERY_READABILITY_SCORE;
+        if (this.queryName === 'llamaguard-unsafe-content')
+            query = QUERY_LLAMAGUARD_UNSAFE_CONTENT;
+        if (!query)
+            commonsLogDie('No such available query');
+        await query(this.args, this.databaseService, this.lists, this.expiry, this.parsersConfig);
+    }
+}
+//# sourceMappingURL=query.app.mjs.map

package/dist/apps/query.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"query.app.mjs","sourceRoot":"","sources":["../../src/apps/query.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAmB,MAAM,oBAAoB,CAAC;AAEpF,OAAO,EAAE,aAAa,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAI/C,OAAO,EAAiB,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAQlF,OAAO,EAAE,KAAK,IAAI,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AACtF,OAAO,EAAE,KAAK,IAAI,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AAClF,OAAO,EAAE,KAAK,IAAI,wBAAwB,EAAE,MAAM,yCAAyC,CAAC;AAC5F,OAAO,EAAE,KAAK,IAAI,uBAAuB,EAAE,MAAM,wCAAwC,CAAC;AAC1F,OAAO,EAAE,KAAK,IAAI,+BAA+B,EAAE,MAAM,gDAAgD,CAAC;AAI1G,MAAM,OAAO,QAAS,SAAQ,UAAU;IAS7B;IARF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IACb,MAAM,CAAS;IAEf,aAAa,CAAiC;IAEtD,YACU,SAAiB;QAE1B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAAQ;QAI1B,IAAI,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEzE,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,6BAA6B,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAClI,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACpC,CAAC;IAEM,UAAU;QAChB,OAAO,eAAe,CAAC;IACxB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;QACnF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAuB,CAAC;QAE5B,IAAI,IAAI,CAAC,SAAS,KAAK,iBAAiB;YAAE,KAAK,GAAG,qBAAqB,CAAC;QACxE,IAAI,IAAI,CAAC,SAAS,KAAK,eAAe;YAAE,KAAK,GAAG,mBAAmB,CAAC;QACpE,IAAI,IAAI,CAAC,SAAS,KAAK,oBAAoB;YAAE,KAAK,GAAG,wBAAwB,CAAC;QAC9E,IAAI,IAAI,CAAC,SAAS,KAAK,mBAAmB;YAAE,KAAK,GAAG,uBAAuB,CAAC;QAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,2BAA2B;YAAE,KAAK,GAAG,+BAA+B,CAAC;QAE5F,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;QAErD,MAAM,KAAK,CACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,aAAa,CACnB,CAAC;IACH,CAAC;CACD"}

package/dist/apps/{reattempt.app.d.ts → reattempt.app.d.mts}

@@ -1,9 +1,9 @@
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app.mjs';
 export declare class ReattemptApp extends CommonsApp implements IInternalHydraCommonListApp {
     private include;
     private databaseService;

package/dist/apps/reattempt.app.mjs

@@ -0,0 +1,81 @@
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { EList } from '../enums/elist.mjs';
+export class ReattemptApp extends CommonsApp {
+    include;
+    databaseService;
+    lists;
+    constructor(include) {
+        super('hydra-crawler');
+        this.include = include;
+        this.lists = new Lists();
+    }
+    getAppName() {
+        return 'Hydra - Reattempt';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('reattempt', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const urls = [];
+        { // scope
+            let tally = 0;
+            const log = commonsLogDoing('reattempt', `Searching for allowlisted ${this.include.join(' and ')}`);
+            const results = this.databaseService.getUrls()
+                .find({
+                status: { $in: this.include }
+            }, {});
+            tally = 0;
+            while (true) {
+                if ((tally % 100) === 0)
+                    log.progress(tally);
+                const row = await results.next();
+                if (row === null)
+                    break;
+                try {
+                    if (!this.lists.match(EList.ALLOW, row.url))
+                        continue;
+                }
+                catch (_e) {
+                    // probably an invalid URL
+                    // so ignore
+                    continue;
+                }
+                urls.push(row.url);
+                tally++;
+            }
+            log.result(tally);
+        }
+        { // scope
+            const log = commonsLogDoing('reattempt', 'Requeuing matches');
+            let tally = 0;
+            for (const url of urls) {
+                tally++;
+                if ((tally % 10) === 0)
+                    log.progress(tally);
+                await this.databaseService.getUrls()
+                    .updateOne({ url: url }, {
+                    $set: { status: EStatus.QUEUED },
+                    $unset: { ttl: 1 }
+                });
+            }
+            log.result(tally);
+        }
+    }
+}
+//# sourceMappingURL=reattempt.app.mjs.map

package/dist/apps/reattempt.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"reattempt.app.mjs","sourceRoot":"","sources":["../../src/apps/reattempt.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,YAAa,SAAQ,UAAU;IAMjC;IALF,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;QACvF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAExH,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAE7C,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC;oBACJ,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;gBACvD,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,0BAA0B;oBAC1B,YAAY;oBACZ,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,WAAW,EAAE,mBAAmB,CAAC,CAAC;YAClF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBAE5C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;YACN,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{requeue-domain.app.d.ts → requeue-domain.app.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class RequeueDomainApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private domain;
     private done200Only;