hydra-crawler 2.8.4 → 3.0.2

package/dist/apps/cross-populate-import.app.mjs

@@ -0,0 +1,61 @@
+import * as fs from 'fs';
+import * as readline from 'readline';
+import { commonsTypeHasPropertyString } from 'tscommons-esm-core';
+import { commonsLogDoing, commonsLogError } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+function isTRow(test) {
+    if (!commonsTypeHasPropertyString(test, 'domain'))
+        return false;
+    if (!commonsTypeHasPropertyString(test, 'url'))
+        return false;
+    return true;
+}
+export class CrossPopulateImportApp extends CommonsApp {
+    filename;
+    databaseService;
+    constructor(filename) {
+        super('hydra-crawler');
+        this.filename = filename;
+    }
+    getAppName() {
+        return 'Hydra - Cross Populate Export';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('cross-populate-import', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const fileStream = fs.createReadStream(this.filename);
+        const rl = readline.createInterface({
+            input: fileStream,
+            crlfDelay: Infinity
+        });
+        const log = commonsLogDoing('cross-populate-import', 'Importing urls');
+        let tally = 0;
+        for await (const line of rl) {
+            if (line.trim() === '')
+                continue;
+            if (tally % 1000 === 0)
+                log.progress(tally);
+            const json = JSON.parse(line);
+            if (!isTRow(json)) {
+                commonsLogError('Invalid row. Skipping');
+                continue;
+            }
+            await this.databaseService.queue(json.url, false);
+            tally++;
+        }
+        log.result(tally);
+        fileStream.close();
+    }
+}
+//# sourceMappingURL=cross-populate-import.app.mjs.map

package/dist/apps/cross-populate-import.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cross-populate-import.app.mjs","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAElE,OAAO,EAAsB,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAajD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAI3C;IAHF,eAAe,CAA4B;IAEnD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;QACnG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;YACtD,KAAK,EAAE,UAAU;YACjB,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAuB,eAAe,CAAC,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;QAE3F,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YAEjC,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;gBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAE5C,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,eAAe,CAAC,uBAAuB,CAAC,CAAC;gBACzC,SAAS;YACV,CAAC;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClD,KAAK,EAAE,CAAC;QACT,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAElB,UAAU,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;CACD"}

package/dist/apps/{denylist.app.d.ts → denylist.app.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app.mjs';
 export declare class DenylistApp extends CommonsApp implements IInternalHydraCommonListApp {
     private databaseService;
     private lists;

package/dist/apps/denylist.app.mjs

@@ -0,0 +1,135 @@
+import { commonsArrayChunk } from 'tscommons-esm-core';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsArgs } from 'nodecommons-esm-cli';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { EList } from '../enums/elist.mjs';
+export class DenylistApp extends CommonsApp {
+    databaseService;
+    lists;
+    constructor() {
+        super('hydra-crawler');
+        this.lists = new Lists();
+    }
+    getAppName() {
+        return 'Hydra - Denylist';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('denylist', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async forward(limit) {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        let tally = 0;
+        let found = 0;
+        const urls = [];
+        { // scope
+            const log = commonsLogDoing('denylist', 'Searching for non-DENY urls on the denylist');
+            const result = this.databaseService.getUrls()
+                .find({ status: { $nin: [EStatus.DENY, EStatus.ARCHIVED] } }, {});
+            while (true) {
+                tally++;
+                if ((tally % 100) === 0)
+                    log.progress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = await result.next();
+                if (row === null)
+                    break;
+                try {
+                    if (!this.lists.match(EList.DENY, row.url))
+                        continue;
+                }
+                catch (_e) {
+                    // probably an invalid URL
+                    // so mark it as denied
+                }
+                found++;
+                urls.push(row.url);
+            }
+            log.result(found);
+        }
+        { // scope
+            const log = commonsLogDoing('denylist', 'Marking detected as DENY');
+            const batches = commonsArrayChunk(urls, 100);
+            tally = 0;
+            for (const batch of batches) {
+                await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
+                        status: EStatus.DENY
+                    }, $unset: {
+                        ttl: true
+                    } });
+                tally += 100;
+                log.progress(tally);
+            }
+            log.success();
+        }
+    }
+    async reverse(limit) {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        let tally = 0;
+        let found = 0;
+        const urls = [];
+        { // scope
+            const log = commonsLogDoing('denylist', 'Searching for DENY urls not on the denylist');
+            const result = this.databaseService.getUrls()
+                .find({ status: EStatus.DENY }, {});
+            while (true) {
+                tally++;
+                if ((tally % 100) === 0)
+                    log.progress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = await result.next();
+                if (row === null)
+                    break;
+                try {
+                    if (this.lists.match(EList.DENY, row.url))
+                        continue;
+                }
+                catch (_e) {
+                    // probably an invalid URL
+                    // so don't re-enable it
+                    continue;
+                }
+                found++;
+                urls.push(row.url);
+            }
+            log.result(found);
+        }
+        { // scope
+            const log = commonsLogDoing('denylist', 'Marking detected as QUEUED');
+            const batches = commonsArrayChunk(urls, 100);
+            tally = 0;
+            for (const batch of batches) {
+                await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
+                        status: EStatus.QUEUED
+                    }, $unset: {
+                        ttl: true
+                    } });
+                tally += 100;
+                log.progress(tally);
+            }
+            log.success();
+        }
+    }
+    async run() {
+        const args = new CommonsArgs();
+        await this.forward(args.getFirstNumberOrUndefined('limit'));
+        await this.reverse(args.getFirstNumberOrUndefined('limit'));
+    }
+}
+//# sourceMappingURL=denylist.app.mjs.map

package/dist/apps/denylist.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"denylist.app.mjs","sourceRoot":"","sources":["../../src/apps/denylist.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,WAAY,SAAQ,UAAU;IAClC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,wBAAwB,CAAC,CAAC;QACtF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAE,EAAE,EAAE,EACxD,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC;oBACJ,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;gBACtD,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,0BAA0B;oBAC1B,uBAAuB;gBACxB,CAAC;gBAED,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAC;YAExF,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAc;QACnC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,6CAA6C,CAAC,CAAC;YAE3G,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EACxB,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC;oBACJ,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;gBACrD,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,0BAA0B;oBAC1B,wBAAwB;oBACxB,SAAS;gBACV,CAAC;gBAED,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,UAAU,EAAE,4BAA4B,CAAC,CAAC;YAE1F,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;QAE5C,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,CAAC;CACD"}

package/dist/apps/{expire.app.d.ts → expire.app.d.mts}

@@ -1,15 +1,16 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { IExpiry } from '../interfaces/iexpiry';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp } from './internal-hydra-common.app';
+import { CommonsFixedDate } from 'tscommons-esm-core';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { IExpiry } from '../interfaces/iexpiry.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp } from './internal-hydra-common.app.mjs';
 export declare class ExpireApp extends CommonsApp implements IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp {
     private threshold;
     private databaseService;
     private lists;
     private expiry;
-    constructor(threshold: Date | undefined);
+    constructor(threshold: CommonsFixedDate | undefined);
     getAppName(): string;
     setDatabaseService(databaseService: DatabaseService): void;
     addToList(list: EList, entries: IMatch[]): void;

package/dist/apps/expire.app.mjs

@@ -0,0 +1,51 @@
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsArgs } from 'nodecommons-esm-cli';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { Expirer } from '../classes/expirer.mjs';
+import { Expiry } from '../classes/expiry.mjs';
+export class ExpireApp extends CommonsApp {
+    threshold;
+    databaseService;
+    lists;
+    expiry;
+    constructor(threshold) {
+        super('hydra-crawler');
+        this.threshold = threshold;
+        this.lists = new Lists();
+        this.expiry = new Expiry();
+    }
+    getAppName() {
+        return 'Hydra - Expire';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    addToExpiry(expiries) {
+        this.expiry.add(expiries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('expire', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const args = new CommonsArgs();
+        const expirer = new Expirer(this.expiry, this.databaseService);
+        if (this.threshold !== undefined) {
+            await expirer.expireFixed(this.threshold.UTCDate);
+        }
+        else {
+            await expirer.expire(args.getFirstNumberOrUndefined('limit'));
+        }
+    }
+}
+//# sourceMappingURL=expire.app.mjs.map

package/dist/apps/expire.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"expire.app.mjs","sourceRoot":"","sources":["../../src/apps/expire.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAW/C,MAAM,OAAO,SAAU,SAAQ,UAAU;IAM9B;IALF,eAAe,CAA4B;IAC3C,KAAK,CAAQ;IACb,MAAM,CAAS;IAEvB,YACU,SAAqC;QAE9C,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAA4B;QAI9C,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QACpF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;QAE5C,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QAExE,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACP,MAAM,OAAO,CAAC,MAAM,CAClB,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CACxC,CAAC;QACH,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{export-domain-urls.d.ts → export-domain-urls.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class ExportDomainUrlsApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private domain;
     private databaseService;

package/dist/apps/export-domain-urls.mjs

@@ -0,0 +1,85 @@
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+// export the QUEUED, DONE and FAILED urls for importing into another instance
+export class ExportDomainUrlsApp extends CommonsApp {
+    domain;
+    databaseService;
+    constructor(domain) {
+        super('hydra-crawler');
+        this.domain = domain;
+    }
+    getAppName() {
+        return 'Hydra - Export Domain Urls';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('export-domain-urls', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const validOnly = this.args.hasAttribute('valid-only');
+        const pagesOnly = this.args.hasAttribute('pages-only');
+        const mergeHttp = this.args.hasAttribute('merge-http');
+        const matchPipeline = [
+            { $match: { domain: this.domain } }
+        ];
+        if (validOnly) {
+            matchPipeline.push({ $match: {
+                    status: { $in: [EStatus.DONE, EStatus.QUEUED, EStatus.ACTIVE] },
+                    statusCode: { $in: [200, 204] }
+                } });
+        }
+        if (pagesOnly) {
+            matchPipeline.push(...[
+                { $match: {
+                        url: { $not: /[?]/ }
+                    } },
+                { $match: {
+                        url: { $not: /(gif|jpeg|jpg|png|webp|avif)$/i }
+                    } }
+            ]);
+        }
+        const cursor = this.databaseService.getUrls()
+            .aggregate([
+            ...matchPipeline,
+            { $project: { _id: false, url: true } }
+        ]);
+        // safer to do this directly rather than a call to listQueryResults
+        // ditto not using CommonsFile
+        const log = commonsLogDoing('export-domain-urls', `Exporting URLs for domain ${this.domain}`);
+        const urls = [];
+        while (true) {
+            if (urls.length % 1000 === 0)
+                log.progress(urls.length);
+            const row = await cursor.next();
+            if (row === null)
+                break;
+            let url = row.url
+                .trim()
+                .replace(/[?].*$/, '')
+                .replace(/\/index\.(htm|html)$/i, '/')
+                .replace(/\/$/, '')
+                .trim();
+            if (mergeHttp) {
+                url = url.replace(/^http(s?):/, '');
+            }
+            if (!urls.includes(url))
+                urls.push(url);
+        }
+        log.result(urls.length);
+        urls.sort();
+        for (const url of urls) {
+            console.log(url);
+        }
+    }
+}
+//# sourceMappingURL=export-domain-urls.mjs.map

package/dist/apps/export-domain-urls.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"export-domain-urls.mjs","sourceRoot":"","sources":["../../src/apps/export-domain-urls.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAIxC;IAHF,eAAe,CAA4B;IAEnD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,SAAS,GAAY,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QAEhE,MAAM,aAAa,GAA4B;YAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;SACpC,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;oBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;oBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;iBAClC,EAAE,CACJ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,GAAG;gBACpB,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;qBACrB,EAAE;gBACH,EAAE,MAAM,EAAE;wBACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;qBAChD,EAAE;aACJ,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACtF,SAAS,CAAC;YACT,GAAG,aAAa;YAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;SACxC,CAAC,CAAC;QAEL,mEAAmE;QACnE,8BAA8B;QAE9B,MAAM,GAAG,GAAuB,eAAe,CAAC,oBAAoB,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAElH,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC;gBAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAExD,MAAM,GAAG,GAAyB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,GAAG,KAAK,IAAI;gBAAE,MAAM;YAExB,IAAI,GAAG,GAAW,GAAG,CAAC,GAAG;iBACtB,IAAI,EAAE;iBACN,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;iBACrB,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;iBACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;iBAClB,IAAI,EAAE,CAAC;YAEV,IAAI,SAAS,EAAE,CAAC;gBACf,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExB,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{extract-text.app.d.ts → extract-text.app.d.mts}

@@ -1,5 +1,5 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { IInternalHydraCommonApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { IInternalHydraCommonApp } from './internal-hydra-common.app.mjs';
 export declare class ExtractTextApp extends CommonsApp implements IInternalHydraCommonApp {
     private url;
     constructor(url: string);

package/dist/apps/extract-text.app.mjs

@@ -0,0 +1,43 @@
+import * as http from 'http';
+import * as https from 'https';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Crawler } from '../classes/crawler.mjs';
+import { TextParser } from '../parsers/text.parser.mjs';
+// extract the text from the HTML of a URL
+class TextParserInstance extends TextParser {
+    // eslint-disable-next-line @typescript-eslint/require-await
+    async parseText(_database, text) {
+        console.log(text);
+    }
+}
+export class ExtractTextApp extends CommonsApp {
+    url;
+    constructor(url) {
+        super('hydra-crawler');
+        this.url = url;
+    }
+    getAppName() {
+        return 'Hydra - Extract Text';
+    }
+    async run() {
+        const log = commonsLogDoing('extract-text', `Downloading from URL at ${this.url}`);
+        const whatwg = new URL(this.url);
+        let handler;
+        switch (whatwg.protocol) {
+            case 'http:':
+                handler = http;
+                break;
+            case 'https:':
+                handler = https;
+                break;
+            default:
+                throw new Error(`unable to handle protocol ${whatwg.protocol}`);
+        }
+        const outcome = await Crawler.request(handler, this.url, 10000, 4096000);
+        log.success();
+        const parser = new TextParserInstance(outcome);
+        await parser.parse(undefined);
+    }
+}
+//# sourceMappingURL=extract-text.app.mjs.map

package/dist/apps/extract-text.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-text.app.mjs","sourceRoot":"","sources":["../../src/apps/extract-text.app.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAIjD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAOxD,0CAA0C;AAE1C,MAAM,kBAAmB,SAAQ,UAAyB;IACzD,4DAA4D;IAClD,KAAK,CAAC,SAAS,CACvB,SAA0B,EAC1B,IAAY;QAEb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;CACD;AAED,MAAM,OAAO,cAAe,SAAQ,UAAU;IAEnC;IADV,YACU,GAAW;QAEpB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,QAAG,GAAH,GAAG,CAAQ;IAGrB,CAAC;IAEM,UAAU;QAChB,OAAO,sBAAsB,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,MAAM,GAAG,GAAuB,eAAe,CAAC,cAAc,EAAE,2BAA2B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAEvG,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEtC,IAAI,OAA2C,CAAC;QAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,KAAK,OAAO;gBAAE,OAAO,GAAG,IAAI,CAAC;gBAAC,MAAM;YACpC,KAAK,QAAQ;gBAAE,OAAO,GAAG,KAAK,CAAC;gBAAC,MAAM;YACtC;gBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,OAAO,GAAoB,MAAM,OAAO,CAAC,OAAO,CACpD,OAAO,EACP,IAAI,CAAC,GAAG,EACR,KAAK,EACL,OAAO,CACR,CAAC;QACF,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,MAAM,GAAuB,IAAI,kBAAkB,CACvD,OAAO,CACR,CAAC;QACF,MAAM,MAAM,CAAC,KAAK,CAAC,SAAuC,CAAC,CAAC;IAC7D,CAAC;CACD"}

package/dist/apps/{hydra.app.d.ts → hydra.app.d.mts}

@@ -1,13 +1,13 @@
-import { TCommonsScheduleTime } from 'tscommons-es-async';
-import { ICommonsExpressConfig } from 'nodecommons-es-express';
-import { CommonsStrictExpressServer } from 'nodecommons-es-express';
-import { CommonsSocketIoApp } from 'nodecommons-es-app-socket-io';
-import { SocketIoServer } from '../servers/socket-io.server';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { IExpiry } from '../interfaces/iexpiry';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp, IInternalHydraCommonMaintenanceApp } from './internal-hydra-common.app';
+import { TCommonsScheduleTime } from 'tscommons-esm-async';
+import { ICommonsExpressConfig } from 'nodecommons-esm-express';
+import { CommonsStrictExpressServer } from 'nodecommons-esm-express';
+import { CommonsSocketIoApp } from 'nodecommons-esm-app-socket-io';
+import { SocketIoServer } from '../servers/socket-io.server.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { IExpiry } from '../interfaces/iexpiry.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp, IInternalHydraCommonMaintenanceApp } from './internal-hydra-common.app.mjs';
 export declare class HydraApp extends CommonsSocketIoApp<SocketIoServer> implements IInternalHydraCommonListApp, IInternalHydraCommonExpiryApp, IInternalHydraCommonMaintenanceApp {
     private databaseService;
     private lists;