hydra-crawler 2.8.4 → 3.0.1

package/dist/apps/requeue-domain.app.mjs

@@ -0,0 +1,52 @@
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+export class RequeueDomainApp extends CommonsApp {
+    domain;
+    done200Only;
+    databaseService;
+    constructor(domain, done200Only) {
+        super('hydra-crawler');
+        this.domain = domain;
+        this.done200Only = done200Only;
+    }
+    getAppName() {
+        return 'Hydra - Requeue domain';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('requeue-domain', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const conditions = {
+            domain: this.domain,
+            status: { $in: [] }
+        };
+        let log;
+        if (this.done200Only) {
+            conditions.status.$in = [EStatus.DONE];
+            conditions.statusCode = 200;
+            log = commonsLogDoing('requeue-domain', `Marking all 200 DONE as QUEUED for ${this.domain}`);
+        }
+        else {
+            conditions.status.$in = [
+                EStatus.DONE,
+                EStatus.FAILED
+            ];
+            log = commonsLogDoing('requeue-domain', `Marking all DONE and FAILED as QUEUED for ${this.domain}`);
+        }
+        await this.databaseService.getUrls()
+            .updateMany(conditions, { $set: { status: EStatus.QUEUED } });
+        log.success();
+    }
+}
+//# sourceMappingURL=requeue-domain.app.mjs.map

package/dist/apps/requeue-domain.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"requeue-domain.app.mjs","sourceRoot":"","sources":["../../src/apps/requeue-domain.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAIrC;IACA;IAJF,eAAe,CAA4B;IAEnD,YACU,MAAc,EACd,WAAoB;QAE7B,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAS;IAG9B,CAAC;IAEM,UAAU;QAChB,OAAO,wBAAwB,CAAC;IACjC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,UAAU,GAMZ;YACF,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;SACpB,CAAC;QAEF,IAAI,GAAuB,CAAC;QAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAE,OAAO,CAAC,IAAI,CAAE,CAAC;YACzC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC;YAE5B,GAAG,GAAG,eAAe,CAAC,gBAAgB,EAAE,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;aAAM,CAAC;YACP,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG;gBACtB,OAAO,CAAC,IAAI;gBACZ,OAAO,CAAC,MAAM;aACf,CAAC;YAEF,GAAG,GAAG,eAAe,CAAC,gBAAgB,EAAE,6CAA6C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACrG,CAAC;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;aACjC,UAAU,CACT,UAAU,EACR,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CACvC,CAAC;QAEJ,GAAG,CAAC,OAAO,EAAE,CAAC;IACf,CAAC;CACD"}

package/dist/apps/{seed.app.d.ts → seed.app.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app.mjs';
 export declare class SeedApp extends CommonsApp implements IInternalHydraCommonListApp {
     private databaseService;
     private lists;

package/dist/apps/seed.app.mjs

@@ -0,0 +1,59 @@
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { EList } from '../enums/elist.mjs';
+export class SeedApp extends CommonsApp {
+    databaseService;
+    lists;
+    constructor() {
+        super('hydra-crawler');
+        this.lists = new Lists();
+    }
+    getAppName() {
+        return 'Hydra - Seed';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('seed', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const unique = [];
+        { // scope
+            const log = commonsLogDoing('seed', 'Building allowlist hostname seeds');
+            const hostnames = this.lists.listHostnames(EList.ALLOW);
+            for (const hostname of hostnames) {
+                if (!unique.includes(hostname))
+                    unique.push(`http://${hostname}/`);
+                if (!unique.includes(`www.${hostname}`))
+                    unique.push(`http://www.${hostname}/`);
+                log.progress(unique.length);
+            }
+            log.result(unique.length);
+        }
+        { // scope
+            const log = commonsLogDoing('seed', 'Seeding into queue');
+            let tally = 0;
+            for (const url of unique) {
+                const outcome = await this.databaseService.queue(url);
+                if (outcome) {
+                    tally++;
+                    log.progress(tally);
+                }
+            }
+            log.result(tally);
+        }
+    }
+}
+//# sourceMappingURL=seed.app.mjs.map

package/dist/apps/seed.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed.app.mjs","sourceRoot":"","sources":["../../src/apps/seed.app.mts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,MAAM,OAAO,OAAQ,SAAQ,UAAU;IAC9B,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,cAAc,CAAC;IACvB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QAClF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,MAAM,EAAE,mCAAmC,CAAC,CAAC;YAC7F,MAAM,SAAS,GAAa,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAElE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,QAAQ,EAAE,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,GAAG,CAAC,CAAC;gBAChF,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7B,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;YAC9E,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{startup.app.d.ts → startup.app.d.mts}

@@ -1,6 +1,6 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app.mjs';
 export declare class StartupApp extends CommonsApp implements IInternalHydraCommonDbApp {
     private databaseService;
     constructor();

package/dist/apps/startup.app.mjs

@@ -0,0 +1,55 @@
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+// load the tables into Mongo's memory
+export class StartupApp extends CommonsApp {
+    databaseService;
+    constructor() {
+        super('hydra-crawler');
+    }
+    getAppName() {
+        return 'Hydra - Startup';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('startup', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const dbo = this.databaseService.getRawDatabase();
+        for (const collection of 'domains,urls,links'.split(',')) {
+            const log = commonsLogDoing('startup', `Loading ${collection} collection into memory (simple enumeration)`);
+            const tally = await dbo.collection(collection).find({}).count();
+            log.result(tally);
+        }
+        { // scope
+            const log = commonsLogDoing('startup', 'Loading domains collection into memory (regex enumeration)');
+            await this.databaseService.getDomains().find({
+                domain: /[0-9]temp[0-9]/
+            }).count();
+            log.success();
+        }
+        { // scope
+            const log = commonsLogDoing('startup', 'Loading urls collection into memory (regex enumeration)');
+            await this.databaseService.getUrls().find({
+                url: /[0-9]temp[0-9]/
+            }).count();
+            log.success();
+        }
+        { // scope
+            const log = commonsLogDoing('startup', 'Loading links collection into memory (regex enumeration)');
+            await this.databaseService.getLinks().find({
+                outgoing: /[0-9]temp[0-9]/
+            }).count();
+            log.success();
+        }
+    }
+}
+//# sourceMappingURL=startup.app.mjs.map

package/dist/apps/startup.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"startup.app.mjs","sourceRoot":"","sources":["../../src/apps/startup.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAMjD,sCAAsC;AAEtC,MAAM,OAAO,UAAW,SAAQ,UAAU;IACjC,eAAe,CAA4B;IAEnD;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;QACrF,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,GAAG,GAAe,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,CAAC;QAE9D,KAAK,MAAM,UAAU,IAAI,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1D,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,WAAW,UAAU,8CAA8C,CAAC,CAAC;YAChI,MAAM,KAAK,GAAW,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;YACxE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4DAA4D,CAAC,CAAC;YACzH,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC;gBAC3C,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,yDAAyD,CAAC,CAAC;YACtH,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACxC,GAAG,EAAE,gBAAgB;aACtB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,0DAA0D,CAAC,CAAC;YACvH,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;aAC3B,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}

package/dist/apps/{unarchive-urls.app.d.ts → unarchive-urls.app.d.mts}

@@ -1,8 +1,8 @@
-import { CommonsApp } from 'nodecommons-es-app';
-import { DatabaseService } from '../services/database.service';
-import { IMatch } from '../interfaces/imatch';
-import { EList } from '../enums/elist';
-import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IMatch } from '../interfaces/imatch.mjs';
+import { EList } from '../enums/elist.mjs';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app.mjs';
 export declare class UnarchiveUrlsApp extends CommonsApp implements IInternalHydraCommonListApp {
     private databaseService;
     private lists;

package/dist/apps/unarchive-urls.app.mjs

@@ -0,0 +1,71 @@
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDoing } from 'nodecommons-esm-log';
+import { CommonsApp } from 'nodecommons-esm-app';
+import { Lists } from '../classes/lists.mjs';
+import { EList } from '../enums/elist.mjs';
+// This only unarchives the status in the urls collection. It doesn't move entries in the archives collection back into urls
+export class UnarchiveUrlsApp extends CommonsApp {
+    databaseService;
+    lists;
+    constructor() {
+        super('hydra-crawler');
+        this.lists = new Lists();
+    }
+    getAppName() {
+        return 'Hydra - Unarchive';
+    }
+    setDatabaseService(databaseService) {
+        this.databaseService = databaseService;
+    }
+    addToList(list, entries) {
+        this.lists.add(list, entries);
+    }
+    async init() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        const log = commonsLogDoing('unarchive-urls', 'Connecting to database');
+        await this.databaseService.init();
+        log.success();
+        await super.init();
+    }
+    async run() {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set');
+        const urls = [];
+        { // scope
+            const log = commonsLogDoing('unarchive-urls', 'Enumerating ARCHIVED URLS');
+            const results = this.databaseService.getUrls()
+                .find({ status: EStatus.ARCHIVED }, {});
+            let tally = 0;
+            let found = 0;
+            while (true) {
+                if ((tally % 100) === 0)
+                    log.progress(`${tally} ${found}`);
+                tally++;
+                const row = await results.next();
+                if (row === null)
+                    break;
+                if (!this.lists.match(EList.ALLOW, row.url))
+                    continue;
+                urls.push(row.url);
+                found++;
+            }
+            log.result(found);
+        }
+        { // scope
+            const log = commonsLogDoing('unarchive-urls', 'Re-QUEUEING matched URLs');
+            let tally = 0;
+            while (urls.length > 0) {
+                log.progress(tally);
+                tally += 100;
+                const batch = urls.slice(0, 100);
+                await this.databaseService.getUrls().updateMany({ url: { $in: batch } }, {
+                    $set: { status: EStatus.QUEUED }
+                });
+                urls.splice(0, 100);
+            }
+            log.success();
+        }
+    }
+}
+//# sourceMappingURL=unarchive-urls.app.mjs.map

package/dist/apps/unarchive-urls.app.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"unarchive-urls.app.mjs","sourceRoot":"","sources":["../../src/apps/unarchive-urls.app.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAsB,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAM7C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAI3C,4HAA4H;AAE5H,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACvC,eAAe,CAA4B;IAE3C,KAAK,CAAQ;IAErB;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEe,KAAK,CAAC,IAAI;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,wBAAwB,CAAC,CAAC;QAC5F,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAClC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAEe,KAAK,CAAC,GAAG;QACxB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAEhF,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,2BAA2B,CAAC,CAAC;YAE/F,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,EAC5B,EAAE,CACH,CAAC;YAEJ,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAC3D,KAAK,EAAE,CAAC;gBAER,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,gBAAgB,EAAE,0BAA0B,CAAC,CAAC;YAC9F,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,IAAI,GAAG,CAAC;gBAEb,MAAM,KAAK,GAAa,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAE3C,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;iBACjC,CACF,CAAC;gBAEF,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;CACD"}

package/dist/classes/{cleaner.d.ts → cleaner.d.mts}

@@ -1,5 +1,5 @@
-import { Lists } from '../classes/lists';
-import { DatabaseService } from '../services/database.service';
+import { Lists } from '../classes/lists.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class Cleaner {
     private lists;
     private databaseService;

package/dist/classes/cleaner.mjs

@@ -0,0 +1,257 @@
+import { commonsTypeHasPropertyNumber } from 'tscommons-esm-core';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogDebug, commonsLogDoing } from 'nodecommons-esm-log';
+import { EList } from '../enums/elist.mjs';
+export class Cleaner {
+    lists;
+    databaseService;
+    constructor(lists, databaseService) {
+        this.lists = lists;
+        this.databaseService = databaseService;
+    }
+    async detectStatusOrphans(statuses, hardLimit) {
+        if (!this.databaseService)
+            throw new Error('Database service has not been set yet');
+        let tally = 0;
+        let found = 0;
+        const log = commonsLogDoing('cleaner', `Detecting ${statuses.join(', ')} orphan urls`);
+        while (true) {
+            const urls = this.databaseService.getUrls()
+                .find({
+                $and: [
+                    { status: { $in: statuses } },
+                    { $or: [
+                            { orphan: null },
+                            { orphan: false }
+                        ] }
+                ]
+            }, {})
+                .sort({ _id: 1 })
+                .skip(tally);
+            try {
+                while (true) {
+                    tally++;
+                    if ((tally % 100) === 0)
+                        log.progress(`${tally} urls, ${found} orphans`);
+                    if (hardLimit && tally >= hardLimit)
+                        break;
+                    const row = await urls.next();
+                    if (row === null)
+                        break;
+                    const incoming = await this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
+                    if (incoming)
+                        continue;
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                    found++;
+                }
+                break;
+            }
+            catch (err) {
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                    throw err;
+                commonsLogDebug('Code 43 encountered. Continuing');
+            }
+        }
+        log.result(`${tally} urls, ${found} orphans`);
+    }
+    async detectNonAllowlistOrphans(statuses, hardLimit) {
+        let tally = 0;
+        let found = 0;
+        const log = commonsLogDoing('cleaner', `Detecting non-allowlist ${statuses.join(', ')} orphan urls`);
+        while (true) {
+            const urls = this.databaseService.getUrls()
+                .find({
+                $and: [
+                    { status: { $in: statuses } },
+                    { $or: [
+                            { orphan: null },
+                            { orphan: false }
+                        ] }
+                ]
+            }, {})
+                .sort({ _id: 1 })
+                .skip(tally);
+            try {
+                while (true) {
+                    tally++;
+                    if ((tally % 100) === 0)
+                        log.progress(`${tally} urls, ${found} orphans`);
+                    if (hardLimit && tally >= hardLimit)
+                        break;
+                    const row = await urls.next();
+                    if (row === null)
+                        break;
+                    if (this.lists.match(EList.ALLOW, row.url))
+                        continue;
+                    const incoming = await this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
+                    if (incoming)
+                        continue;
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                    found++;
+                }
+                break;
+            }
+            catch (err) {
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                    throw err;
+                commonsLogDebug('Code 43 encountered. Continuing');
+            }
+        }
+        log.result(`${tally} urls, ${found} orphans`);
+    }
+    async detectStatusCodeOrphans(gt, lt, hardLimit) {
+        let tally = 0;
+        let found = 0;
+        const log = commonsLogDoing('cleaner', `Detecting DONE status code between ${gt}<${lt} orphan urls`);
+        while (true) {
+            const urls = this.databaseService.getUrls()
+                .find({
+                $and: [
+                    { status: EStatus.DONE },
+                    { statusCode: { $gt: gt } },
+                    { statusCode: { $lt: lt } },
+                    { $or: [
+                            { orphan: null },
+                            { orphan: false }
+                        ] }
+                ]
+            }, {})
+                .sort({ _id: 1 })
+                .skip(tally);
+            try {
+                while (true) {
+                    tally++;
+                    if ((tally % 100) === 0)
+                        log.progress(`${tally} urls, ${found} orphans`);
+                    if (hardLimit && tally >= hardLimit)
+                        break;
+                    const row = await urls.next();
+                    if (row === null)
+                        break;
+                    const incoming = await this.databaseService.getLinks().findOne({ outgoing: row.url }, { limit: 1 });
+                    if (incoming)
+                        continue;
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                    await this.databaseService.getUrls().updateOne({ _id: row['_id'] }, { $set: { orphan: true } });
+                    found++;
+                }
+                break;
+            }
+            catch (err) {
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                    throw err;
+                commonsLogDebug('Code 43 encountered. Continuing');
+            }
+        }
+        log.result(`${tally} urls, ${found} orphans`);
+    }
+    async purgeOrphanUrls(hardLimit) {
+        await this.detectStatusOrphans([
+            EStatus.DENY,
+            EStatus.FAILED,
+            EStatus.DEAD,
+            EStatus.DISALLOWED
+        ], hardLimit);
+        await this.detectNonAllowlistOrphans([
+            EStatus.QUEUED,
+            EStatus.DONE
+        ], hardLimit);
+        await this.detectStatusCodeOrphans(300, 310, hardLimit);
+        await this.detectStatusCodeOrphans(400, 500, hardLimit);
+        let tally = 0;
+        { // scope
+            const log = commonsLogDoing('clealer', 'Removing orphan outgoing links');
+            while (true) {
+                const urls2 = this.databaseService.getUrls()
+                    .find({ orphan: true }, {})
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            log.progress(tally);
+                        const row = await urls2.next();
+                        if (row === null)
+                            break;
+                        await this.databaseService.getLinks().deleteMany({ url: row.url });
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
+                    commonsLogDebug('Code 43 encountered. Continuing');
+                }
+            }
+            log.result(tally);
+        }
+        { // scope
+            const log = commonsLogDoing('clealer', 'Archiving orphans');
+            await this.databaseService.getUrls().updateMany({ orphan: true }, {
+                $set: { status: EStatus.ARCHIVED },
+                $unset: {
+                    orphan: true,
+                    headers: true,
+                    server: true,
+                    hash: true,
+                    hashSet: true,
+                    ttl: true,
+                    reason: true,
+                    links: true,
+                    title: true
+                }
+            });
+            log.success();
+        }
+    }
+    async purgeEmptyDomains() {
+        let tally = 0;
+        let found = 0;
+        const domains = [];
+        { // scope
+            const log = commonsLogDoing('clealer', 'Enumerating domains');
+            const results = this.databaseService.getDomains().find({}, {});
+            try {
+                tally = 0;
+                while (true) {
+                    tally++;
+                    if ((tally % 100) === 0)
+                        log.progress(tally);
+                    const row = await results.next();
+                    if (row === null)
+                        break;
+                    domains.push(row.domain);
+                }
+            }
+            catch (err) {
+                if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                    throw err;
+            }
+            log.result(tally);
+        }
+        { // scope
+            tally = 0;
+            found = 0;
+            const log = commonsLogDoing('clealer', 'Detecting and removing empty domains');
+            for (const domain of domains) {
+                tally++;
+                if ((tally % 10) === 0)
+                    log.progress(`${tally} domains, ${found} removed`);
+                const urls = await this.databaseService.getUrls()
+                    .findOne({
+                    domain: domain,
+                    status: { $ne: EStatus.ARCHIVED }
+                }, { limit: 1 });
+                if (urls)
+                    continue;
+                await this.databaseService.getDomains().deleteOne({ domain: domain });
+                found++;
+            }
+            log.result(`${tally} domains, ${found} removed`);
+        }
+    }
+}
+//# sourceMappingURL=cleaner.mjs.map

package/dist/classes/cleaner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleaner.mjs","sourceRoot":"","sources":["../../src/classes/cleaner.mts"],"names":[],"mappings":"AAEA,OAAO,EAAE,4BAA4B,EAAmB,MAAM,oBAAoB,CAAC;AAGnF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAM3F,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,KAAY,EACZ,eAAgC;QADhC,UAAK,GAAL,KAAK,CAAO;QACZ,oBAAe,GAAf,eAAe,CAAiB;IACvC,CAAC;IAEI,KAAK,CAAC,mBAAmB,CAC/B,QAAmB,EACnB,SAAkB;QAEnB,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAEpF,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAE3G,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBAClE,IAAI,CACH;gBACE,IAAI,EAAE;oBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;oBAC7B,EAAE,GAAG,EAAE;4BACL,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBAClB,EAAE;iBACJ;aACF,EACD,EAAE,CACH;iBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;oBAEzE,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;wBAAE,MAAM;oBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;oBACzC,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;oBACF,IAAI,QAAQ;wBAAE,SAAS;oBAEvB,mEAAmE;oBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAG,GAAuB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;oBACrH,KAAK,EAAE,CAAC;gBACT,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;gBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,yBAAyB,CACrC,QAAmB,EACnB,SAAkB;QAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,2BAA2B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEzH,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBAClE,IAAI,CACH;gBACE,IAAI,EAAE;oBACJ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;oBAC7B,EAAE,GAAG,EAAE;4BACL,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBAClB,EAAE;iBACJ;aACF,EACD,EAAE,CACH;iBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;oBAEzE,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;wBAAE,MAAM;oBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;oBACzC,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAErD,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;oBACF,IAAI,QAAQ;wBAAE,SAAS;oBAEvB,mEAAmE;oBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAG,GAAuB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;oBACrH,KAAK,EAAE,CAAC;gBACT,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;gBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,EAAU,EACV,EAAU,EACV,SAAkB;QAEnB,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAEzH,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBAClE,IAAI,CACH;gBACE,IAAI,EAAE;oBACJ,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE;oBACxB,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;oBAC3B,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE;oBAC3B,EAAE,GAAG,EAAE;4BACL,EAAE,MAAM,EAAE,IAAI,EAAE;4BAChB,EAAE,MAAM,EAAE,KAAK,EAAE;yBAClB,EAAE;iBACJ;aACF,EACD,EAAE,CACH;iBACA,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;iBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;YAEf,IAAI,CAAC;gBACJ,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;oBAEzE,IAAI,SAAS,IAAI,KAAK,IAAI,SAAS;wBAAE,MAAM;oBAE3C,MAAM,GAAG,GAAc,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;oBACzC,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,MAAM,QAAQ,GAAe,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,CACxE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,EACrB,EAAE,KAAK,EAAE,CAAC,EAAE,CACb,CAAC;oBACF,IAAI,QAAQ;wBAAE,SAAS;oBAEvB,mEAAmE;oBACnE,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAG,GAAuB,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;oBACrH,KAAK,EAAE,CAAC;gBACT,CAAC;gBAED,MAAM;YACP,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;gBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;YACpD,CAAC;QACF,CAAC;QAED,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU,KAAK,UAAU,CAAC,CAAC;IAC/C,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,SAAkB;QAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC7B,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;YACZ,OAAO,CAAC,UAAU;SACnB,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,yBAAyB,CAAC;YACnC,OAAO,CAAC,MAAM;YACd,OAAO,CAAC,IAAI;SACb,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QACxD,MAAM,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;QAExD,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAC;YAC7F,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,KAAK,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACnE,IAAI,CAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;qBAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI,CAAC;oBACJ,OAAO,IAAI,EAAE,CAAC;wBACb,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAE7C,MAAM,GAAG,GAAc,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBACpE,CAAC;oBAED,MAAM;gBACP,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;oBAC7E,eAAe,CAAC,iCAAiC,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAChF,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,MAAM,EAAE,IAAI,EAAE,EAChB;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE;oBACN,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,IAAI;oBACZ,IAAI,EAAE,IAAI;oBACV,OAAO,EAAE,IAAI;oBACb,GAAG,EAAE,IAAI;oBACT,MAAM,EAAE,IAAI;oBACZ,KAAK,EAAE,IAAI;oBACX,KAAK,EAAE,IAAI;iBACZ;aACF,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,iBAAiB;QAC7B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;YAElF,MAAM,OAAO,GAAgC,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAU,EAAE,EAAE,EAAE,CAAC,CAAC;YAErG,IAAI,CAAC;gBACJ,KAAK,GAAG,CAAC,CAAC;gBAEV,OAAO,IAAI,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC;oBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;wBAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAE7C,MAAM,GAAG,GAAiB,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,IAAI;wBAAE,MAAM;oBAExB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;oBAAE,MAAM,GAAG,CAAC;YAC9E,CAAC;YAED,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,KAAK,GAAG,CAAC,CAAC;YAAC,KAAK,GAAG,CAAC,CAAC;YACrB,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,sCAAsC,CAAC,CAAC;YACnG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;gBAE3E,MAAM,IAAI,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACzD,OAAO,CAAO;oBACb,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE;iBAClC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,IAAI,IAAI;oBAAE,SAAS;gBAEnB,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;gBAEtE,KAAK,EAAE,CAAC;YACT,CAAC;YACD,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAa,KAAK,UAAU,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;CAED"}

package/dist/classes/{crawler.d.ts → crawler.d.mts}

@@ -1,16 +1,14 @@
-/// <reference types="node" />
-/// <reference types="node" />
 import * as http from 'http';
 import * as https from 'https';
-import { TKeyObject } from 'tscommons-es-core';
-import { Lists } from '../classes/lists';
-import { Tracker } from '../classes/tracker';
-import { DatabaseService } from '../services/database.service';
-import { IRequestOutcome } from '../interfaces/irequest-outcome';
-import { IParserConfig } from '../interfaces/iparser-config';
-import { TCrawlConfig } from '../types/tcrawl-config';
-import { TRobotsConfig } from '../types/trobots-config';
-import { TParserCtor } from '../types/tparser-ctor';
+import { TPropertyObject } from 'tscommons-esm-core';
+import { Lists } from '../classes/lists.mjs';
+import { Tracker } from '../classes/tracker.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
+import { IRequestOutcome } from '../interfaces/irequest-outcome.mjs';
+import { IParserConfig } from '../interfaces/iparser-config.mjs';
+import { TCrawlConfig } from '../types/tcrawl-config.mjs';
+import { TRobotsConfig } from '../types/trobots-config.mjs';
+import { TParserCtor } from '../types/tparser-ctor.mjs';
 export declare class Crawler {
     private domain;
     private database;
@@ -25,7 +23,7 @@ export declare class Crawler {
     private static pruneHeaders;
     private isPaused;
     private isAborted;
-    constructor(domain: string, database: DatabaseService, crawlConfig: TCrawlConfig, parsersConfig: TKeyObject<IParserConfig>, robotsConfig: TRobotsConfig, parsers: TParserCtor[], lists: Lists, tracker?: Tracker | undefined);
+    constructor(domain: string, database: DatabaseService, crawlConfig: TCrawlConfig, parsersConfig: TPropertyObject<IParserConfig>, robotsConfig: TRobotsConfig, parsers: TParserCtor[], lists: Lists, tracker?: Tracker | undefined);
     private abort;
     pause(): void;
     resume(): void;