hydra-crawler 1.4.4 → 2.0.0

package/dist/apps/cleanup.app.js

@@ -1,18 +1,25 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CleanupApp = void 0;
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_2 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
-const lists_1 = require("../classes/lists");
-const cleaner_1 = require("../classes/cleaner");
-class CleanupApp extends nodecommons_app_1.CommonsApp {
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { commonsTypeHasPropertyNumber } from 'tscommons-es-core';
+import { isIUrl } from 'hydra-crawler-ts-assets';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
+import { Lists } from '../classes/lists';
+import { Cleaner } from '../classes/cleaner';
+export class CleanupApp extends CommonsApp {
     constructor(includeReset = false, includeLinks = false) {
         super('hydra-crawler');
         this.includeReset = includeReset;
         this.includeLinks = includeLinks;
-        this.lists = new lists_1.Lists();
+        this.lists = new Lists();
     }
     getAppName() {
         return 'Hydra - Cleanup';
@@ -23,100 +30,111 @@ class CleanupApp extends nodecommons_app_1.CommonsApp {
     addToList(list, entries) {
         this.lists.add(list, entries);
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async reset() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        nodecommons_cli_1.CommonsOutput.doing(`Reseting urls for DENY, FAILED, DEAD and DISALLOWED`);
-        let tally = 0;
-        while (true) {
-            const result = this.databaseService.getUrls()
-                .find({ status: { $in: [
-                        hydra_crawler_ts_assets_2.EStatus.DENY,
-                        hydra_crawler_ts_assets_2.EStatus.FAILED,
-                        hydra_crawler_ts_assets_2.EStatus.DEAD,
-                        hydra_crawler_ts_assets_2.EStatus.DISALLOWED
-                    ] } })
-                .sort({ _id: 1 })
-                .skip(tally);
-            try {
-                while (true) {
-                    tally++;
-                    if ((tally % 100) === 0)
-                        nodecommons_cli_1.CommonsOutput.progress(tally);
-                    const row = await result.next();
-                    if (row === null)
-                        break;
-                    await this.databaseService.getLinks().deleteMany({ url: row.url });
+    reset() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            commonsOutputDoing('Reseting urls for DENY, FAILED, DEAD and DISALLOWED');
+            let tally = 0;
+            while (true) {
+                const result = this.databaseService.getUrls()
+                    .find({ status: { $in: [
+                            EStatus.DENY,
+                            EStatus.FAILED,
+                            EStatus.DEAD,
+                            EStatus.DISALLOWED
+                        ] } })
+                    .sort({ _id: 1 })
+                    .skip(tally);
+                try {
+                    while (true) {
+                        tally++;
+                        if ((tally % 100) === 0)
+                            commonsOutputProgress(tally);
+                        const row = yield result.next();
+                        if (row === null)
+                            break;
+                        yield this.databaseService.getLinks().deleteMany({ url: row.url });
+                    }
+                    break;
+                }
+                catch (err) {
+                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
+                        throw err;
                 }
-                break;
-            }
-            catch (err) {
-                if (err.code !== 43)
-                    throw err;
             }
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
+            commonsOutputResult(tally);
+        });
     }
-    async links() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        nodecommons_cli_1.CommonsOutput.doing(`Searching for link orphans`);
-        let tally = 0;
-        const result = this.databaseService.getLinks().find();
-        let batch = [];
-        const orphans = [];
-        while (true) {
-            tally++;
-            if ((tally % 1000) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}, ${orphans.length}`);
-            const link = await result.next();
-            if (link === null)
-                break;
-            if (!batch.includes(link.url))
-                batch.push(link.url);
-            if (!batch.includes(link.outgoing))
-                batch.push(link.outgoing);
-            if (batch.length < 10000)
-                continue;
-            const result2 = this.databaseService.getUrls().find({ url: { $in: batch } });
-            const matches = (await this.databaseService.listQueryResults(result2, hydra_crawler_ts_assets_1.isIUrl))
-                .map((u) => u.url);
-            for (const l of batch) {
-                if (!matches.includes(l))
-                    orphans.push(l);
+    links() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            commonsOutputDoing('Searching for link orphans');
+            let tally = 0;
+            const result = this.databaseService.getLinks().find({}, {});
+            let batch = [];
+            const orphans = [];
+            while (true) {
+                tally++;
+                if ((tally % 1000) === 0)
+                    commonsOutputProgress(`${tally}, ${orphans.length}`);
+                const link = yield result.next();
+                if (link === null)
+                    break;
+                if (!batch.includes(link.url))
+                    batch.push(link.url);
+                if (!batch.includes(link.outgoing))
+                    batch.push(link.outgoing);
+                if (batch.length < 10000)
+                    continue;
+                const result2 = this.databaseService.getUrls().find({ url: { $in: batch } });
+                const matches = (yield this.databaseService.listQueryResults(result2, isIUrl))
+                    .map((u) => u.url);
+                for (const l of batch) {
+                    if (!matches.includes(l))
+                        orphans.push(l);
+                }
+                batch = [];
+            }
+            commonsOutputResult(orphans.length);
+            commonsOutputDoing('Removing orphan links');
+            tally = 0;
+            for (const orphan of orphans) {
+                tally++;
+                if ((tally % 100) === 0)
+                    commonsOutputProgress(`${tally}`);
+                yield this.databaseService.getLinks().deleteMany({ url: orphan });
+                yield this.databaseService.getLinks().deleteMany({ outgoing: orphan });
             }
-            batch = [];
-        }
-        nodecommons_cli_1.CommonsOutput.result(orphans.length);
-        nodecommons_cli_1.CommonsOutput.doing(`Removing orphan links`);
-        tally = 0;
-        for (const orphan of orphans) {
-            tally++;
-            if ((tally % 100) === 0)
-                nodecommons_cli_1.CommonsOutput.progress(`${tally}`);
-            await this.databaseService.getLinks().deleteMany({ url: orphan });
-            await this.databaseService.getLinks().deleteMany({ outgoing: orphan });
-        }
-        nodecommons_cli_1.CommonsOutput.success();
+            commonsOutputSuccess();
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        if (!this.includeReset)
-            await this.reset();
-        if (!this.includeLinks)
-            await this.links();
-        const cleaner = new cleaner_1.Cleaner(this.lists, this.databaseService);
-        await cleaner.purgeOrphanUrls();
-        await cleaner.purgeEmptyDomains();
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            if (!this.includeReset)
+                yield this.reset();
+            if (!this.includeLinks)
+                yield this.links();
+            const cleaner = new Cleaner(this.lists, this.databaseService);
+            yield cleaner.purgeOrphanUrls();
+            yield cleaner.purgeEmptyDomains();
+        });
     }
 }
-exports.CleanupApp = CleanupApp;
+//# sourceMappingURL=cleanup.app.js.map

package/dist/apps/cleanup.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAU7C,MAAM,OAAO,UAAW,SAAQ,UAAU;IAKzC,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,qDAAqD,CAAC,CAAC;YAE1E,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACxD,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACrB,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,MAAM;4BACd,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,UAAU;yBACnB,EAAE,EAAE,CAAC;qBACL,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC3C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;qBACnE;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YACjD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAAkB,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAElF,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE/E,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC/D,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;oBACtB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC1C;gBAED,KAAK,GAAG,EAAE,CAAC;aACX;YACD,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEpC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAC5C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;aACvE;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAE3C,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;YAChC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACnC,CAAC;KAAA;CACD"}

package/dist/apps/cross-populate-export.app.d.ts

@@ -0,0 +1,12 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class CrossPopulateExportApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private filename;
+    private databaseService;
+    constructor(filename: string);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/cross-populate-export.app.js

@@ -1,12 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CrossPopulateExportApp = void 0;
-const fs = require("fs");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as fs from 'fs';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
 // export the QUEUED, DONE and FAILED urls for importing into another instance
-class CrossPopulateExportApp extends nodecommons_app_1.CommonsApp {
+export class CrossPopulateExportApp extends CommonsApp {
     constructor(filename) {
         super('hydra-crawler');
         this.filename = filename;
@@ -17,46 +23,53 @@ class CrossPopulateExportApp extends nodecommons_app_1.CommonsApp {
     setDatabaseService(databaseService) {
         this.databaseService = databaseService;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const results = this.databaseService.getUrls().find({
-            status: { $in: [
-                    hydra_crawler_ts_assets_1.EStatus.QUEUED,
-                    hydra_crawler_ts_assets_1.EStatus.DONE,
-                    hydra_crawler_ts_assets_1.EStatus.FAILED
-                ] }
+    run() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const results = this.databaseService.getUrls().find({
+                status: { $in: [
+                        EStatus.QUEUED,
+                        EStatus.DONE,
+                        EStatus.FAILED
+                    ] }
+            }, {});
+            // safer to do this directly rather than a call to listQueryResults
+            // ditto not using CommonsFile
+            const fd = fs.openSync(this.filename, 'w');
+            commonsOutputDoing('Exporting QUEUED, DONE and FAILED urls');
+            let tally = 0;
+            while (true) {
+                if (tally % 1000 === 0)
+                    commonsOutputProgress(tally);
+                const row = yield results.next();
+                if (row === null)
+                    break;
+                fs.writeSync(fd, JSON.stringify({
+                    domain: row.domain,
+                    url: row.url
+                }) + '\n');
+                if (tally % 1000 === 0)
+                    fs.fdatasyncSync(fd);
+                tally++;
+            }
+            fs.fdatasyncSync(fd);
+            fs.closeSync(fd);
+            commonsOutputResult(tally);
         });
-        // safer to do this directly rather than a call to listQueryResults
-        // ditto not using CommonsFile
-        const fd = fs.openSync(this.filename, 'w');
-        nodecommons_cli_1.CommonsOutput.doing('Exporting QUEUED, DONE and FAILED urls');
-        let tally = 0;
-        while (true) {
-            if (tally % 1000 === 0)
-                nodecommons_cli_1.CommonsOutput.progress(tally);
-            const row = await results.next();
-            if (row === null)
-                break;
-            fs.writeSync(fd, JSON.stringify({
-                domain: row.domain,
-                url: row.url
-            }) + '\n');
-            if (tally % 1000 === 0)
-                fs.fdatasyncSync(fd);
-            tally++;
-        }
-        fs.fdatasyncSync(fd);
-        fs.closeSync(fd);
-        nodecommons_cli_1.CommonsOutput.result(tally);
     }
 }
-exports.CrossPopulateExportApp = CrossPopulateExportApp;
+//# sourceMappingURL=cross-populate-export.app.js.map

package/dist/apps/cross-populate-export.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cross-populate-export.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,OAAO,GAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;gBACtE,MAAM,EAAE,EAAE,GAAG,EAAE;wBACb,OAAO,CAAC,MAAM;wBACd,OAAO,CAAC,IAAI;wBACZ,OAAO,CAAC,MAAM;qBACf,EAAE;aACJ,EAAE,EAAE,CAAC,CAAC;YAEP,mEAAmE;YACnE,8BAA8B;YAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEnD,kBAAkB,CAAC,wCAAwC,CAAC,CAAC;YAE7D,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;oBACb,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;iBACb,CAAC,GAAG,IAAI,CACV,CAAC;gBAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC7C,KAAK,EAAE,CAAC;aACR;YAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAEjB,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/cross-populate-import.app.d.ts

@@ -0,0 +1,12 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IInternalHydraCommonDbApp } from './internal-hydra-common.app';
+export declare class CrossPopulateImportApp extends CommonsApp implements IInternalHydraCommonDbApp {
+    private filename;
+    private databaseService;
+    constructor(filename: string);
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    init(): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/apps/cross-populate-import.app.js

@@ -1,4 +1,12 @@
-"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 var __asyncValues = (this && this.__asyncValues) || function (o) {
     if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
     var m = o[Symbol.asyncIterator], i;
@@ -6,21 +14,19 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
     function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
     function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CrossPopulateImportApp = void 0;
-const fs = require("fs");
-const readline = require("readline");
-const tscommons_core_1 = require("tscommons-core");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_app_1 = require("nodecommons-app");
+import * as fs from 'fs';
+import * as readline from 'readline';
+import { commonsTypeHasPropertyString } from 'tscommons-es-core';
+import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { CommonsApp } from 'nodecommons-es-app';
 function isTRow(test) {
-    if (!tscommons_core_1.CommonsType.hasPropertyString(test, 'domain'))
+    if (!commonsTypeHasPropertyString(test, 'domain'))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyString(test, 'url'))
+    if (!commonsTypeHasPropertyString(test, 'url'))
         return false;
     return true;
 }
-class CrossPopulateImportApp extends nodecommons_app_1.CommonsApp {
+export class CrossPopulateImportApp extends CommonsApp {
     constructor(filename) {
         super('hydra-crawler');
         this.filename = filename;
@@ -31,50 +37,57 @@ class CrossPopulateImportApp extends nodecommons_app_1.CommonsApp {
     setDatabaseService(databaseService) {
         this.databaseService = databaseService;
     }
-    async init() {
-        if (!this.databaseService)
-            throw new Error('Database service has not been set yet');
-        nodecommons_cli_1.CommonsOutput.doing('Connecting to database');
-        await this.databaseService.init();
-        nodecommons_cli_1.CommonsOutput.success();
-        await super.init();
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set yet');
+            commonsOutputDoing('Connecting to database');
+            yield this.databaseService.init();
+            commonsOutputSuccess();
+            yield _super.init.call(this);
+        });
     }
-    async run() {
+    run() {
         var e_1, _a;
-        if (!this.databaseService)
-            throw new Error('Database service has not been set');
-        const fileStream = fs.createReadStream(this.filename);
-        const rl = readline.createInterface({
-            input: fileStream,
-            crlfDelay: Infinity
-        });
-        nodecommons_cli_1.CommonsOutput.doing('Importing urls');
-        let tally = 0;
-        try {
-            for (var rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = await rl_1.next(), !rl_1_1.done;) {
-                const line = rl_1_1.value;
-                if (line.trim() === '')
-                    continue;
-                if (tally % 1000 === 0)
-                    nodecommons_cli_1.CommonsOutput.progress(tally);
-                const json = JSON.parse(line);
-                if (!isTRow(json)) {
-                    nodecommons_cli_1.CommonsOutput.error('Invalid row. Skipping');
-                    continue;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.databaseService)
+                throw new Error('Database service has not been set');
+            const fileStream = fs.createReadStream(this.filename);
+            const rl = readline.createInterface({
+                input: fileStream,
+                crlfDelay: Infinity
+            });
+            commonsOutputDoing('Importing urls');
+            let tally = 0;
+            try {
+                for (var rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), !rl_1_1.done;) {
+                    const line = rl_1_1.value;
+                    if (line.trim() === '')
+                        continue;
+                    if (tally % 1000 === 0)
+                        commonsOutputProgress(tally);
+                    const json = JSON.parse(line);
+                    if (!isTRow(json)) {
+                        commonsOutputError('Invalid row. Skipping');
+                        continue;
+                    }
+                    yield this.databaseService.queue(json.url, false);
+                    tally++;
                 }
-                await this.databaseService.queue(json.url, false);
-                tally++;
             }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (rl_1_1 && !rl_1_1.done && (_a = rl_1.return)) await _a.call(rl_1);
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (rl_1_1 && !rl_1_1.done && (_a = rl_1.return)) yield _a.call(rl_1);
+                }
+                finally { if (e_1) throw e_1.error; }
             }
-            finally { if (e_1) throw e_1.error; }
-        }
-        nodecommons_cli_1.CommonsOutput.result(tally);
-        fileStream.close();
+            commonsOutputResult(tally);
+            fileStream.close();
+        });
     }
 }
-exports.CrossPopulateImportApp = CrossPopulateImportApp;
+//# sourceMappingURL=cross-populate-import.app.js.map

package/dist/apps/cross-populate-import.app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cross-populate-import.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAahD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;gBACtD,KAAK,EAAE,UAAU;gBACjB,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;YAErC,IAAI,KAAK,GAAW,CAAC,CAAC;;gBACtB,KAAyB,IAAA,OAAA,cAAA,EAAE,CAAA,QAAA;oBAAhB,MAAM,IAAI,eAAA,CAAA;oBACpB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;wBAAE,SAAS;oBAEjC,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;wBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;oBAErD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;wBAClB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;wBAC5C,SAAS;qBACT;oBAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBAClD,KAAK,EAAE,CAAC;iBACR;;;;;;;;;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,UAAU,CAAC,KAAK,EAAE,CAAC;;KACnB;CACD"}

package/dist/apps/denylist.app.d.ts

@@ -0,0 +1,17 @@
+import { CommonsApp } from 'nodecommons-es-app';
+import { DatabaseService } from '../services/database.service';
+import { IMatch } from '../interfaces/imatch';
+import { EList } from '../enums/elist';
+import { IInternalHydraCommonListApp } from './internal-hydra-common.app';
+export declare class DenylistApp extends CommonsApp implements IInternalHydraCommonListApp {
+    private databaseService;
+    private lists;
+    constructor();
+    getAppName(): string;
+    setDatabaseService(databaseService: DatabaseService): void;
+    addToList(list: EList, entries: IMatch[]): void;
+    init(): Promise<void>;
+    private forward;
+    private reverse;
+    run(): Promise<void>;
+}