hydra-crawler 2.8.3 → 3.0.1

package/dist/apps/import.app.js

@@ -1,57 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsInputReadAllStdIn, commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-export class ImportApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-    }
-    getAppName() {
-        return 'Hydra - Import';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputDoing('Loading URLs from stdin');
-            const urls = yield commonsInputReadAllStdIn();
-            commonsOutputResult(urls.length);
-            commonsOutputDoing('Seeding into queue');
-            let tally = 0;
-            let done = 0;
-            for (const url of urls) {
-                if ((++tally % 100) === 0)
-                    commonsOutputProgress(`${tally}, imported ${done}`);
-                const outcome = yield this.databaseService.queue(url);
-                if (outcome) {
-                    done++;
-                }
-            }
-            commonsOutputResult(done);
-        });
-    }
-}
-//# sourceMappingURL=import.app.js.map

package/dist/apps/import.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"import.app.js","sourceRoot":"","sources":["../../src/apps/import.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACpJ,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,MAAM,OAAO,SAAU,SAAQ,UAAU;IAGxC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,yBAAyB,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAa,MAAM,wBAAwB,EAAE,CAAC;YACxD,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,KAAK,GAAW,CAAC,CAAC;YAAC,IAAI,IAAI,GAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,IAAI,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,cAAc,IAAI,EAAE,CAAC,CAAC;gBAE/E,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE;oBACZ,IAAI,EAAE,CAAC;iBACP;aACD;YACD,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;KAAA;CACD"}

package/dist/apps/internal-hydra-common.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"internal-hydra-common.app.js","sourceRoot":"","sources":["../../src/apps/internal-hydra-common.app.ts"],"names":[],"mappings":"AAyBA,MAAM,UAAU,2BAA2B,CAAC,IAA6B;IACxE,OAAO,oBAAoB,IAAI,IAAI,CAAC;AACrC,CAAC;AAQD,MAAM,UAAU,6BAA6B,CAAC,IAA6B;IAC1E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,WAAW,IAAI,IAAI,CAAC;AAC5B,CAAC;AAKD,MAAM,UAAU,+BAA+B,CAAC,IAA6B;IAC5E,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,aAAa,IAAI,IAAI,CAAC;AAC9B,CAAC;AAKD,MAAM,UAAU,oCAAoC,CAAC,IAA6B;IACjF,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,OAAO,wBAAwB,IAAI,IAAI,CAAC;AACzC,CAAC"}

package/dist/apps/move-to-archive.app.js

@@ -1,46 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDoing, commonsOutputInfo, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-export class MoveToArchiveApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-    }
-    getAppName() {
-        return 'Hydra - Move to archive';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const batchSize = this.getArgs().getNumber('batch-size');
-            const limit = this.getArgs().getNumberOrUndefined('limit');
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputInfo('Moving ARCHIVED urls into archives collection.');
-            yield this.databaseService.moveArchivedUrlsToArchive(batchSize, limit);
-        });
-    }
-}
-//# sourceMappingURL=move-to-archive.app.js.map

package/dist/apps/move-to-archive.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"move-to-archive.app.js","sourceRoot":"","sources":["../../src/apps/move-to-archive.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACjG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAIhD,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAG/C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,yBAAyB,CAAC;IAClC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,MAAM,SAAS,GAAW,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjE,MAAM,KAAK,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAE7E,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,iBAAiB,CAAC,gDAAgD,CAAC,CAAC;YACpE,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;KAAA;CACD"}

package/dist/apps/prune-archive.app.js

@@ -1,55 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputAlert, commonsOutputDoing, commonsOutputInfo, commonsOutputProgress, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-export class PruneArchiveApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-    }
-    getAppName() {
-        return 'Hydra - Prune archive';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputInfo('Searching for duplicate entries in archiveds table.');
-            const duplicates = yield this.databaseService.listDuplicateArchivedUrls();
-            if (this.getArgs().hasAttribute('purge')) {
-                commonsOutputDoing('Purging duplicates');
-                for (const duplicate of duplicates) {
-                    commonsOutputProgress(duplicate.substring(0, 90));
-                    yield this.databaseService.purgeArchiveDuplicates(duplicate);
-                }
-                commonsOutputSuccess();
-            }
-            else {
-                commonsOutputAlert('Specify --purge flag to perform the purge');
-            }
-        });
-    }
-}
-//# sourceMappingURL=prune-archive.app.js.map

package/dist/apps/prune-archive.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"prune-archive.app.js","sourceRoot":"","sources":["../../src/apps/prune-archive.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC5I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAIhD,MAAM,OAAO,eAAgB,SAAQ,UAAU;IAG9C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,uBAAuB,CAAC;IAChC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,iBAAiB,CAAC,qDAAqD,CAAC,CAAC;YACzE,MAAM,UAAU,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,EAAE,CAAC;YAEpF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE;gBACzC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;gBAEzC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;oBACnC,qBAAqB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBAElD,MAAM,IAAI,CAAC,eAAe,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;iBAC7D;gBAED,oBAAoB,EAAE,CAAC;aACvB;iBAAM;gBACN,kBAAkB,CAAC,2CAA2C,CAAC,CAAC;aAChE;QACF,CAAC;KAAA;CACD"}

package/dist/apps/query.app.js

@@ -1,80 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeIsTKeyObject } from 'tscommons-es-core';
-import { commonsOutputDie, commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { Expiry } from '../classes/expiry';
-import { isIParserConfig } from '../interfaces/iparser-config';
-import { QUERY as QUERY_COMPLEX_ENGLISH } from '../queries/complex-english.query';
-import { QUERY as QUERY_FLASH_CONTENT } from '../queries/flash-content.query';
-import { QUERY as QUERY_LINKING_TO_DOMAINS } from '../queries/linking-to-domains.query';
-import { QUERY as QUERY_READABILITY_SCORE } from '../queries/readability-score.query';
-import { QUERY as QUERY_LLAMAGUARD_UNSAFE_CONTENT } from '../queries/llamaguard-unsafe-content.query';
-export class QueryApp extends CommonsApp {
-    constructor(queryName) {
-        super('hydra-crawler');
-        this.queryName = queryName;
-        if (/[^-a-z0-9]/i.test(queryName))
-            throw new Error('Invalid query name');
-        this.lists = new Lists();
-        this.expiry = new Expiry();
-        const parsersConfig = this.getConfigArea('parsers');
-        if (!commonsTypeIsTKeyObject(parsersConfig, isIParserConfig))
-            throw new Error('Parsers config is not valid');
-        this.parsersConfig = parsersConfig;
-    }
-    getAppName() {
-        return 'Hydra - Query';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    addToExpiry(expiries) {
-        this.expiry.add(expiries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            let query;
-            if (this.queryName === 'complex-english')
-                query = QUERY_COMPLEX_ENGLISH;
-            if (this.queryName === 'flash-content')
-                query = QUERY_FLASH_CONTENT;
-            if (this.queryName === 'linking-to-domains')
-                query = QUERY_LINKING_TO_DOMAINS;
-            if (this.queryName === 'readability-score')
-                query = QUERY_READABILITY_SCORE;
-            if (this.queryName === 'llamaguard-unsafe-content')
-                query = QUERY_LLAMAGUARD_UNSAFE_CONTENT;
-            if (!query)
-                commonsOutputDie('No such available query');
-            yield query(this.getArgs(), this.databaseService, this.lists, this.expiry, this.parsersConfig);
-        });
-    }
-}
-//# sourceMappingURL=query.app.js.map

package/dist/apps/query.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"query.app.js","sourceRoot":"","sources":["../../src/apps/query.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAExE,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAI3C,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAQ9E,OAAO,EAAE,KAAK,IAAI,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,KAAK,IAAI,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,KAAK,IAAI,wBAAwB,EAAE,MAAM,qCAAqC,CAAC;AACxF,OAAO,EAAE,KAAK,IAAI,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AACtF,OAAO,EAAE,KAAK,IAAI,+BAA+B,EAAE,MAAM,4CAA4C,CAAC;AAItG,MAAM,OAAO,QAAS,SAAQ,UAAU;IAQvC,YACU,SAAiB;QAE1B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAAQ;QAI1B,IAAI,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEzE,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACpC,CAAC;IAEM,UAAU;QAChB,OAAO,eAAe,CAAC;IACxB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAuB,CAAC;YAE5B,IAAI,IAAI,CAAC,SAAS,KAAK,iBAAiB;gBAAE,KAAK,GAAG,qBAAqB,CAAC;YACxE,IAAI,IAAI,CAAC,SAAS,KAAK,eAAe;gBAAE,KAAK,GAAG,mBAAmB,CAAC;YACpE,IAAI,IAAI,CAAC,SAAS,KAAK,oBAAoB;gBAAE,KAAK,GAAG,wBAAwB,CAAC;YAC9E,IAAI,IAAI,CAAC,SAAS,KAAK,mBAAmB;gBAAE,KAAK,GAAG,uBAAuB,CAAC;YAC5E,IAAI,IAAI,CAAC,SAAS,KAAK,2BAA2B;gBAAE,KAAK,GAAG,+BAA+B,CAAC;YAE5F,IAAI,CAAC,KAAK;gBAAE,gBAAgB,CAAC,yBAAyB,CAAC,CAAC;YAExD,MAAM,KAAK,CACT,IAAI,CAAC,OAAO,EAAE,EACd,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,aAAa,CACnB,CAAC;QACH,CAAC;KAAA;CACD"}

package/dist/apps/reattempt.app.js

@@ -1,83 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { EList } from '../enums/elist';
-export class ReattemptApp extends CommonsApp {
-    constructor(include) {
-        super('hydra-crawler');
-        this.include = include;
-        this.lists = new Lists();
-    }
-    getAppName() {
-        return 'Hydra - Reattempt';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            let tally = 0;
-            commonsOutputDoing(`Searching for allowlisted ${this.include.join(' and ')}`);
-            const results = this.databaseService.getUrls()
-                .find({
-                status: { $in: this.include }
-            }, {});
-            const urls = [];
-            tally = 0;
-            while (true) {
-                if ((tally % 100) === 0)
-                    commonsOutputProgress(tally);
-                const row = yield results.next();
-                if (row === null)
-                    break;
-                if (!this.lists.match(EList.ALLOW, row.url))
-                    continue;
-                urls.push(row.url);
-                tally++;
-            }
-            commonsOutputResult(tally);
-            commonsOutputDoing('Requeuing matches');
-            tally = 0;
-            for (const url of urls) {
-                tally++;
-                if ((tally % 10) === 0)
-                    commonsOutputProgress(tally);
-                yield this.databaseService.getUrls()
-                    .updateOne({ url: url }, {
-                    $set: { status: EStatus.QUEUED },
-                    $unset: { ttl: 1 }
-                });
-            }
-            commonsOutputResult(tally);
-        });
-    }
-}
-//# sourceMappingURL=reattempt.app.js.map

package/dist/apps/reattempt.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"reattempt.app.js","sourceRoot":"","sources":["../../src/apps/reattempt.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,YAAa,SAAQ,UAAU;IAK3C,YACU,OAAkB;QAE3B,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,YAAO,GAAP,OAAO,CAAW;QAI3B,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE9E,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACrE,IAAI,CACH;gBACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;aAC9B,EACD,EAAE,CACH,CAAC;YAEJ,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,KAAK,GAAG,CAAC,CAAC;YAEV,OAAO,IAAI,EAAE;gBACZ,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAEtD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEtD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEnB,KAAK,EAAE,CAAC;aACR;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;YACxC,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACjC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;oBACvB,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;iBACnB,CAAC,CAAC;aACL;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/requeue-domain.app.js

@@ -1,64 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-export class RequeueDomainApp extends CommonsApp {
-    constructor(domain, done200Only) {
-        super('hydra-crawler');
-        this.domain = domain;
-        this.done200Only = done200Only;
-    }
-    getAppName() {
-        return 'Hydra - Requeue domain';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const conditions = {
-                domain: this.domain,
-                status: { $in: [] }
-            };
-            if (this.done200Only) {
-                conditions.status.$in = [EStatus.DONE];
-                conditions.statusCode = 200;
-                commonsOutputDoing(`Marking all 200 DONE as QUEUED for ${this.domain}`);
-            }
-            else {
-                conditions.status.$in = [
-                    EStatus.DONE,
-                    EStatus.FAILED
-                ];
-                commonsOutputDoing(`Marking all DONE and FAILED as QUEUED for ${this.domain}`);
-            }
-            yield this.databaseService.getUrls()
-                .updateMany(conditions, { $set: { status: EStatus.QUEUED } });
-            commonsOutputSuccess();
-        });
-    }
-}
-//# sourceMappingURL=requeue-domain.app.js.map

package/dist/apps/requeue-domain.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"requeue-domain.app.js","sourceRoot":"","sources":["../../src/apps/requeue-domain.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IAG/C,YACU,MAAc,EACd,WAAoB;QAE7B,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,WAAM,GAAN,MAAM,CAAQ;QACd,gBAAW,GAAX,WAAW,CAAS;IAG9B,CAAC;IAEM,UAAU;QAChB,OAAO,wBAAwB,CAAC;IACjC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAMZ;gBACF,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,EAAE;gBACrB,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG,CAAE,OAAO,CAAC,IAAI,CAAE,CAAC;gBACzC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC;gBAE5B,kBAAkB,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aACxE;iBAAM;gBACN,UAAU,CAAC,MAAM,CAAC,GAAG,GAAG;oBACtB,OAAO,CAAC,IAAI;oBACZ,OAAO,CAAC,MAAM;iBACf,CAAC;gBAEF,kBAAkB,CAAC,6CAA6C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;aAC/E;YAED,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACjC,UAAU,CACT,UAAU,EACR,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CACvC,CAAC;YAEJ,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}

package/dist/apps/seed.app.js

@@ -1,69 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { EList } from '../enums/elist';
-export class SeedApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-        this.lists = new Lists();
-    }
-    getAppName() {
-        return 'Hydra - Seed';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputDoing('Building allowlist hostname seeds');
-            const hostnames = this.lists.listHostnames(EList.ALLOW);
-            const unique = [];
-            for (const hostname of hostnames) {
-                if (!unique.includes(hostname))
-                    unique.push(`http://${hostname}/`);
-                if (!unique.includes(`www.${hostname}`))
-                    unique.push(`http://www.${hostname}/`);
-                commonsOutputProgress(unique.length);
-            }
-            commonsOutputResult(unique.length);
-            commonsOutputDoing('Seeding into queue');
-            let tally = 0;
-            for (const url of unique) {
-                const outcome = yield this.databaseService.queue(url);
-                if (outcome) {
-                    tally++;
-                    commonsOutputProgress(tally);
-                }
-            }
-            commonsOutputResult(tally);
-        });
-    }
-}
-//# sourceMappingURL=seed.app.js.map

package/dist/apps/seed.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"seed.app.js","sourceRoot":"","sources":["../../src/apps/seed.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,OAAQ,SAAQ,UAAU;IAKtC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,cAAc,CAAC;IACvB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,mCAAmC,CAAC,CAAC;YACxD,MAAM,SAAS,GAAa,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAElE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;gBACjC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,UAAU,QAAQ,GAAG,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,QAAQ,EAAE,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,GAAG,CAAC,CAAC;gBAChF,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACrC;YAED,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;gBACzB,MAAM,OAAO,GAAY,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,OAAO,EAAE;oBACZ,KAAK,EAAE,CAAC;oBACR,qBAAqB,CAAC,KAAK,CAAC,CAAC;iBAC7B;aACD;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/startup.app.js

@@ -1,64 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDoing, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-// load the tables into Mongo's memory
-export class StartupApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-    }
-    getAppName() {
-        return 'Hydra - Startup';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const dbo = this.databaseService.getRawDatabase();
-            for (const collection of 'domains,urls,links'.split(',')) {
-                commonsOutputDoing(`Loading ${collection} collection into memory (simple enumeration)`);
-                const tally = yield dbo.collection(collection).find({}).count();
-                commonsOutputResult(tally);
-            }
-            commonsOutputDoing('Loading domains collection into memory (regex enumeration)');
-            yield this.databaseService.getDomains().find({
-                domain: /[0-9]temp[0-9]/
-            }).count();
-            commonsOutputSuccess();
-            commonsOutputDoing('Loading urls collection into memory (regex enumeration)');
-            yield this.databaseService.getUrls().find({
-                url: /[0-9]temp[0-9]/
-            }).count();
-            commonsOutputSuccess();
-            commonsOutputDoing('Loading links collection into memory (regex enumeration)');
-            yield this.databaseService.getLinks().find({
-                outgoing: /[0-9]temp[0-9]/
-            }).count();
-            commonsOutputSuccess();
-        });
-    }
-}
-//# sourceMappingURL=startup.app.js.map

package/dist/apps/startup.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"startup.app.js","sourceRoot":"","sources":["../../src/apps/startup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,sCAAsC;AAEtC,MAAM,OAAO,UAAW,SAAQ,UAAU;IAGzC;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;IACxB,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,GAAG,GAAe,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,CAAC;YAE9D,KAAK,MAAM,UAAU,IAAI,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACzD,kBAAkB,CAAC,WAAW,UAAU,8CAA8C,CAAC,CAAC;gBACxF,MAAM,KAAK,GAAW,MAAM,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACxE,mBAAmB,CAAC,KAAK,CAAC,CAAC;aAC3B;YAED,kBAAkB,CAAC,4DAA4D,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC;gBAC3C,MAAM,EAAE,gBAAgB;aACzB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,yDAAyD,CAAC,CAAC;YAC9E,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACxC,GAAG,EAAE,gBAAgB;aACtB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;YAEvB,kBAAkB,CAAC,0DAA0D,CAAC,CAAC;YAC/E,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC;gBACzC,QAAQ,EAAE,gBAAgB;aAC3B,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;CACD"}