hydra-crawler 2.8.4 → 3.0.2

package/dist/apis/bugs.api.js

@@ -1,26 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeEncode } from 'tscommons-es-core';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-export class BugsApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        super.getHandler(`${path}rest/bugs/php`, (_req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const result = yield this.databaseService.listPhpErrors();
-            return commonsTypeEncode(result);
-        }));
-        super.getHandler(`${path}rest/bugs/asp`, (_req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const result = yield this.databaseService.listAspErrors();
-            return commonsTypeEncode(result);
-        }));
-    }
-}
-//# sourceMappingURL=bugs.api.js.map

package/dist/apis/bugs.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"bugs.api.js","sourceRoot":"","sources":["../../src/apis/bugs.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAMhE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAA,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,eAAe,EACtB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,MAAM,GAA4B,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;YAEnF,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/crawl.api.js

@@ -1,25 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { CommonsRestApi } from 'nodecommons-es-rest';
-export class CrawlApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        super.getHandler(`${path}rest/crawl/queue`, (_req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const tallies = yield this.databaseService.listDomainQueuedTallies();
-            return Array.from(tallies.keys())
-                .map((domain) => ({
-                domain: domain,
-                queue: tallies.get(domain) || 0
-            }));
-        }));
-    }
-}
-//# sourceMappingURL=crawl.api.js.map

package/dist/apis/crawl.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawl.api.js","sourceRoot":"","sources":["../../src/apis/crawl.api.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,QAAS,SAAQ,cAAiE;IAC9F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,kBAAkB,EACzB,CAAO,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,OAAO,GAAwB,MAAM,IAAI,CAAC,eAAe,CAAC,uBAAuB,EAAE,CAAC;YAE1F,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAC9B,GAAG,CAAC,CAAC,MAAc,EAAgB,EAAE,CAAC,CAAC;gBACtC,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;aAChC,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CACF,CAAC;IAEH,CAAC;CACD"}

package/dist/apis/domains.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"domains.api.js","sourceRoot":"","sources":["../../src/apis/domains.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,mCAAmC,EAAE,iBAAiB,EAA0B,MAAM,mBAAmB,CAAC;AAKnH,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAMpG,MAAM,OAAO,UAAW,SAAQ,cAAiE;IAChG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,cAAc,EACrB,CAAO,IAAqB,EAAE,IAAsB,EAAqB,EAAE;YAC1E,MAAM,OAAO,GAAc,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEpE,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAA,CACF,CAAC;QAEF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,mCAAmC,EAC1C,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAgB,CAAC;gBAAE,OAAO,wBAAwB,CAAC,gBAAgB,CAAC,CAAC;YAE/H,MAAM,IAAI,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,qBAAqB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAE3F,MAAM,MAAM,GAAe,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,iBAAiB,GAAG,CAAC,MAAM,CAAC,MAAM,oBAAoB,CAAC,CAAC;gBACzF,MAAM,KAAK,GAAyB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEpD,IAAI,KAAK,KAAK,IAAI;oBAAE,OAAO,mBAAmB,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;gBAE7E,IAAI,OAAO,GAAW,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAEjE,MAAM,QAAQ,GAAa,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9C,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,kBAAkB;gBAElC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;aACtB;YAED,MAAM,IAAI,GAAiB,mCAAmC,CAAC,MAAM,CAAC,CAAC;YACvE,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/images.api.js

@@ -1,27 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeEncode } from 'tscommons-es-core';
-import { toEComparator, ECOMPARATORS } from 'hydra-crawler-ts-assets';
-import { CommonsRestApi, commonsRestApiBadRequest } from 'nodecommons-es-rest';
-export class ImagesApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        super.getHandler(`${path}rest/images/:comparator(${ECOMPARATORS.join('|')})/:size[int]`, (req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const comparator = toEComparator(req.strictParams.comparator);
-            if (!comparator)
-                return commonsRestApiBadRequest('Invalid comparator');
-            const size = req.strictParams.size;
-            const matches = yield this.databaseService.listImagesBySizeThreshold(size, comparator);
-            return commonsTypeEncode(matches);
-        }));
-    }
-}
-//# sourceMappingURL=images.api.js.map

package/dist/apis/images.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"images.api.js","sourceRoot":"","sources":["../../src/apis/images.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAe,aAAa,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGnF,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAM/E,MAAM,OAAO,SAAU,SAAQ,cAAiE;IAC/F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,2BAA2B,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EACtE,CAAO,GAAoC,EAAE,IAAsB,EAAqB,EAAE;YACzF,MAAM,UAAU,GAA0B,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,UAAoB,CAAC,CAAC;YAC/F,IAAI,CAAC,UAAU;gBAAE,OAAO,wBAAwB,CAAC,oBAAoB,CAAC,CAAC;YAEvE,MAAM,IAAI,GAAW,GAAG,CAAC,YAAY,CAAC,IAAc,CAAC;YAErD,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,yBAAyB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEjG,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/statistics.api.js

@@ -1,41 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import Cache from 'memcached-promisify';
-import { commonsTypeHasPropertyEnum, commonsTypeHasPropertyNumber, commonsTypeIsTArray } from 'tscommons-es-core';
-import { isEStatus } from 'hydra-crawler-ts-assets';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-function isTStatusTally(test) {
-    if (!commonsTypeHasPropertyEnum(test, 'status', isEStatus))
-        return false;
-    if (!commonsTypeHasPropertyNumber(test, 'tally'))
-        return false;
-    return true;
-}
-export class StatisticsApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
-        this.memcached = new Cache({ keyPrefix: 'hydra', cacheHost: 'localhost:11211' });
-        super.getHandler(`${path}rest/statistics/status`, (_req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const index = 'statistics/status';
-            const cached = yield this.memcached.get(index);
-            if (cached && commonsTypeIsTArray(cached, isTStatusTally))
-                return cached;
-            const result = yield this.databaseService.listStatusTallies();
-            const tallies = [];
-            for (const key of result.keys())
-                tallies.push({ status: key, tally: result.get(key) || 0 });
-            yield this.memcached.set(index, tallies, 10);
-            return tallies;
-        }));
-    }
-}
-//# sourceMappingURL=statistics.api.js.map

package/dist/apis/statistics.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"statistics.api.js","sourceRoot":"","sources":["../../src/apis/statistics.api.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,KAAK,MAAM,qBAAqB,CAAC;AAExC,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAElH,OAAO,EAAW,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAarD,SAAS,cAAc,CAAC,IAAa;IACpC,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC;QAAE,OAAO,KAAK,CAAC;IACzE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAE/D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,cAAiE;IAGnG,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,sGAAsG;QACtG,IAAI,CAAC,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAEjF,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,wBAAwB,EAC/B,CAAO,IAAqB,EAAE,IAAsB,EAA2B,EAAE;YAChF,MAAM,KAAK,GAAW,mBAAmB,CAAC;YAE1C,MAAM,MAAM,GAAsB,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClE,IAAI,MAAM,IAAI,mBAAmB,CAAe,MAAM,EAAE,cAAc,CAAC;gBAAE,OAAO,MAAM,CAAC;YAEvF,MAAM,MAAM,GAAyB,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC;YAEpF,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAE5F,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YAE7C,OAAO,OAAO,CAAC;QAChB,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apis/test.api.js

@@ -1,19 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { CommonsRestApi } from 'nodecommons-es-rest';
-export class TestApi extends CommonsRestApi {
-    constructor(restServer, path) {
-        super(restServer);
-        super.getHandler(`${path}test/hello`, 
-        // eslint-disable-next-line @typescript-eslint/require-await
-        (_) => __awaiter(this, void 0, void 0, function* () { return 'Hello world'; }));
-    }
-}
-//# sourceMappingURL=test.api.js.map

package/dist/apis/test.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"test.api.js","sourceRoot":"","sources":["../../src/apis/test.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAKrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACtB,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAElB,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,YAAY;QACnB,4DAA4D;QAC5D,CAAO,CAAU,EAAmB,EAAE,gDAAC,OAAA,aAAa,CAAA,GAAA,CACrD,CAAC;IACH,CAAC;CACD"}

package/dist/apis/urls.api.js

@@ -1,29 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeEncode } from 'tscommons-es-core';
-import { EDirection } from 'hydra-crawler-ts-assets';
-import { CommonsRestApi } from 'nodecommons-es-rest';
-export class UrlsApi extends CommonsRestApi {
-    constructor(restServer, databaseService, path) {
-        super(restServer);
-        this.databaseService = databaseService;
-        // Can't use strict and ICommonsRequestWithStrictParams as the url could be anything, and idname won't accept that
-        super.getHandler(`${path}rest/urls/:url/links`, (req, _res) => __awaiter(this, void 0, void 0, function* () {
-            const outbound = yield this.databaseService.listOutboundLinks(req.params.url);
-            const inbound = yield this.databaseService.listInboundLinks(req.params.url);
-            const urlLinks = {
-                [EDirection.INBOUND]: inbound,
-                [EDirection.OUTBOUND]: outbound
-            };
-            return commonsTypeEncode(urlLinks);
-        }));
-    }
-}
-//# sourceMappingURL=urls.api.js.map

package/dist/apis/urls.api.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"urls.api.js","sourceRoot":"","sources":["../../src/apis/urls.api.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAY,MAAM,mBAAmB,CAAC;AAGhE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOrD,MAAM,OAAO,OAAQ,SAAQ,cAAiE;IAC7F,YACE,UAAsB,EACd,eAAgC,EACxC,IAAY;QAEb,KAAK,CAAC,UAAU,CAAC,CAAC;QAHT,oBAAe,GAAf,eAAe,CAAiB;QAKzC,kHAAkH;QAElH,KAAK,CAAC,UAAU,CACd,GAAG,IAAI,sBAAsB,EAC7B,CAAO,GAAoB,EAAE,IAAsB,EAAqB,EAAE;YACzE,MAAM,QAAQ,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,OAAO,GAAa,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAEtF,MAAM,QAAQ,GAAc;gBAC1B,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO;gBAC7B,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,QAAQ;aAChC,CAAC;YAEF,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAA,CACF,CAAC;IACH,CAAC;CACD"}

package/dist/apps/cleanup.app.js

@@ -1,151 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsTypeHasPropertyNumber } from 'tscommons-es-core';
-import { isIUrl } from 'hydra-crawler-ts-assets';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputPercent, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { Cleaner } from '../classes/cleaner';
-export class CleanupApp extends CommonsApp {
-    constructor(includeReset = false, includeLinks = false) {
-        super('hydra-crawler');
-        this.includeReset = includeReset;
-        this.includeLinks = includeLinks;
-        this.lists = new Lists();
-    }
-    getAppName() {
-        return 'Hydra - Cleanup';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    reset() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputDoing('Enumerating urls for DENY, FAILED, DEAD and DISALLOWED');
-            const urls = [];
-            let tally = 0;
-            while (true) {
-                const result = this.databaseService.getUrls()
-                    .find({ status: { $in: [
-                            EStatus.DENY,
-                            EStatus.FAILED,
-                            EStatus.DEAD,
-                            EStatus.DISALLOWED
-                        ] } })
-                    .sort({ _id: 1 })
-                    .skip(tally);
-                try {
-                    while (true) {
-                        tally++;
-                        if ((tally % 100) === 0)
-                            commonsOutputProgress(tally);
-                        const row = yield result.next();
-                        if (row === null)
-                            break;
-                        urls.push(row.url);
-                    }
-                    break;
-                }
-                catch (err) {
-                    if (!commonsTypeHasPropertyNumber(err, 'code') || err.code !== 43)
-                        throw err;
-                }
-            }
-            commonsOutputResult(tally);
-            commonsOutputDoing('Resetting links for DENY, FAILED, DEAD and DISALLOWED');
-            let i = 0;
-            for (const url of urls) {
-                i++;
-                if ((i % 100) === 0)
-                    commonsOutputPercent(i, urls.length);
-                yield this.databaseService.getLinks().deleteMany({ url: url });
-            }
-            commonsOutputSuccess();
-        });
-    }
-    links() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            commonsOutputDoing('Searching for link orphans');
-            let tally = 0;
-            const result = this.databaseService.getLinks().find({}, {});
-            let batch = [];
-            const orphans = [];
-            while (true) {
-                tally++;
-                if ((tally % 1000) === 0)
-                    commonsOutputProgress(`${tally}, ${orphans.length}`);
-                const link = yield result.next();
-                if (link === null)
-                    break;
-                if (!batch.includes(link.url))
-                    batch.push(link.url);
-                if (!batch.includes(link.outgoing))
-                    batch.push(link.outgoing);
-                if (batch.length < 10000)
-                    continue;
-                const result2 = this.databaseService.getUrls().find({ url: { $in: batch } });
-                const matches = (yield this.databaseService.listQueryResults(result2, isIUrl))
-                    .map((u) => u.url);
-                for (const l of batch) {
-                    if (!matches.includes(l))
-                        orphans.push(l);
-                }
-                batch = [];
-            }
-            commonsOutputResult(orphans.length);
-            commonsOutputDoing('Removing orphan links');
-            tally = 0;
-            for (const orphan of orphans) {
-                tally++;
-                if ((tally % 100) === 0)
-                    commonsOutputProgress(`${tally}`);
-                yield this.databaseService.getLinks().deleteMany({ url: orphan });
-                yield this.databaseService.getLinks().deleteMany({ outgoing: orphan });
-            }
-            commonsOutputSuccess();
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            if (this.includeReset)
-                yield this.reset();
-            if (this.includeLinks)
-                yield this.links();
-            const hardLimit = this.getArgs().getNumberOrUndefined('hard-limit');
-            const cleaner = new Cleaner(this.lists, this.databaseService);
-            yield cleaner.purgeOrphanUrls(hardLimit);
-            yield cleaner.purgeEmptyDomains();
-        });
-    }
-}
-//# sourceMappingURL=cleanup.app.js.map

package/dist/apps/cleanup.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cleanup.app.js","sourceRoot":"","sources":["../../src/apps/cleanup.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAQ,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAChJ,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAU7C,MAAM,OAAO,UAAW,SAAQ,UAAU;IAKzC,YACU,eAAwB,KAAK,EAC7B,eAAwB,KAAK;QAEtC,KAAK,CAAC,eAAe,CAAC,CAAC;QAHd,iBAAY,GAAZ,YAAY,CAAiB;QAC7B,iBAAY,GAAZ,YAAY,CAAiB;QAItC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,iBAAiB,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,wDAAwD,CAAC,CAAC;YAE7E,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;qBACpE,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE;4BACrB,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,MAAM;4BACd,OAAO,CAAC,IAAI;4BACZ,OAAO,CAAC,UAAU;yBACnB,EAAE,EAAE,CAAC;qBACL,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;qBAChB,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEf,IAAI;oBACH,OAAO,IAAI,EAAE;wBACZ,KAAK,EAAE,CAAC;wBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAEtD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC3C,IAAI,GAAG,KAAK,IAAI;4BAAE,MAAM;wBAExB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;qBACnB;oBAED,MAAM;iBACN;gBAAC,OAAO,GAAG,EAAE;oBACb,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE;wBAAE,MAAM,GAAG,CAAC;iBAC7E;aACD;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,uDAAuD,CAAC,CAAC;YAC5E,IAAI,CAAC,GAAW,CAAC,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,CAAC,EAAE,CAAC;gBACJ,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,oBAAoB,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE1D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;aAC/D;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEa,KAAK;;YAClB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YACjD,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,MAAM,MAAM,GAA8B,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9F,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAE/E,MAAM,IAAI,GAAe,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC7C,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM;gBAEzB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;oBAAE,SAAS;gBAEnC,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAC3E,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CACxB,CAAC;gBAEF,MAAM,OAAO,GAAa,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;qBACrF,GAAG,CAAC,CAAC,CAAO,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAEnC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;oBACtB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC1C;gBAED,KAAK,GAAG,EAAE,CAAC;aACX;YACD,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEpC,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;YAC5C,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;gBAClE,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;aACvE;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAE1C,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAEtF,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvE,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC;QACnC,CAAC;KAAA;CACD"}

package/dist/apps/cross-populate-export.app.js

@@ -1,75 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import * as fs from 'fs';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-// export the QUEUED, DONE and FAILED urls for importing into another instance
-export class CrossPopulateExportApp extends CommonsApp {
-    constructor(filename) {
-        super('hydra-crawler');
-        this.filename = filename;
-    }
-    getAppName() {
-        return 'Hydra - Cross Populate Export';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const results = this.databaseService.getUrls().find({
-                status: { $in: [
-                        EStatus.QUEUED,
-                        EStatus.DONE,
-                        EStatus.FAILED
-                    ] }
-            }, {});
-            // safer to do this directly rather than a call to listQueryResults
-            // ditto not using CommonsFile
-            const fd = fs.openSync(this.filename, 'w');
-            commonsOutputDoing('Exporting QUEUED, DONE and FAILED urls');
-            let tally = 0;
-            while (true) {
-                if (tally % 1000 === 0)
-                    commonsOutputProgress(tally);
-                const row = yield results.next();
-                if (row === null)
-                    break;
-                fs.writeSync(fd, JSON.stringify({
-                    domain: row.domain,
-                    url: row.url
-                }) + '\n');
-                if (tally % 1000 === 0)
-                    fs.fdatasyncSync(fd);
-                tally++;
-            }
-            fs.fdatasyncSync(fd);
-            fs.closeSync(fd);
-            commonsOutputResult(tally);
-        });
-    }
-}
-//# sourceMappingURL=cross-populate-export.app.js.map

package/dist/apps/cross-populate-export.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cross-populate-export.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-export.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,8EAA8E;AAE9E,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,OAAO,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,IAAI,CAAO;gBAClF,MAAM,EAAE,EAAE,GAAG,EAAE;wBACb,OAAO,CAAC,MAAM;wBACd,OAAO,CAAC,IAAI;wBACZ,OAAO,CAAC,MAAM;qBACf,EAAE;aACJ,EAAE,EAAE,CAAC,CAAC;YAEP,mEAAmE;YACnE,8BAA8B;YAE9B,MAAM,EAAE,GAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEnD,kBAAkB,CAAC,wCAAwC,CAAC,CAAC;YAE7D,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE;gBACZ,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;gBAErD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,EAAE,CAAC,SAAS,CACV,EAAE,EACF,IAAI,CAAC,SAAS,CAAC;oBACb,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;iBACb,CAAC,GAAG,IAAI,CACV,CAAC;gBAEF,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;oBAAE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC7C,KAAK,EAAE,CAAC;aACR;YAED,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACrB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAEjB,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;KAAA;CACD"}

package/dist/apps/cross-populate-import.app.js

@@ -1,100 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-import * as fs from 'fs';
-import * as readline from 'readline';
-import { commonsTypeHasPropertyString } from 'tscommons-es-core';
-import { commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-function isTRow(test) {
-    if (!commonsTypeHasPropertyString(test, 'domain'))
-        return false;
-    if (!commonsTypeHasPropertyString(test, 'url'))
-        return false;
-    return true;
-}
-export class CrossPopulateImportApp extends CommonsApp {
-    constructor(filename) {
-        super('hydra-crawler');
-        this.filename = filename;
-    }
-    getAppName() {
-        return 'Hydra - Cross Populate Export';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        var _a, e_1, _b, _c;
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const fileStream = fs.createReadStream(this.filename);
-            const rl = readline.createInterface({
-                input: fileStream,
-                crlfDelay: Infinity
-            });
-            commonsOutputDoing('Importing urls');
-            let tally = 0;
-            try {
-                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a;) {
-                    _c = rl_1_1.value;
-                    _d = false;
-                    try {
-                        const line = _c;
-                        if (line.trim() === '')
-                            continue;
-                        if (tally % 1000 === 0)
-                            commonsOutputProgress(tally);
-                        const json = JSON.parse(line);
-                        if (!isTRow(json)) {
-                            commonsOutputError('Invalid row. Skipping');
-                            continue;
-                        }
-                        yield this.databaseService.queue(json.url, false);
-                        tally++;
-                    }
-                    finally {
-                        _d = true;
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            commonsOutputResult(tally);
-            fileStream.close();
-        });
-    }
-}
-//# sourceMappingURL=cross-populate-import.app.js.map

package/dist/apps/cross-populate-import.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cross-populate-import.app.js","sourceRoot":"","sources":["../../src/apps/cross-populate-import.app.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAahD,SAAS,MAAM,CAAC,IAAa;IAC5B,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,IAAI,CAAC,4BAA4B,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAE7D,OAAO,IAAI,CAAC;AACb,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,UAAU;IAGrD,YACU,QAAgB;QAEzB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,aAAQ,GAAR,QAAQ,CAAQ;IAG1B,CAAC;IAEM,UAAU;QAChB,OAAO,+BAA+B,CAAC;IACxC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,UAAU,GAAkB,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAErE,MAAM,EAAE,GAAuB,QAAQ,CAAC,eAAe,CAAC;gBACtD,KAAK,EAAE,UAAU;gBACjB,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;YAErC,IAAI,KAAK,GAAW,CAAC,CAAC;;gBACtB,KAAyB,eAAA,OAAA,cAAA,EAAE,CAAA,QAAA;oBAAF,kBAAE;oBAAF,WAAE;;wBAAhB,MAAM,IAAI,KAAA,CAAA;wBACpB,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;4BAAE,SAAS;wBAEjC,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;4BAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC;wBAErD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;4BAClB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;4BAC5C,SAAS;yBACT;wBAED,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBAClD,KAAK,EAAE,CAAC;;;;;iBACR;;;;;;;;;YACD,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,UAAU,CAAC,KAAK,EAAE,CAAC;;KACnB;CACD"}