hydra-crawler 2.8.3 → 3.0.1

package/dist/servers/maintenance.server.js

@@ -1,103 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsAsyncTimeout, CommonsSchedule } from 'tscommons-es-async';
-import { commonsOutputAlert, commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
-import { commonsGracefulAbortAddCallback } from 'nodecommons-es-process';
-import { Expirer } from '../classes/expirer';
-import { Cleaner } from '../classes/cleaner';
-var EAction;
-(function (EAction) {
-    EAction["EXPIRE"] = "expire";
-    EAction["PURGE_EMPTY_DOMAINS"] = "purge-empty-domains";
-    EAction["PURGE_ORPHAN_URLS"] = "purge-orphan-urls";
-})(EAction || (EAction = {}));
-export function toEAction(value) {
-    switch (value) {
-        case EAction.EXPIRE.toString():
-            return EAction.EXPIRE;
-        case EAction.PURGE_EMPTY_DOMAINS.toString():
-            return EAction.PURGE_EMPTY_DOMAINS;
-        case EAction.PURGE_ORPHAN_URLS.toString():
-            return EAction.PURGE_ORPHAN_URLS;
-    }
-    return undefined;
-}
-export class MaintenanceServer {
-    constructor(times, expiry, lists, database, crawl, hardLimit) {
-        this.crawl = crawl;
-        this.hardLimit = hardLimit;
-        this.isPaused = false;
-        this.isRunning = false;
-        this.schedule = new CommonsSchedule('hydra-maintenance');
-        this.schedule.parse(times, (action) => toEAction(action), (action) => __awaiter(this, void 0, void 0, function* () {
-            // prevent duplicate runs
-            if (this.isRunning) {
-                commonsOutputDebug('Maintenance is already running. Not re-running');
-                return;
-            }
-            this.isRunning = true;
-            try {
-                let claimedPause = false;
-                if (!this.isPaused) {
-                    this.isPaused = true;
-                    claimedPause = true;
-                    this.crawl.pause();
-                    for (let i = 30; i-- > 0;) {
-                        commonsOutputAlert(`Going down for maintenance ... ${i}`);
-                        try {
-                            yield commonsAsyncTimeout(1000);
-                        }
-                        catch (ex) {
-                            /* do nothing */
-                        }
-                    }
-                }
-                yield this.perform(action);
-                if (claimedPause) {
-                    commonsOutputAlert('Resuming from maintenance');
-                    this.isPaused = false;
-                    this.crawl.resume();
-                }
-            }
-            catch (e) {
-                console.log(e);
-                commonsOutputError(e.message);
-            }
-            finally {
-                this.isRunning = false;
-            }
-        }));
-        this.expirer = new Expirer(expiry, database);
-        this.cleaner = new Cleaner(lists, database);
-        commonsGracefulAbortAddCallback(() => {
-            commonsOutputAlert('SIGINT abort flag is set. Aborting maintenance server.');
-            this.schedule.stop();
-        });
-    }
-    perform(action) {
-        return __awaiter(this, void 0, void 0, function* () {
-            switch (action) {
-                case EAction.EXPIRE:
-                    yield this.expirer.expire();
-                    break;
-                case EAction.PURGE_EMPTY_DOMAINS:
-                    yield this.cleaner.purgeEmptyDomains();
-                    break;
-                case EAction.PURGE_ORPHAN_URLS:
-                    yield this.cleaner.purgeOrphanUrls(this.hardLimit);
-                    break;
-            }
-        });
-    }
-    start() {
-        this.schedule.start();
-    }
-}
-//# sourceMappingURL=maintenance.server.js.map

package/dist/servers/maintenance.server.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"maintenance.server.js","sourceRoot":"","sources":["../../src/servers/maintenance.server.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAIzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE;QACd,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;KAClC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAQ7B,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB,EAClB,SAAkB;QADlB,UAAK,GAAL,KAAK,CAAa;QAClB,cAAS,GAAT,SAAS,CAAS;QATpB,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAY,KAAK,CAAC;QAUlC,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,CAAO,MAAe,EAAiB,EAAE;YACxC,yBAAyB;YACzB,IAAI,IAAI,CAAC,SAAS,EAAE;gBACnB,kBAAkB,CAAC,gDAAgD,CAAC,CAAC;gBACrE,OAAO;aACP;YACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YAEtB,IAAI;gBACH,IAAI,YAAY,GAAY,KAAK,CAAC;gBAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;oBACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACrB,YAAY,GAAG,IAAI,CAAC;oBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBAEnB,KAAK,IAAI,CAAC,GAAW,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG;wBAClC,kBAAkB,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;wBAE1D,IAAI;4BACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;yBAChC;wBAAC,OAAO,EAAE,EAAE;4BACZ,gBAAgB;yBAChB;qBACD;iBACD;gBAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAE3B,IAAI,YAAY,EAAE;oBACjB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;oBAEhD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;oBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;iBACpB;aACD;YAAC,OAAO,CAAC,EAAE;gBACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,kBAAkB,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;aACzC;oBAAS;gBACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;aACvB;QACF,CAAC,CAAA,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,wDAAwD,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEa,OAAO,CAAC,MAAe;;YACpC,QAAQ,MAAM,EAAE;gBACf,KAAK,OAAO,CAAC,MAAM;oBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAC5B,MAAM;gBACP,KAAK,OAAO,CAAC,mBAAmB;oBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;oBACvC,MAAM;gBACP,KAAK,OAAO,CAAC,iBAAiB;oBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACnD,MAAM;aACP;QACF,CAAC;KAAA;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}

package/dist/servers/rest.server.js

@@ -1,46 +0,0 @@
-import { isICommonsExpressConfig } from 'nodecommons-es-express';
-import { CommonsRestServer } from 'nodecommons-es-rest';
-import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { commonsNginxProxyPath } from 'nodecommons-es-http';
-import { TestApi } from '../apis/test.api';
-import { StatisticsApi } from '../apis/statistics.api';
-import { CrawlApi } from '../apis/crawl.api';
-import { BugsApi } from '../apis/bugs.api';
-import { DomainsApi } from '../apis/domains.api';
-import { UrlsApi } from '../apis/urls.api';
-import { ImagesApi } from '../apis/images.api';
-import { AutocompleteApi } from '../apis/autocomplete.api';
-export class RestServer extends CommonsRestServer {
-    constructor(expressServer, database, config) {
-        super(expressServer);
-        const expressConfig = config.getObject('express');
-        if (!isICommonsExpressConfig(expressConfig))
-            throw new Error('Express config is not valid');
-        const path = commonsNginxProxyPath(expressConfig.path);
-        commonsOutputDoing('Installing REST Test API');
-        new TestApi(this, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Statistics API');
-        new StatisticsApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Crawl API');
-        new CrawlApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Bugs API');
-        new BugsApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Domains API');
-        new DomainsApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST URLs API');
-        new UrlsApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Images API');
-        new ImagesApi(this, database, path);
-        commonsOutputSuccess();
-        commonsOutputDoing('Installing REST Autocomplete API');
-        new AutocompleteApi(this, database, path);
-        commonsOutputSuccess();
-    }
-}
-//# sourceMappingURL=rest.server.js.map

package/dist/servers/rest.server.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAI3D,MAAM,OAAO,UAAW,SAAQ,iBAAoE;IACnG,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,qBAAqB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAE/D,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACxC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;QACvD,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC1C,oBAAoB,EAAE,CAAC;IACxB,CAAC;CAED"}

package/dist/servers/socket-io.server.js

@@ -1,61 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { CommonsAppSocketIoServer } from 'nodecommons-es-app-socket-io';
-export class SocketIoServer extends CommonsAppSocketIoServer {
-    fetching(fetch) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'fetching', fetch);
-        });
-    }
-    outcome(outcome) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'outcome', outcome);
-        });
-    }
-    statusTallies(tallies) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'statusTallies', tallies);
-        });
-    }
-    linkTallies(tally) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'linkTallies', tally);
-        });
-    }
-    domainTallies(tally) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'domainTallies', tally);
-        });
-    }
-    bandwidth(bps) {
-        const _super = Object.create(null, {
-            broadcast: { get: () => super.broadcast }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.broadcast.call(this, 'bandwidth', bps);
-        });
-    }
-}
-//# sourceMappingURL=socket-io.server.js.map

package/dist/servers/socket-io.server.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAExE,MAAM,OAAO,cAAe,SAAQ,wBAAwB;IAC9C,QAAQ,CAAC,KAAa;;;;;YAClC,MAAM,OAAM,SAAS,YAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;KAAA;IAEY,OAAO,CAAC,OAAiB;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,aAAa,CAAC,OAAuB;;;;;YACjD,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;KAAA;IAEY,WAAW,CAAC,KAAa;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEY,aAAa,CAAC,KAAa;;;;;YACvC,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;KAAA;IAEY,SAAS,CAAC,GAAW;;;;;YACjC,MAAM,OAAM,SAAS,YAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;KAAA;CACD"}