hydra-crawler 1.4.6 → 2.0.2

package/dist/servers/maintenance.server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"maintenance.server.js","sourceRoot":"","sources":["../../src/servers/maintenance.server.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAwB,MAAM,oBAAoB,CAAC;AAEhG,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAIzE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAM7C,IAAK,OAIJ;AAJD,WAAK,OAAO;IACX,4BAAiB,CAAA;IACjB,sDAA2C,CAAA;IAC3C,kDAAuC,CAAA;AACxC,CAAC,EAJI,OAAO,KAAP,OAAO,QAIX;AAED,MAAM,UAAU,SAAS,CAAC,KAAa;IACtC,QAAQ,KAAK,EAAE;QACd,KAAK,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAC7B,OAAO,OAAO,CAAC,MAAM,CAAC;QACvB,KAAK,OAAO,CAAC,mBAAmB,CAAC,QAAQ,EAAE;YAC1C,OAAO,OAAO,CAAC,mBAAmB,CAAC;QACpC,KAAK,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE;YACxC,OAAO,OAAO,CAAC,iBAAiB,CAAC;KAClC;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAO7B,YACE,KAA6B,EAC7B,MAAc,EACd,KAAY,EACZ,QAAyB,EACjB,KAAkB;QAAlB,UAAK,GAAL,KAAK,CAAa;QAPpB,aAAQ,GAAY,KAAK,CAAC;QASjC,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAEzD,IAAI,CAAC,QAAQ,CAAC,KAAK,CACjB,KAAK,EACL,CAAC,MAAc,EAAqB,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EACxD,CAAO,MAAe,EAAiB,EAAE;YACxC,IAAI,YAAY,GAAY,KAAK,CAAC;YAElC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,YAAY,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAEnB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,GAAG;oBACjC,kBAAkB,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;oBAE1D,IAAI;wBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;qBAChC;oBAAC,OAAO,EAAE,EAAE;wBACZ,gBAAgB;qBAChB;iBACD;aACD;YAED,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAE3B,IAAI,YAAY,EAAE;gBACjB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAEhD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;gBAEtB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;aACpB;QACF,CAAC,CAAA,CACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE5C,+BAA+B,CAAC,GAAS,EAAE;YAC1C,kBAAkB,CAAC,wDAAwD,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEa,OAAO,CAAC,MAAe;;YACpC,QAAQ,MAAM,EAAE;gBACf,KAAK,OAAO,CAAC,MAAM;oBAClB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBAC5B,MAAM;gBACP,KAAK,OAAO,CAAC,mBAAmB;oBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC;oBACvC,MAAM;gBACP,KAAK,OAAO,CAAC,iBAAiB;oBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;oBACrC,MAAM;aACP;QACF,CAAC;KAAA;IAEM,KAAK;QACX,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACD"}

package/dist/servers/rest.server.d.ts

@@ -0,0 +1,7 @@
+import { CommonsConfig } from 'tscommons-es-config';
+import { CommonsStrictExpressServer } from 'nodecommons-es-express';
+import { CommonsRestServer } from 'nodecommons-es-rest';
+import { DatabaseService } from '../services/database.service';
+export declare class RestServer extends CommonsRestServer {
+    constructor(expressServer: CommonsStrictExpressServer, database: DatabaseService, config: CommonsConfig);
+}

package/dist/servers/rest.server.js

@@ -1,57 +1,46 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RestServer = void 0;
-const nodecommons_express_1 = require("nodecommons-express");
-const nodecommons_express_2 = require("nodecommons-express");
-const nodecommons_rest_1 = require("nodecommons-rest");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const test_api_1 = require("../apis/test.api");
-const statistics_api_1 = require("../apis/statistics.api");
-const crawl_api_1 = require("../apis/crawl.api");
-const bugs_api_1 = require("../apis/bugs.api");
-const domains_api_1 = require("../apis/domains.api");
-const urls_api_1 = require("../apis/urls.api");
-const images_api_1 = require("../apis/images.api");
-const autocomplete_api_1 = require("../apis/autocomplete.api");
-class RestServer extends nodecommons_rest_1.CommonsRestServer {
+import { commonsExpressNginxProxyPath } from 'nodecommons-es-express';
+import { isICommonsExpressConfig } from 'nodecommons-es-express';
+import { CommonsRestServer } from 'nodecommons-es-rest';
+import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
+import { TestApi } from '../apis/test.api';
+import { StatisticsApi } from '../apis/statistics.api';
+import { CrawlApi } from '../apis/crawl.api';
+import { BugsApi } from '../apis/bugs.api';
+import { DomainsApi } from '../apis/domains.api';
+import { UrlsApi } from '../apis/urls.api';
+import { ImagesApi } from '../apis/images.api';
+import { AutocompleteApi } from '../apis/autocomplete.api';
+export class RestServer extends CommonsRestServer {
     constructor(expressServer, database, config) {
         super(expressServer);
         const expressConfig = config.getObject('express');
-        if (!nodecommons_express_2.isICommonsExpressConfig(expressConfig))
+        if (!isICommonsExpressConfig(expressConfig))
             throw new Error('Express config is not valid');
-        const path = nodecommons_express_1.CommonsExpressServer.nginxProxyPath(expressConfig.path);
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Test API');
-        // tslint:disable-next-line:no-unused-expression
-        new test_api_1.TestApi(this, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Statistics API');
-        // tslint:disable-next-line:no-unused-expression
-        new statistics_api_1.StatisticsApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Crawl API');
-        // tslint:disable-next-line:no-unused-expression
-        new crawl_api_1.CrawlApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Bugs API');
-        // tslint:disable-next-line:no-unused-expression
-        new bugs_api_1.BugsApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Domains API');
-        // tslint:disable-next-line:no-unused-expression
-        new domains_api_1.DomainsApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST URLs API');
-        // tslint:disable-next-line:no-unused-expression
-        new urls_api_1.UrlsApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Images API');
-        // tslint:disable-next-line:no-unused-expression
-        new images_api_1.ImagesApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
-        nodecommons_cli_1.CommonsOutput.doing('Installing REST Autocomplete API');
-        // tslint:disable-next-line:no-unused-expression
-        new autocomplete_api_1.AutocompleteApi(this, database, path);
-        nodecommons_cli_1.CommonsOutput.success();
+        const path = commonsExpressNginxProxyPath(expressConfig.path);
+        commonsOutputDoing('Installing REST Test API');
+        new TestApi(this, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Statistics API');
+        new StatisticsApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Crawl API');
+        new CrawlApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Bugs API');
+        new BugsApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Domains API');
+        new DomainsApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST URLs API');
+        new UrlsApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Images API');
+        new ImagesApi(this, database, path);
+        commonsOutputSuccess();
+        commonsOutputDoing('Installing REST Autocomplete API');
+        new AutocompleteApi(this, database, path);
+        commonsOutputSuccess();
     }
 }
-exports.RestServer = RestServer;
+//# sourceMappingURL=rest.server.js.map

package/dist/servers/rest.server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rest.server.js","sourceRoot":"","sources":["../../src/servers/rest.server.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,4BAA4B,EAA8B,MAAM,wBAAwB,CAAC;AAClG,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAI3D,MAAM,OAAO,UAAW,SAAQ,iBAAiB;IAChD,YACE,aAAyC,EACzC,QAAyB,EACzB,MAAqB;QAEtB,KAAK,CAAC,aAAa,CAAC,CAAC;QAErB,MAAM,aAAa,GAAoB,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnE,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAW,4BAA4B,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAEtE,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACxB,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,gCAAgC,CAAC,CAAC;QACrD,IAAI,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACxC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACnC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAClC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,oBAAoB,EAAE,CAAC;QAEvB,kBAAkB,CAAC,kCAAkC,CAAC,CAAC;QACvD,IAAI,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC1C,oBAAoB,EAAE,CAAC;IACxB,CAAC;CAED"}

package/dist/servers/socket-io.server.d.ts

@@ -0,0 +1,12 @@
+import { TStatusTallies } from 'hydra-crawler-ts-assets';
+import { TFetch } from 'hydra-crawler-ts-assets';
+import { TOutcome } from 'hydra-crawler-ts-assets';
+import { CommonsSocketIoServer } from 'nodecommons-es-socket-io';
+export declare class SocketIoServer extends CommonsSocketIoServer {
+    fetching(fetch: TFetch): Promise<void>;
+    outcome(outcome: TOutcome): Promise<void>;
+    statusTallies(tallies: TStatusTallies): Promise<void>;
+    linkTallies(tally: number): Promise<void>;
+    domainTallies(tally: number): Promise<void>;
+    bandwidth(bps: number): Promise<void>;
+}

package/dist/servers/socket-io.server.js

@@ -1,28 +1,61 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SocketIoServer = void 0;
-const nodecommons_app_angular_1 = require("nodecommons-app-angular");
-class SocketIoServer extends nodecommons_app_angular_1.CommonsAngularSocketIoServer {
-    constructor(expressServer, expressConfig) {
-        super(expressServer, expressConfig);
-    }
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { CommonsSocketIoServer } from 'nodecommons-es-socket-io';
+export class SocketIoServer extends CommonsSocketIoServer {
     fetching(fetch) {
-        super.broadcast('fetching', fetch);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'fetching', fetch);
+        });
     }
     outcome(outcome) {
-        super.broadcast('outcome', outcome);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'outcome', outcome);
+        });
     }
     statusTallies(tallies) {
-        super.broadcast('statusTallies', tallies);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'statusTallies', tallies);
+        });
     }
     linkTallies(tally) {
-        super.broadcast('linkTallies', tally);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'linkTallies', tally);
+        });
     }
     domainTallies(tally) {
-        super.broadcast('domainTallies', tally);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'domainTallies', tally);
+        });
     }
     bandwidth(bps) {
-        super.broadcast('bandwidth', bps);
+        const _super = Object.create(null, {
+            broadcast: { get: () => super.broadcast }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.broadcast.call(this, 'bandwidth', bps);
+        });
     }
 }
-exports.SocketIoServer = SocketIoServer;
+//# sourceMappingURL=socket-io.server.js.map

package/dist/servers/socket-io.server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"socket-io.server.js","sourceRoot":"","sources":["../../src/servers/socket-io.server.ts"],"names":[],"mappings":";;;;;;;;;AAIA,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,MAAM,OAAO,cAAe,SAAQ,qBAAqB;IAC3C,QAAQ,CAAC,KAAa;;;;;YAClC,MAAM,OAAM,SAAS,YAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;KAAA;IAEY,OAAO,CAAC,OAAiB;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;KAAA;IAEY,aAAa,CAAC,OAAuB;;;;;YACjD,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;KAAA;IAEY,WAAW,CAAC,KAAa;;;;;YACrC,MAAM,OAAM,SAAS,YAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEY,aAAa,CAAC,KAAa;;;;;YACvC,MAAM,OAAM,SAAS,YAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;KAAA;IAEY,SAAS,CAAC,GAAW;;;;;YACjC,MAAM,OAAM,SAAS,YAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;KAAA;CACD"}

package/dist/services/database.service.d.ts

@@ -0,0 +1,68 @@
+import { Db, Collection } from 'mongodb';
+import { IUrlErrors } from 'hydra-crawler-ts-assets';
+import { IImage } from 'hydra-crawler-ts-assets';
+import { IUrl } from 'hydra-crawler-ts-assets';
+import { TPhpError } from 'hydra-crawler-ts-assets';
+import { TAspError } from 'hydra-crawler-ts-assets';
+import { TDomain } from 'hydra-crawler-ts-assets';
+import { TLink } from 'hydra-crawler-ts-assets';
+import { EStatus } from 'hydra-crawler-ts-assets';
+import { EComparator } from 'hydra-crawler-ts-assets';
+import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
+import { TParserCtor } from '../types/tparser-ctor';
+import { EAvailableStrategy } from '../enums/eavailable-strategy';
+export declare type TMongoIdRow = {
+    _id: string;
+};
+export declare function isTMongoIdRow(test: unknown): test is TMongoIdRow;
+export declare type TMongoIdTallyRow = {
+    _id: string;
+    tally: number;
+};
+export declare function isTMongoIdTallyRow(test: unknown): test is TMongoIdTallyRow;
+export declare class DatabaseService extends CommonsMongodbService {
+    private domains;
+    private urls;
+    private links;
+    getDomains(): Collection<TDomain>;
+    getUrls(): Collection<IUrl>;
+    getLinks(): Collection<TLink>;
+    init(): Promise<void>;
+    initParser(ctor: TParserCtor): Promise<void>;
+    getRawDatabase(): Db;
+    wipe(): Promise<void>;
+    resetActive(): Promise<void>;
+    domain(domain: string, ip: string): Promise<boolean>;
+    queue(url: string, isDeny?: boolean): Promise<boolean>;
+    available(strategy: EAvailableStrategy, threshold: number, limit: number, existing: string[]): Promise<string[]>;
+    next(domain: string): Promise<string | undefined>;
+    setStatus(url: string, status: EStatus): Promise<boolean>;
+    setStatusCode(url: string, code: number): Promise<boolean>;
+    setHeaders(url: string, headers: {
+        [key: string]: string | number | Date;
+    }): Promise<boolean>;
+    setData(url: string, context: string, data: unknown): Promise<boolean>;
+    unsetData(url: string, context: string): Promise<boolean>;
+    getTtl(url: string): Promise<number | undefined>;
+    setTtl(url: string, ttl: number): Promise<boolean>;
+    unsetTtl(url: string): Promise<boolean>;
+    getHash(url: string): Promise<string | undefined>;
+    setHash(url: string, hash: string): Promise<boolean>;
+    unsetHash(url: string): Promise<boolean>;
+    setFailReason(url: string, reason: string): Promise<boolean>;
+    unsetFailReason(url: string): Promise<boolean>;
+    link(url: string, links: string[]): Promise<boolean>;
+    markDead(domain: string): Promise<boolean>;
+    listStatusTallies(): Promise<Map<EStatus, number>>;
+    getLinkTalliesCount(): Promise<number>;
+    getDomainTalliesCount(): Promise<number>;
+    listDomainQueuedTallies(): Promise<Map<string, number>>;
+    listPhpErrors(): Promise<IUrlErrors<TPhpError>[]>;
+    listAspErrors(): Promise<IUrlErrors<TAspError>[]>;
+    listDone200DomainUrls(domain: string): Promise<string[]>;
+    listDomains(): Promise<TDomain[]>;
+    listDomainsByLike(term: string): Promise<TDomain[]>;
+    listInboundLinks(url: string): Promise<string[]>;
+    listOutboundLinks(url: string): Promise<string[]>;
+    listImagesBySizeThreshold(size: number, comparator: EComparator): Promise<IImage[]>;
+}