hydra-crawler 1.4.6 → 2.0.2

package/dist/services/database.service.js

@@ -1,42 +1,37 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatabaseService = exports.isTMongoIdTallyRow = exports.isTMongoIdRow = void 0;
-const url_1 = require("url");
-const tscommons_core_1 = require("tscommons-core");
-const tscommons_core_2 = require("tscommons-core");
-const tscommons_core_3 = require("tscommons-core");
-const hydra_crawler_ts_assets_1 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_2 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_3 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_4 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_5 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_6 = require("hydra-crawler-ts-assets");
-const hydra_crawler_ts_assets_7 = require("hydra-crawler-ts-assets");
-const nodecommons_cli_1 = require("nodecommons-cli");
-const nodecommons_database_mongodb_1 = require("nodecommons-database-mongodb");
-const eavailable_strategy_1 = require("../enums/eavailable-strategy");
-function isTMongoIdRow(test) {
-    if (!tscommons_core_1.CommonsType.hasPropertyString(test, '_id'))
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { URL } from 'url';
+import { commonsObjectStripNulls, commonsStringRegexLike, commonsTypeAttemptNumber, commonsTypeHasPropertyNumber, commonsTypeHasPropertyString, commonsTypeHasPropertyTArray } from 'tscommons-es-core';
+import { isIUrl } from 'hydra-crawler-ts-assets';
+import { isTPhpError } from 'hydra-crawler-ts-assets';
+import { isTAspError } from 'hydra-crawler-ts-assets';
+import { isTDomain } from 'hydra-crawler-ts-assets';
+import { isTLink } from 'hydra-crawler-ts-assets';
+import { EStatus, toEStatus } from 'hydra-crawler-ts-assets';
+import { EComparator } from 'hydra-crawler-ts-assets';
+import { commonsOutputDebug, commonsOutputError } from 'nodecommons-es-cli';
+import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
+import { EAvailableStrategy } from '../enums/eavailable-strategy';
+export function isTMongoIdRow(test) {
+    if (!commonsTypeHasPropertyString(test, '_id'))
         return false;
     return true;
 }
-exports.isTMongoIdRow = isTMongoIdRow;
-function isTMongoIdTallyRow(test) {
+export function isTMongoIdTallyRow(test) {
     if (!isTMongoIdRow(test))
         return false;
-    if (!tscommons_core_1.CommonsType.hasPropertyNumber(test, 'tally'))
+    if (!commonsTypeHasPropertyNumber(test, 'tally'))
         return false;
     return true;
 }
-exports.isTMongoIdTallyRow = isTMongoIdTallyRow;
-class DatabaseService extends nodecommons_database_mongodb_1.CommonsMongodbService {
-    constructor(credentials) {
-        super(credentials);
-        //		CommonsGracefulAbort.addCallback((): void => {
-        //			CommonsOutput.alert(`SIGINT abort flag is set. Closing MongoDB connection.`);
-        //			super.close();
-        //		});
-    }
+export class DatabaseService extends CommonsMongodbService {
     getDomains() {
         if (!this.domains)
             throw new Error('Domains collected has not been instantiated yet');
@@ -52,454 +47,524 @@ class DatabaseService extends nodecommons_database_mongodb_1.CommonsMongodbServi
             throw new Error('Links collected has not been instantiated yet');
         return this.links;
     }
-    async init() {
-        await super.init();
-        if (!this.database)
-            throw new Error('Database has not been instantiated yet');
-        this.domains = await this.ensureCollection('domains');
-        await this.domains.createIndex({ domain: 1 }, { unique: true });
-        await this.domains.createIndex({ ip: 1 }, { unique: false });
-        this.urls = await this.ensureCollection('urls');
-        await this.urls.createIndex({ url: 1 }, { unique: true });
-        await this.urls.createIndex({ domain: 1 }, { unique: false });
-        await this.urls.createIndex({ status: 1 }, { unique: false });
-        await this.urls.createIndex({ attempted: 1 }, { unique: false });
-        await this.urls.createIndex({ done: 1 }, { unique: false });
-        await this.urls.createIndex({ orphan: 1 }, { unique: false });
-        await this.urls.createIndex({ statusCode: 1 }, { unique: false });
-        await this.urls.createIndex({ reason: 1 }, { unique: false });
-        await this.urls.createIndex({ 'headers.content-type': 1 }, { unique: false });
-        this.links = await this.ensureCollection('links');
-        await this.links.createIndex({ url: 1 }, { unique: false });
-        await this.links.createIndex({ outgoing: 1 }, { unique: false });
-        await this.links.createIndex({ url: 1, outgoing: 1 }, { unique: true });
-    }
-    async initParser(ctor) {
-        const parser = new ctor();
-        await parser.init(this);
+    init() {
+        const _super = Object.create(null, {
+            init: { get: () => super.init }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            yield _super.init.call(this);
+            if (!this.database)
+                throw new Error('Database has not been instantiated yet');
+            this.domains = yield this.ensureCollection('domains');
+            yield this.domains.createIndex({ domain: 1 }, { unique: true });
+            yield this.domains.createIndex({ ip: 1 }, { unique: false });
+            this.urls = yield this.ensureCollection('urls');
+            yield this.urls.createIndex({ url: 1 }, { unique: true });
+            yield this.urls.createIndex({ domain: 1 }, { unique: false });
+            yield this.urls.createIndex({ status: 1 }, { unique: false });
+            yield this.urls.createIndex({ attempted: 1 }, { unique: false });
+            yield this.urls.createIndex({ orphan: 1 }, { unique: false });
+            yield this.urls.createIndex({ statusCode: 1 }, { unique: false });
+            yield this.urls.createIndex({ 'headers.content-type': 1 }, { unique: false });
+            yield this.urls.createIndex({ 'headers.content-length': 1 }, { unique: false });
+            this.links = yield this.ensureCollection('links');
+            yield this.links.createIndex({ url: 1 }, { unique: false });
+            yield this.links.createIndex({ outgoing: 1 }, { unique: false });
+            yield this.links.createIndex({ url: 1, outgoing: 1 }, { unique: true });
+        });
+    }
+    initParser(ctor) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const parser = new ctor();
+            yield parser.init(this);
+        });
     }
     getRawDatabase() {
         return super.getRawDatabase();
     }
-    async wipe() {
-        await this.getLinks().deleteMany({});
-        await this.getUrls().deleteMany({});
-        await this.getDomains().deleteMany({});
+    wipe() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.getLinks().deleteMany({});
+            yield this.getUrls().deleteMany({});
+            yield this.getDomains().deleteMany({});
+        });
     }
-    async resetActive() {
-        await this.getUrls().updateMany({ status: hydra_crawler_ts_assets_6.EStatus.ACTIVE }, { $set: { status: hydra_crawler_ts_assets_6.EStatus.QUEUED } });
+    resetActive() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.getUrls().updateMany({ status: EStatus.ACTIVE }, { $set: { status: EStatus.QUEUED } });
+        });
     }
-    async domain(domain, ip) {
-        try {
-            await this.getDomains().updateOne({ domain: domain }, { $set: { ip: ip } }, { upsert: true });
-            return true;
-        }
-        catch (ex) {
-            nodecommons_cli_1.CommonsOutput.debug('debug position 7');
-            console.log(ex);
-            return false;
-        }
-    }
-    async queue(url, isDeny) {
-        const whatwg = new url_1.URL(url);
-        if (!whatwg.protocol.match(/^http(s?):$/))
-            return false;
-        try {
-            const status = isDeny ? hydra_crawler_ts_assets_6.EStatus.DENY : hydra_crawler_ts_assets_6.EStatus.QUEUED;
-            // un-archive if currently archived
-            await this.getUrls().deleteOne({ url: url, status: hydra_crawler_ts_assets_6.EStatus.ARCHIVED });
-            await this.getUrls().insertOne({
-                url: url,
-                domain: whatwg.hostname,
-                status: status
-            });
-            return true;
-        }
-        catch (ex) {
-            return false;
-        }
-    }
-    async available(strategy, threshold, limit, existing) {
-        if (limit === 0)
-            return [];
-        const comparator = strategy === eavailable_strategy_1.EAvailableStrategy.LARGEST ? hydra_crawler_ts_assets_7.EComparator.GTE : hydra_crawler_ts_assets_7.EComparator.LT;
-        let thresholdMatch = {};
-        switch (comparator) {
-            case hydra_crawler_ts_assets_7.EComparator.GTE:
-                thresholdMatch = { tally: { $gte: threshold } };
-                break;
-            case hydra_crawler_ts_assets_7.EComparator.LT:
-                thresholdMatch = { tally: { $lt: threshold } };
-                break;
-        }
-        const sortOrder = strategy === eavailable_strategy_1.EAvailableStrategy.SMALLEST ? 1 : -1;
-        const results = this.getUrls().aggregate([
-            { $match: { status: hydra_crawler_ts_assets_6.EStatus.QUEUED, domain: { $nin: existing } } },
-            { $group: { _id: '$domain', tally: { $sum: 1 } } },
-            { $match: thresholdMatch },
-            { $sort: { tally: sortOrder } },
-            { $limit: limit }
-        ], { allowDiskUse: true });
-        const entries = await this.listQueryResults(results, isTMongoIdTallyRow);
-        return entries
-            .map((entry) => entry._id);
-    }
-    async next(domain) {
-        const next = await this.getUrls().findOne({
-            status: hydra_crawler_ts_assets_6.EStatus.QUEUED,
-            domain: domain
-        });
-        if (next === null)
-            return undefined;
-        return next.url;
-    }
-    async setStatus(url, status) {
-        const updates = {};
-        updates['status'] = status;
-        if (![hydra_crawler_ts_assets_6.EStatus.QUEUED, hydra_crawler_ts_assets_6.EStatus.ACTIVE].includes(status))
-            updates['attempted'] = new Date();
-        if (status === hydra_crawler_ts_assets_6.EStatus.DONE)
-            updates['done'] = new Date();
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: updates });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async setStatusCode(url, code) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: { statusCode: code }
-            });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async setHeaders(url, headers) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: { headers: headers }
-            });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async setData(url, context, data) {
-        try {
-            const update = {};
-            update[context] = data;
-            await this.getUrls().updateOne({ url: url }, { $set: update });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async unsetData(url, context) {
-        try {
-            const update = {};
-            update[context] = true;
-            await this.getUrls().updateOne({ url: url }, { $unset: update });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async getTtl(url) {
-        const row = await this.getUrls().findOne({ url: url });
-        if (row === null)
-            return undefined;
-        if (!tscommons_core_1.CommonsType.hasPropertyNumber(row, 'ttl'))
-            return undefined;
-        return row['ttl'];
-    }
-    async setTtl(url, ttl) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: { ttl: ttl }
-            });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async unsetTtl(url) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $unset: { ttl: true }
-            });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async getHash(url) {
-        const row = await this.getUrls().findOne({ url: url });
-        if (row === null)
-            return undefined;
-        if (!tscommons_core_1.CommonsType.hasPropertyString(row, 'hash'))
-            return undefined;
-        return row['hash'];
-    }
-    async setHash(url, hash) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: {
-                    hash: hash,
-                    hashSet: new Date()
-                } });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async unsetHash(url) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $unset: {
-                    hash: true,
-                    hashSet: true
-                } });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async setFailReason(url, reason) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $set: {
-                    reason: reason
-                } });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async unsetFailReason(url) {
-        try {
-            await this.getUrls().updateOne({ url: url }, { $unset: {
-                    reason: true
-                } });
-            return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async link(url, links) {
-        // more efficient to only remove removed and only add new
-        // rather than just wiping all existing and re-adding
-        const find = this.getLinks().find({
-            url: url
-        });
-        const existing = (await this.listQueryResults(find, hydra_crawler_ts_assets_5.isTLink))
-            .map((link) => link.outgoing);
-        const removed = [];
-        const added = [];
-        for (const link of links) {
-            if (!existing.includes(link) && !added.includes(link))
-                added.push(link);
-        }
-        for (const link of existing) {
-            if (!links.includes(link))
-                removed.push(link);
-        }
-        for (const outgoing of removed) {
+    domain(domain, ip) {
+        return __awaiter(this, void 0, void 0, function* () {
             try {
-                await this.getLinks().deleteMany({
-                    url: url,
-                    outgoing: outgoing
-                });
+                yield this.getDomains().updateOne({ domain: domain }, { $set: { ip: ip } }, { upsert: true });
+                return true;
             }
             catch (ex) {
-                /* do nothing */
+                commonsOutputDebug('debug position 7');
+                console.log(ex);
+                return false;
             }
-        }
-        for (const outgoing of added) {
+        });
+    }
+    queue(url, isDeny) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const whatwg = new URL(url);
+            if (!whatwg.protocol.match(/^http(s?):$/))
+                return false;
             try {
-                await this.getLinks().insertOne({
+                const status = isDeny ? EStatus.DENY : EStatus.QUEUED;
+                // un-archive if currently archived
+                yield this.getUrls().deleteOne({ url: url, status: EStatus.ARCHIVED });
+                yield this.getUrls().insertOne({
                     url: url,
-                    outgoing: outgoing
+                    domain: whatwg.hostname,
+                    status: status
                 });
+                return true;
             }
             catch (ex) {
-                switch (ex.code) {
-                    case 11000:
-                        // ignore duplicates
-                        nodecommons_cli_1.CommonsOutput.error(`DUPLICATE: ${url}, ${outgoing}`);
-                        break;
-                    case 17280:
-                    case 17282:
-                        nodecommons_cli_1.CommonsOutput.error(`INDEX TOO LARGE: ${url}, ${outgoing}`);
-                        // ignore index too large
-                        break;
-                    default:
-                        nodecommons_cli_1.CommonsOutput.debug('debug position 8');
-                        console.log(ex);
-                        throw ex;
-                }
+                return false;
+            }
+        });
+    }
+    available(strategy, threshold, limit, existing) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (limit === 0)
+                return [];
+            const comparator = strategy === EAvailableStrategy.LARGEST ? EComparator.GTE : EComparator.LT;
+            const thresholdMatch = { tally: {} };
+            switch (comparator) {
+                case EComparator.GTE:
+                    thresholdMatch.tally = { $gte: threshold };
+                    break;
+                case EComparator.LT:
+                    thresholdMatch.tally = { $lt: threshold };
+                    break;
+            }
+            const sortOrder = strategy === EAvailableStrategy.SMALLEST ? 1 : -1;
+            const results = this.getUrls().aggregate([
+                { $match: { status: EStatus.QUEUED, domain: { $nin: existing } } },
+                { $group: { _id: '$domain', tally: { $sum: 1 } } },
+                { $match: thresholdMatch },
+                { $sort: { tally: sortOrder } },
+                { $limit: limit }
+            ], { allowDiskUse: true });
+            const entries = yield this.listQueryResults(results, isTMongoIdTallyRow);
+            return entries
+                // eslint-disable-next-line no-underscore-dangle
+                .map((entry) => entry._id);
+        });
+    }
+    next(domain) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const next = yield this.getUrls().findOne({
+                status: EStatus.QUEUED,
+                domain: domain
+            });
+            if (next === null)
+                return undefined;
+            return next.url;
+        });
+    }
+    setStatus(url, status) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const updates = { status: status };
+            if (![EStatus.QUEUED, EStatus.ACTIVE].includes(status))
+                updates['attempted'] = new Date();
+            if (status === EStatus.DONE)
+                updates['done'] = new Date();
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: updates });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    setStatusCode(url, code) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: { statusCode: code } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    setHeaders(url, headers) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: { headers: headers } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    setData(url, context, data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const update = {};
+                update[context] = data;
+                yield this.getUrls().updateOne({ url: url }, { $set: update });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    unsetData(url, context) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const update = {};
+                update[context] = true;
+                yield this.getUrls().updateOne({ url: url }, { $unset: update });
+                return true;
             }
-        }
-        return true;
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
     }
-    async markDead(domain) {
-        try {
-            await this.getUrls().updateMany({ domain: domain, status: { $in: [hydra_crawler_ts_assets_6.EStatus.QUEUED, hydra_crawler_ts_assets_6.EStatus.ACTIVE] } }, { $set: { status: hydra_crawler_ts_assets_6.EStatus.DEAD, attempted: new Date() } });
+    getTtl(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const row = yield this.getUrls().findOne({ url: url });
+            if (row === null)
+                return undefined;
+            if (!commonsTypeHasPropertyNumber(row, 'ttl'))
+                return undefined;
+            return row['ttl'];
+        });
+    }
+    setTtl(url, ttl) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: { ttl: ttl } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    unsetTtl(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $unset: { ttl: true } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    getHash(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const row = yield this.getUrls().findOne({ url: url });
+            if (row === null)
+                return undefined;
+            if (!commonsTypeHasPropertyString(row, 'hash'))
+                return undefined;
+            return row['hash'];
+        });
+    }
+    setHash(url, hash) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: {
+                        hash: hash,
+                        hashSet: new Date()
+                    } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    unsetHash(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $unset: {
+                        hash: true,
+                        hashSet: true
+                    } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    setFailReason(url, reason) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $set: {
+                        reason: reason
+                    } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    unsetFailReason(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateOne({ url: url }, { $unset: {
+                        reason: true
+                    } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    link(url, links) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // more efficient to only remove removed and only add new
+            // rather than just wiping all existing and re-adding
+            const find = this.getLinks().find({
+                url: url
+            }, {});
+            const existing = (yield this.listQueryResults(find, isTLink))
+                .map((link) => link.outgoing);
+            const removed = [];
+            const added = [];
+            for (const link of links) {
+                if (!existing.includes(link) && !added.includes(link))
+                    added.push(link);
+            }
+            for (const link of existing) {
+                if (!links.includes(link))
+                    removed.push(link);
+            }
+            for (const outgoing of removed) {
+                try {
+                    yield this.getLinks().deleteMany({
+                        url: url,
+                        outgoing: outgoing
+                    });
+                }
+                catch (ex) {
+                    /* do nothing */
+                }
+            }
+            for (const outgoing of added) {
+                try {
+                    yield this.getLinks().insertOne({
+                        url: url,
+                        outgoing: outgoing
+                    });
+                }
+                catch (ex) {
+                    switch (ex.code || -1) {
+                        case 11000:
+                            // ignore duplicates
+                            commonsOutputError(`DUPLICATE: ${url}, ${outgoing}`);
+                            break;
+                        case 17280:
+                        case 17282:
+                            commonsOutputError(`INDEX TOO LARGE: ${url}, ${outgoing}`);
+                            // ignore index too large
+                            break;
+                        default:
+                            commonsOutputDebug('debug position 8');
+                            console.log(ex);
+                            throw ex;
+                    }
+                }
+            }
             return true;
-        }
-        catch (ex) {
-            console.error(ex);
-            return false;
-        }
-    }
-    async listStatusTallies() {
-        const results = this.getUrls().aggregate([
-            { $match: { status: { $ne: hydra_crawler_ts_assets_6.EStatus.ARCHIVED } } },
-            { $group: {
-                    _id: '$status',
-                    tally: { $sum: 1 }
-                } }
-        ]);
-        const rows = await this.listQueryResults(results, isTMongoIdTallyRow);
-        const map = new Map();
-        for (const row of rows) {
-            const status = hydra_crawler_ts_assets_6.toEStatus(row._id);
-            if (status)
-                map.set(status, row.tally);
-        }
-        return map;
-    }
-    async getLinkTalliesCount() {
-        return await this.getLinks().countDocuments();
-    }
-    async getDomainTalliesCount() {
-        return await this.getDomains().countDocuments();
-    }
-    async listDomainQueuedTallies() {
-        const results = this.getUrls().aggregate([
-            { $match: {
-                    status: hydra_crawler_ts_assets_6.EStatus.QUEUED
-                } },
-            { $group: {
-                    _id: '$domain',
-                    tally: { $sum: 1 }
-                } }
-        ]);
-        const rows = await this.listQueryResults(results, isTMongoIdTallyRow);
-        const map = new Map();
-        for (const row of rows) {
-            map.set(row._id, row.tally);
-        }
-        return map;
-    }
-    async listPhpErrors() {
-        const results = this.getUrls().find({
-            status: { $ne: hydra_crawler_ts_assets_6.EStatus.ARCHIVED },
-            phpErrors: { $exists: true }
-        });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_1.isIUrl))
-            .map((url) => {
-            if (!tscommons_core_1.CommonsType.hasPropertyTArray(url, 'phpErrors', hydra_crawler_ts_assets_2.isTPhpError))
-                throw new Error('Invalid PHP error object');
-            return {
-                url: url.url,
-                errors: url['phpErrors']
-            };
-        });
-    }
-    async listAspErrors() {
-        const results = this.getUrls().find({
-            status: { $ne: hydra_crawler_ts_assets_6.EStatus.ARCHIVED },
-            aspErrors: { $exists: true }
-        });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_1.isIUrl))
-            .map((url) => {
-            if (!tscommons_core_1.CommonsType.hasPropertyTArray(url, 'aspErrors', hydra_crawler_ts_assets_3.isTAspError))
-                throw new Error('Invalid PHP error object');
-            return {
-                url: url.url,
-                errors: url['aspErrors']
-            };
-        });
-    }
-    async listDone200DomainUrls(domain) {
-        const results = this.getUrls().find({ $and: [
-                { domain: domain },
-                { status: hydra_crawler_ts_assets_6.EStatus.DONE },
-                { statusCode: { $gte: 200 } },
-                { statusCode: { $lt: 300 } }
-            ] });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_1.isIUrl))
-            .map((url) => url.url);
-    }
-    async listDomains() {
-        const results = this.getDomains().find({ ip: { $exists: true, $ne: null } } // this is ok, despite the type objection to null
-        );
-        // since we're doing $ne: null above, we don't need to strip nulls, as there won't be any
-        return await this.listQueryResults(results, hydra_crawler_ts_assets_4.isTDomain);
-    }
-    async listDomainsByLike(term) {
-        const results = this.getDomains().find({
-            domain: new RegExp(tscommons_core_3.CommonsString.regexLike(`%${term}%`), 'i')
-        });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_4.isTDomain))
-            .map((encoded) => {
-            return tscommons_core_2.CommonsObject.stripNulls(encoded);
-        });
-    }
-    async listInboundLinks(url) {
-        const results = this.getLinks().find({ outgoing: url });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_5.isTLink))
-            .map((link) => link.url);
-    }
-    async listOutboundLinks(url) {
-        const results = this.getLinks().find({ url: url });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_5.isTLink))
-            .map((link) => link.outgoing);
-    }
-    async listImagesBySizeThreshold(size, comparator) {
-        const queries = [
-            { status: hydra_crawler_ts_assets_6.EStatus.DONE },
-            { 'headers.content-type': /^image\/(jpeg)/ },
-            { 'headers.content-length': { $exists: true } }
-        ];
-        switch (comparator) {
-            case hydra_crawler_ts_assets_7.EComparator.GT:
-                queries.push({ 'headers.content-length': { $gt: size } });
-                break;
-            case hydra_crawler_ts_assets_7.EComparator.LT:
-                queries.push({ 'headers.content-length': { $lt: size } });
-                break;
-            case hydra_crawler_ts_assets_7.EComparator.GTE:
-                queries.push({ 'headers.content-length': { $gte: size } });
-                break;
-            case hydra_crawler_ts_assets_7.EComparator.LTE:
-                queries.push({ 'headers.content-length': { $lte: size } });
-                break;
-        }
-        const results = this.getUrls().find({ $and: queries });
-        return (await this.listQueryResults(results, hydra_crawler_ts_assets_1.isIUrl))
-            .map((row) => ({
-            url: row.url,
-            size: row['headers']['content-length']
-        }));
+        });
+    }
+    markDead(domain) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.getUrls().updateMany({ domain: domain, status: { $in: [EStatus.QUEUED, EStatus.ACTIVE] } }, { $set: { status: EStatus.DEAD, attempted: new Date() } });
+                return true;
+            }
+            catch (ex) {
+                console.error(ex);
+                return false;
+            }
+        });
+    }
+    listStatusTallies() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getUrls().aggregate([
+                { $match: { status: { $ne: EStatus.ARCHIVED } } },
+                { $group: {
+                        _id: '$status',
+                        tally: { $sum: 1 }
+                    } }
+            ]);
+            const rows = yield this.listQueryResults(results, isTMongoIdTallyRow);
+            const map = new Map();
+            for (const row of rows) {
+                // eslint-disable-next-line no-underscore-dangle
+                const status = toEStatus(row._id);
+                if (status)
+                    map.set(status, row.tally);
+            }
+            return map;
+        });
+    }
+    getLinkTalliesCount() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.getLinks().countDocuments();
+        });
+    }
+    getDomainTalliesCount() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return yield this.getDomains().countDocuments();
+        });
+    }
+    listDomainQueuedTallies() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getUrls().aggregate([
+                { $match: {
+                        status: EStatus.QUEUED
+                    } },
+                { $group: {
+                        _id: '$domain',
+                        tally: { $sum: 1 }
+                    } }
+            ]);
+            const rows = yield this.listQueryResults(results, isTMongoIdTallyRow);
+            const map = new Map();
+            for (const row of rows) {
+                // eslint-disable-next-line no-underscore-dangle
+                map.set(row._id, row.tally);
+            }
+            return map;
+        });
+    }
+    listPhpErrors() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getUrls().find({
+                status: { $ne: EStatus.ARCHIVED },
+                phpErrors: { $exists: true }
+            }, {});
+            return (yield this.listQueryResults(results, isIUrl))
+                .map((url) => {
+                if (!commonsTypeHasPropertyTArray(url, 'phpErrors', isTPhpError))
+                    throw new Error('Invalid PHP error object');
+                return {
+                    url: url.url,
+                    errors: url.phpErrors
+                };
+            });
+        });
+    }
+    listAspErrors() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getUrls().find({
+                status: { $ne: EStatus.ARCHIVED },
+                aspErrors: { $exists: true }
+            }, {});
+            return (yield this.listQueryResults(results, isIUrl))
+                .map((url) => {
+                if (!commonsTypeHasPropertyTArray(url, 'aspErrors', isTAspError))
+                    throw new Error('Invalid PHP error object');
+                return {
+                    url: url.url,
+                    errors: url.aspErrors
+                };
+            });
+        });
+    }
+    listDone200DomainUrls(domain) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getUrls().find({ $and: [
+                    { domain: domain },
+                    { status: EStatus.DONE },
+                    { statusCode: { $gte: 200 } },
+                    { statusCode: { $lt: 300 } }
+                ] }, {});
+            return (yield this.listQueryResults(results, isIUrl))
+                .map((url) => url.url);
+        });
+    }
+    listDomains() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getDomains().find(
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+            { ip: { $exists: true, $ne: null } }, // this is ok, despite the type objection to null
+            {});
+            // since we're doing $ne: null above, we don't need to strip nulls, as there won't be any
+            return yield this.listQueryResults(results, isTDomain);
+        });
+    }
+    listDomainsByLike(term) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getDomains().find({
+                domain: new RegExp(commonsStringRegexLike(`%${term}%`), 'i')
+            }, {});
+            return (yield this.listQueryResults(results, isTDomain))
+                .map((encoded) => commonsObjectStripNulls(encoded));
+        });
+    }
+    listInboundLinks(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getLinks().find({ outgoing: url }, {});
+            return (yield this.listQueryResults(results, isTLink))
+                .map((link) => link.url);
+        });
+    }
+    listOutboundLinks(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = this.getLinks().find({ url: url }, {});
+            return (yield this.listQueryResults(results, isTLink))
+                .map((link) => link.outgoing);
+        });
+    }
+    listImagesBySizeThreshold(size, comparator) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const queries = [
+                { status: EStatus.DONE },
+                { 'headers.content-type': /^image\/(jpeg)/ },
+                { 'headers.content-length': { $exists: true } }
+            ];
+            switch (comparator) {
+                case EComparator.GT:
+                    queries.push({ 'headers.content-length': { $gt: size } });
+                    break;
+                case EComparator.LT:
+                    queries.push({ 'headers.content-length': { $lt: size } });
+                    break;
+                case EComparator.GTE:
+                    queries.push({ 'headers.content-length': { $gte: size } });
+                    break;
+                case EComparator.LTE:
+                    queries.push({ 'headers.content-length': { $lte: size } });
+                    break;
+            }
+            const results = this.getUrls().find({ $and: queries }, {});
+            return (yield this.listQueryResults(results, isIUrl))
+                .map((row) => ({
+                url: row.url,
+                size: commonsTypeAttemptNumber(row['headers']['content-length']) || -1
+            }));
+        });
     }
 }
-exports.DatabaseService = DatabaseService;
+//# sourceMappingURL=database.service.js.map