hydra-crawler 2.8.3 → 3.0.1

package/dist/services/database.service.js

@@ -1,795 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-import { URL } from 'url';
-import mongodb from 'mongodb';
-import { commonsObjectStripNulls, commonsStringRegexLike, commonsTypeAttemptNumber, commonsTypeHasPropertyNumber, commonsTypeHasPropertyString, commonsTypeHasPropertyTArray } from 'tscommons-es-core';
-import { isIUrl } from 'hydra-crawler-ts-assets';
-import { isTPhpError } from 'hydra-crawler-ts-assets';
-import { isTAspError } from 'hydra-crawler-ts-assets';
-import { isTDomain } from 'hydra-crawler-ts-assets';
-import { isTLink } from 'hydra-crawler-ts-assets';
-import { EStatus, toEStatus } from 'hydra-crawler-ts-assets';
-import { EComparator } from 'hydra-crawler-ts-assets';
-import { commonsOutputDebug, commonsOutputDoing, commonsOutputError, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsMongodbService } from 'nodecommons-es-database-mongodb';
-import { EAvailableStrategy } from '../enums/eavailable-strategy';
-export function isTMongoIdRow(test) {
-    if (!commonsTypeHasPropertyString(test, '_id'))
-        return false;
-    return true;
-}
-export function isTMongoIdTallyRow(test) {
-    if (!isTMongoIdRow(test))
-        return false;
-    if (!commonsTypeHasPropertyNumber(test, 'tally'))
-        return false;
-    return true;
-}
-export class DatabaseService extends CommonsMongodbService {
-    getDomains() {
-        if (!this.domains)
-            throw new Error('Domains collected has not been instantiated yet');
-        return this.domains;
-    }
-    getUrls() {
-        if (!this.urls)
-            throw new Error('Urls collected has not been instantiated yet');
-        return this.urls;
-    }
-    getLinks() {
-        if (!this.links)
-            throw new Error('Links collected has not been instantiated yet');
-        return this.links;
-    }
-    getArchiveds() {
-        if (!this.archiveds)
-            throw new Error('Archiveds collected has not been instantiated yet');
-        return this.archiveds;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            yield _super.init.call(this);
-            if (!this.database)
-                throw new Error('Database has not been instantiated yet');
-            this.domains = yield this.ensureCollection('domains');
-            yield this.domains.createIndex({ domain: 1 }, { unique: true });
-            yield this.domains.createIndex({ ip: 1 }, { unique: false });
-            this.urls = yield this.ensureCollection('urls');
-            yield this.urls.createIndex({ url: 1 }, { unique: true });
-            yield this.urls.createIndex({ domain: 1 }, { unique: false });
-            yield this.urls.createIndex({ status: 1 }, { unique: false });
-            yield this.urls.createIndex({ attempted: 1 }, { unique: false });
-            yield this.urls.createIndex({ orphan: 1 }, { unique: false });
-            yield this.urls.createIndex({ statusCode: 1 }, { unique: false });
-            yield this.urls.createIndex({ 'headers.content-type': 1 }, { unique: false });
-            yield this.urls.createIndex({ 'headers.content-length': 1 }, { unique: false });
-            this.links = yield this.ensureCollection('links');
-            yield this.links.createIndex({ url: 1 }, { unique: false });
-            yield this.links.createIndex({ outgoing: 1 }, { unique: false });
-            yield this.links.createIndex({ url: 1, outgoing: 1 }, { unique: true });
-            this.archiveds = yield this.ensureCollection('archiveds');
-            // no indices for the archiveds
-        });
-    }
-    initParser(ctor) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const parser = new ctor();
-            yield parser.init(this);
-        });
-    }
-    getRawDatabase() {
-        return super.getRawDatabase();
-    }
-    wipe() {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.getLinks().deleteMany({});
-            yield this.getUrls().deleteMany({});
-            yield this.getDomains().deleteMany({});
-        });
-    }
-    resetActive() {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield this.getUrls().updateMany({ status: EStatus.ACTIVE }, { $set: { status: EStatus.QUEUED } });
-        });
-    }
-    domain(domain, ip) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getDomains().updateOne({ domain: domain }, { $set: { ip: ip } }, { upsert: true });
-                return true;
-            }
-            catch (ex) {
-                commonsOutputDebug('debug position 7');
-                console.log(ex);
-                return false;
-            }
-        });
-    }
-    queue(url, isDeny) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const whatwg = new URL(url);
-            if (!whatwg.protocol.match(/^http(s?):$/))
-                return false;
-            try {
-                const status = isDeny ? EStatus.DENY : EStatus.QUEUED;
-                // un-archive if currently archived
-                yield this.getUrls().deleteOne({ url: url, status: EStatus.ARCHIVED });
-                yield this.getUrls().insertOne({
-                    url: url,
-                    domain: whatwg.hostname,
-                    status: status
-                });
-                return true;
-            }
-            catch (ex) {
-                return false;
-            }
-        });
-    }
-    available(strategy, threshold, limit, existing, restrictTo) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (limit === 0)
-                return [];
-            const comparator = strategy === EAvailableStrategy.LARGEST ? EComparator.GTE : EComparator.LT;
-            const thresholdMatch = { tally: {} };
-            switch (comparator) {
-                case EComparator.GTE:
-                    thresholdMatch.tally = { $gte: threshold };
-                    break;
-                case EComparator.LT:
-                    thresholdMatch.tally = { $lt: threshold };
-                    break;
-            }
-            const sortOrder = strategy === EAvailableStrategy.SMALLEST ? 1 : -1;
-            const domainMatch = { $nin: existing };
-            if (restrictTo.length > 0)
-                domainMatch['$in'] = restrictTo;
-            const results = this.getUrls().aggregate([
-                { $match: { status: EStatus.QUEUED, domain: domainMatch } },
-                { $group: { _id: '$domain', tally: { $sum: 1 } } },
-                { $match: thresholdMatch },
-                { $sort: { tally: sortOrder } },
-                { $limit: limit }
-            ], { allowDiskUse: true });
-            const entries = yield this.listQueryResults(results, isTMongoIdTallyRow);
-            return entries
-                // eslint-disable-next-line no-underscore-dangle
-                .map((entry) => entry._id);
-        });
-    }
-    next(domain) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const next = yield this.getUrls().findOne({
-                status: EStatus.QUEUED,
-                domain: domain
-            });
-            if (next === null)
-                return undefined;
-            return next.url;
-        });
-    }
-    setStatus(url, status) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const updates = { status: status };
-            if (![EStatus.QUEUED, EStatus.ACTIVE].includes(status))
-                updates['attempted'] = new Date();
-            if (status === EStatus.DONE)
-                updates['done'] = new Date();
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: updates });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    setStatusCode(url, code) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: { statusCode: code } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    setHeaders(url, headers) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: { headers: headers } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    setData(url, context, data) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                const update = {};
-                update[context] = data;
-                yield this.getUrls().updateOne({ url: url }, { $set: update });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    unsetData(url, context) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                const update = {};
-                update[context] = true;
-                yield this.getUrls().updateOne({ url: url }, { $unset: update });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    getTtl(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const row = yield this.getUrls().findOne({ url: url });
-            if (row === null)
-                return undefined;
-            if (!commonsTypeHasPropertyNumber(row, 'ttl'))
-                return undefined;
-            return row['ttl'];
-        });
-    }
-    setTtl(url, ttl) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: { ttl: ttl } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    unsetTtl(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $unset: { ttl: true } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    getHash(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const row = yield this.getUrls().findOne({ url: url });
-            if (row === null)
-                return undefined;
-            if (!commonsTypeHasPropertyString(row, 'hash'))
-                return undefined;
-            return row['hash'];
-        });
-    }
-    setHash(url, hash) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: {
-                        hash: hash,
-                        hashSet: new Date()
-                    } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    unsetHash(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $unset: {
-                        hash: true,
-                        hashSet: true
-                    } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    setFailReason(url, reason) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $set: {
-                        reason: reason
-                    } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    unsetFailReason(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateOne({ url: url }, { $unset: {
-                        reason: true
-                    } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    link(url, links) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // more efficient to only remove removed and only add new
-            // rather than just wiping all existing and re-adding
-            const find = this.getLinks().find({
-                url: url
-            }, {});
-            const existing = (yield this.listQueryResults(find, isTLink))
-                .map((link) => link.outgoing);
-            const removed = [];
-            const added = [];
-            for (const link of links) {
-                if (!existing.includes(link) && !added.includes(link))
-                    added.push(link);
-            }
-            for (const link of existing) {
-                if (!links.includes(link))
-                    removed.push(link);
-            }
-            for (const outgoing of removed) {
-                try {
-                    yield this.getLinks().deleteMany({
-                        url: url,
-                        outgoing: outgoing
-                    });
-                }
-                catch (ex) {
-                    /* do nothing */
-                }
-            }
-            for (const outgoing of added) {
-                try {
-                    yield this.getLinks().insertOne({
-                        url: url,
-                        outgoing: outgoing
-                    });
-                }
-                catch (ex) {
-                    switch (ex.code || -1) {
-                        case 11000:
-                            // ignore duplicates
-                            commonsOutputError(`DUPLICATE: ${url}, ${outgoing}`);
-                            break;
-                        case 17280:
-                        case 17282:
-                            commonsOutputError(`INDEX TOO LARGE: ${url}, ${outgoing}`);
-                            // ignore index too large
-                            break;
-                        default:
-                            commonsOutputDebug('debug position 8');
-                            console.log(ex);
-                            throw ex;
-                    }
-                }
-            }
-            return true;
-        });
-    }
-    markDead(domain) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.getUrls().updateMany({ domain: domain, status: { $in: [EStatus.QUEUED, EStatus.ACTIVE] } }, { $set: { status: EStatus.DEAD, attempted: new Date() } });
-                return true;
-            }
-            catch (ex) {
-                console.error(ex);
-                return false;
-            }
-        });
-    }
-    listStatusTallies() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getUrls().aggregate([
-                { $match: { status: { $ne: EStatus.ARCHIVED } } },
-                { $group: {
-                        _id: '$status',
-                        tally: { $sum: 1 }
-                    } }
-            ]);
-            const rows = yield this.listQueryResults(results, isTMongoIdTallyRow);
-            const map = new Map();
-            for (const row of rows) {
-                // eslint-disable-next-line no-underscore-dangle
-                const status = toEStatus(row._id);
-                if (status)
-                    map.set(status, row.tally);
-            }
-            return map;
-        });
-    }
-    getLinkTalliesCount() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return yield this.getLinks().estimatedDocumentCount();
-        });
-    }
-    getDomainTalliesCount() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return yield this.getDomains().estimatedDocumentCount();
-        });
-    }
-    listDomainQueuedTallies() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getUrls().aggregate([
-                { $match: {
-                        status: EStatus.QUEUED
-                    } },
-                { $group: {
-                        _id: '$domain',
-                        tally: { $sum: 1 }
-                    } }
-            ]);
-            const rows = yield this.listQueryResults(results, isTMongoIdTallyRow);
-            const map = new Map();
-            for (const row of rows) {
-                // eslint-disable-next-line no-underscore-dangle
-                map.set(row._id, row.tally);
-            }
-            return map;
-        });
-    }
-    listPhpErrors() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getUrls().find({
-                status: { $ne: EStatus.ARCHIVED },
-                phpErrors: { $exists: true }
-            }, {});
-            return (yield this.listQueryResults(results, isIUrl))
-                .map((url) => {
-                if (!commonsTypeHasPropertyTArray(url, 'phpErrors', isTPhpError))
-                    throw new Error('Invalid PHP error object');
-                return {
-                    url: url.url,
-                    errors: url.phpErrors
-                };
-            });
-        });
-    }
-    listAspErrors() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getUrls().find({
-                status: { $ne: EStatus.ARCHIVED },
-                aspErrors: { $exists: true }
-            }, {});
-            return (yield this.listQueryResults(results, isIUrl))
-                .map((url) => {
-                if (!commonsTypeHasPropertyTArray(url, 'aspErrors', isTAspError))
-                    throw new Error('Invalid PHP error object');
-                return {
-                    url: url.url,
-                    errors: url.aspErrors
-                };
-            });
-        });
-    }
-    listDone200DomainUrls(domain) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getUrls().find({ $and: [
-                    { domain: domain },
-                    { status: EStatus.DONE },
-                    { statusCode: { $gte: 200 } },
-                    { statusCode: { $lt: 300 } }
-                ] }, {});
-            return (yield this.listQueryResults(results, isIUrl))
-                .map((url) => url.url);
-        });
-    }
-    listDomains() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getDomains().find(
-            // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-            { ip: { $exists: true, $ne: null } }, // this is ok, despite the type objection to null
-            {});
-            // since we're doing $ne: null above, we don't need to strip nulls, as there won't be any
-            return yield this.listQueryResults(results, isTDomain);
-        });
-    }
-    listDomainsByLike(term) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getDomains().find({
-                domain: new RegExp(commonsStringRegexLike(`%${term}%`), 'i')
-            }, {});
-            return (yield this.listQueryResults(results, isTDomain))
-                .map((encoded) => commonsObjectStripNulls(encoded));
-        });
-    }
-    listInboundLinks(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getLinks().find({ outgoing: url }, {});
-            return (yield this.listQueryResults(results, isTLink))
-                .map((link) => link.url);
-        });
-    }
-    listOutboundLinks(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const results = this.getLinks().find({ url: url }, {});
-            return (yield this.listQueryResults(results, isTLink))
-                .map((link) => link.outgoing);
-        });
-    }
-    listImagesBySizeThreshold(size, comparator) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const queries = [
-                { status: EStatus.DONE },
-                { 'headers.content-type': /^image\/(jpeg)/ },
-                { 'headers.content-length': { $exists: true } }
-            ];
-            switch (comparator) {
-                case EComparator.GT:
-                    queries.push({ 'headers.content-length': { $gt: size } });
-                    break;
-                case EComparator.LT:
-                    queries.push({ 'headers.content-length': { $lt: size } });
-                    break;
-                case EComparator.GTE:
-                    queries.push({ 'headers.content-length': { $gte: size } });
-                    break;
-                case EComparator.LTE:
-                    queries.push({ 'headers.content-length': { $lte: size } });
-                    break;
-            }
-            const results = this.getUrls().find({ $and: queries }, {});
-            return (yield this.listQueryResults(results, isIUrl))
-                .map((row) => ({
-                url: row.url,
-                size: commonsTypeAttemptNumber(row['headers']['content-length']) || -1
-            }));
-        });
-    }
-    insertArchiveBatch(documents) {
-        var _a, documents_1, documents_1_1;
-        var _b, e_1, _c, _d;
-        return __awaiter(this, void 0, void 0, function* () {
-            const bulkInsert = this.getArchiveds().initializeUnorderedBulkOp();
-            const insertIds = [];
-            try {
-                for (_a = true, documents_1 = __asyncValues(documents); documents_1_1 = yield documents_1.next(), _b = documents_1_1.done, !_b;) {
-                    _d = documents_1_1.value;
-                    _a = false;
-                    try {
-                        const document = _d;
-                        const typecast = document;
-                        // eslint-disable-next-line no-underscore-dangle
-                        const id = typecast._id;
-                        // Insert without raising an error for duplicates
-                        bulkInsert
-                            .find({ _id: id })
-                            .upsert()
-                            .replaceOne(document);
-                        insertIds.push(id);
-                    }
-                    finally {
-                        _a = true;
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_a && !_b && (_c = documents_1.return)) yield _c.call(documents_1);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            if (insertIds.length > 0)
-                yield bulkInsert.execute();
-            return insertIds;
-        });
-    }
-    deleteUrlsBatch(documents) {
-        var _a, documents_2, documents_2_1;
-        var _b, e_2, _c, _d;
-        return __awaiter(this, void 0, void 0, function* () {
-            // NB, this presumes that the links to and from this have already been removed by the cleaner
-            const bulkDelete = this.getUrls().initializeUnorderedBulkOp();
-            let tally = 0;
-            try {
-                for (_a = true, documents_2 = __asyncValues(documents); documents_2_1 = yield documents_2.next(), _b = documents_2_1.done, !_b;) {
-                    _d = documents_2_1.value;
-                    _a = false;
-                    try {
-                        const document = _d;
-                        const typecast = document;
-                        // eslint-disable-next-line no-underscore-dangle
-                        const id = typecast._id;
-                        bulkDelete
-                            .find({ _id: id })
-                            .deleteOne();
-                        tally++;
-                    }
-                    finally {
-                        _a = true;
-                    }
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (!_a && !_b && (_c = documents_2.return)) yield _c.call(documents_2);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-            if (tally > 0)
-                yield bulkDelete.execute();
-        });
-    }
-    moveArchivedUrlsToArchive(batchSize, limit) {
-        return __awaiter(this, void 0, void 0, function* () {
-            let total = 0;
-            while (true) {
-                commonsOutputDoing(`Inserting up to ${batchSize} urls entries into archiveds`);
-                const existings = this.getUrls().find({ status: EStatus.ARCHIVED }).limit(batchSize);
-                const done = (yield this.insertArchiveBatch(existings))
-                    .map((id) => new mongodb.ObjectId(id));
-                yield existings.close();
-                total += done.length;
-                commonsOutputResult(`${done.length}, ${total} total`);
-                if (done.length === 0)
-                    break;
-                commonsOutputDoing('Deleting previously moved urls');
-                const toDelete = this.getUrls().find({ _id: { $in: done } }).limit(batchSize);
-                yield this.deleteUrlsBatch(toDelete);
-                yield toDelete.close();
-                if (limit !== undefined)
-                    limit -= done.length;
-                commonsOutputSuccess();
-                if (limit !== undefined && limit <= 0)
-                    break;
-            }
-        });
-    }
-    listDuplicateArchivedUrls() {
-        return __awaiter(this, void 0, void 0, function* () {
-            commonsOutputDoing('Building cursor for all archiveds');
-            // we have to use this strange sharded increasing array of sets, as the maximum values a Set (or Map) can hold is 16777216 (2^24)
-            const urls = [];
-            const duplicates = new Set();
-            let tally = 0;
-            while (true) {
-                let subTally = 0;
-                const currentUrls = new Set();
-                try {
-                    const archiveds = this.getArchiveds().find({}, {
-                        sort: { _id: 1 },
-                        skip: tally,
-                        limit: 1000000
-                    });
-                    commonsOutputSuccess();
-                    commonsOutputDoing('Searching for duplicates');
-                    yield archiveds.forEach((archived) => {
-                        try {
-                            let match = false;
-                            for (const s of [...urls, currentUrls]) {
-                                if (s.has(archived.url))
-                                    match = true;
-                            }
-                            if (match) {
-                                duplicates.add(archived.url);
-                            }
-                            else {
-                                currentUrls.add(archived.url);
-                            }
-                            tally++;
-                            subTally++;
-                            if ((tally % 10000) === 0) {
-                                let total = 0;
-                                for (const s of [...urls, currentUrls])
-                                    total += s.size;
-                                commonsOutputProgress(`${tally}, ${total} unique urls, ${duplicates.size} duplicates`);
-                            }
-                        }
-                        catch (e) {
-                            console.log(e);
-                            process.exit(1);
-                        }
-                    });
-                    let batchTotal = 0;
-                    for (const s of [...urls, currentUrls])
-                        batchTotal += s.size;
-                    commonsOutputResult(`${tally}, ${batchTotal} unique urls, ${duplicates.size} duplicates`);
-                    yield archiveds.close();
-                }
-                catch (e2) {
-                    console.log(e2);
-                    process.exit(1);
-                }
-                urls.push(currentUrls);
-                if (subTally === 0)
-                    break;
-            }
-            return Array.from(duplicates.values());
-        });
-    }
-    purgeArchiveDuplicates(url) {
-        var _a, e_3, _b, _c;
-        return __awaiter(this, void 0, void 0, function* () {
-            const archiveds = this.getArchiveds().find({
-                url: url
-            }, {});
-            const bulkDelete = this.getArchiveds().initializeUnorderedBulkOp();
-            let first = true;
-            let tally = 0;
-            try {
-                for (var _d = true, archiveds_1 = __asyncValues(archiveds), archiveds_1_1; archiveds_1_1 = yield archiveds_1.next(), _a = archiveds_1_1.done, !_a;) {
-                    _c = archiveds_1_1.value;
-                    _d = false;
-                    try {
-                        const document = _c;
-                        if (first) {
-                            first = false;
-                            continue;
-                        }
-                        const typecast = document;
-                        // eslint-disable-next-line no-underscore-dangle
-                        const id = typecast._id;
-                        bulkDelete
-                            .find({ _id: id })
-                            .deleteOne();
-                        tally++;
-                    }
-                    finally {
-                        _d = true;
-                    }
-                }
-            }
-            catch (e_3_1) { e_3 = { error: e_3_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = archiveds_1.return)) yield _b.call(archiveds_1);
-                }
-                finally { if (e_3) throw e_3.error; }
-            }
-            if (tally > 0)
-                yield bulkDelete.execute();
-            yield archiveds.close();
-        });
-    }
-}
-//# sourceMappingURL=database.service.js.map