hydra-crawler 2.8.3 → 3.0.1

package/dist/apps/denylist.app.js

@@ -1,132 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsArrayChunk } from 'tscommons-es-core';
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsArgs } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { EList } from '../enums/elist';
-export class DenylistApp extends CommonsApp {
-    constructor() {
-        super('hydra-crawler');
-        this.lists = new Lists();
-    }
-    getAppName() {
-        return 'Hydra - Denylist';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    forward(limit) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            let tally = 0;
-            let found = 0;
-            commonsOutputDoing('Searching for non-DENY urls on the denylist');
-            const urls = [];
-            const result = this.databaseService.getUrls()
-                .find({ status: { $nin: [EStatus.DENY, EStatus.ARCHIVED] } }, {});
-            while (true) {
-                tally++;
-                if ((tally % 100) === 0)
-                    commonsOutputProgress(`${tally}, ${found}`);
-                if (limit !== undefined && tally > limit)
-                    break;
-                const row = yield result.next();
-                if (row === null)
-                    break;
-                if (!this.lists.match(EList.DENY, row.url))
-                    continue;
-                found++;
-                urls.push(row.url);
-            }
-            commonsOutputResult(found);
-            commonsOutputDoing('Marking detected as DENY');
-            const batches = commonsArrayChunk(urls, 100);
-            tally = 0;
-            for (const batch of batches) {
-                yield this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
-                        status: EStatus.DENY
-                    }, $unset: {
-                        ttl: true
-                    } });
-                tally += 100;
-                commonsOutputProgress(tally);
-            }
-            commonsOutputSuccess();
-        });
-    }
-    reverse(limit) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            let tally = 0;
-            let found = 0;
-            commonsOutputDoing('Searching for DENY urls not on the denylist');
-            const urls = [];
-            const result = this.databaseService.getUrls()
-                .find({ status: EStatus.DENY }, {});
-            while (true) {
-                tally++;
-                if ((tally % 100) === 0)
-                    commonsOutputProgress(`${tally}, ${found}`);
-                if (limit !== undefined && tally > limit)
-                    break;
-                const row = yield result.next();
-                if (row === null)
-                    break;
-                if (this.lists.match(EList.DENY, row.url))
-                    continue;
-                found++;
-                urls.push(row.url);
-            }
-            commonsOutputResult(found);
-            commonsOutputDoing('Marking detected as QUEUED');
-            const batches = commonsArrayChunk(urls, 100);
-            tally = 0;
-            for (const batch of batches) {
-                yield this.databaseService.getUrls().updateMany({ url: { $in: batch } }, { $set: {
-                        status: EStatus.QUEUED
-                    }, $unset: {
-                        ttl: true
-                    } });
-                tally += 100;
-                commonsOutputProgress(tally);
-            }
-            commonsOutputSuccess();
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const args = new CommonsArgs();
-            yield this.forward(args.getNumberOrUndefined('limit'));
-            yield this.reverse(args.getNumberOrUndefined('limit'));
-        });
-    }
-}
-//# sourceMappingURL=denylist.app.js.map

package/dist/apps/denylist.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"denylist.app.js","sourceRoot":"","sources":["../../src/apps/denylist.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAMzC,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAIvC,MAAM,OAAO,WAAY,SAAQ,UAAU;IAK1C;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEvB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,CAAC;IAEM,UAAU;QAChB,OAAO,kBAAkB,CAAC;IAC3B,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAE,EAAE,EAAE,EACxD,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAErD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,0BAA0B,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,IAAI;qBACrB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEa,OAAO,CAAC,KAAc;;YACnC,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YAEtB,kBAAkB,CAAC,6CAA6C,CAAC,CAAC;YAElE,MAAM,IAAI,GAAa,EAAE,CAAC;YAE1B,MAAM,MAAM,GAA6B,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACpE,IAAI,CACH,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,EACxB,EAAE,CACH,CAAC;YAEJ,OAAO,IAAI,EAAE;gBACZ,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;oBAAE,qBAAqB,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAEpD,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACnB;YAED,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAE3B,kBAAkB,CAAC,4BAA4B,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAe,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEzD,KAAK,GAAG,CAAC,CAAC;YACV,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAC7C,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EACvB,EAAE,IAAI,EAAE;wBACN,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,EAAE,MAAM,EAAE;wBACT,GAAG,EAAE,IAAI;qBACV,EAAE,CACJ,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,qBAAqB,CAAC,KAAK,CAAC,CAAC;aAC7B;YACD,oBAAoB,EAAE,CAAC;QACxB,CAAC;KAAA;IAEY,GAAG;;YACf,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;YAE5C,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;YACvD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACxD,CAAC;KAAA;CACD"}

package/dist/apps/expire.app.js

@@ -1,63 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsArgs } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Lists } from '../classes/lists';
-import { Expirer } from '../classes/expirer';
-import { Expiry } from '../classes/expiry';
-export class ExpireApp extends CommonsApp {
-    constructor(threshold) {
-        super('hydra-crawler');
-        this.threshold = threshold;
-        this.lists = new Lists();
-        this.expiry = new Expiry();
-    }
-    getAppName() {
-        return 'Hydra - Expire';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    addToExpiry(expiries) {
-        this.expiry.add(expiries);
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const args = new CommonsArgs();
-            const expirer = new Expirer(this.expiry, this.databaseService);
-            if (this.threshold !== undefined) {
-                yield expirer.expireFixed(this.threshold);
-            }
-            else {
-                yield expirer.expire(args.getNumberOrUndefined('limit'));
-            }
-        });
-    }
-}
-//# sourceMappingURL=expire.app.js.map

package/dist/apps/expire.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"expire.app.js","sourceRoot":"","sources":["../../src/apps/expire.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAW3C,MAAM,OAAO,SAAU,SAAQ,UAAU;IAKxC,YACU,SAAyB;QAElC,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,cAAS,GAAT,SAAS,CAAgB;QAIlC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC5B,CAAC;IAEM,UAAU;QAChB,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,IAAI,GAAgB,IAAI,WAAW,EAAE,CAAC;YAE5C,MAAM,OAAO,GAAY,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YAExE,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE;gBACjC,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aAC1C;iBAAM;gBACN,MAAM,OAAO,CAAC,MAAM,CAClB,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CACnC,CAAC;aACF;QACF,CAAC;KAAA;CACD"}

package/dist/apps/export-domain-urls.js

@@ -1,99 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { EStatus } from 'hydra-crawler-ts-assets';
-import { commonsOutputDoing, commonsOutputProgress, commonsOutputResult, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-// export the QUEUED, DONE and FAILED urls for importing into another instance
-export class ExportDomainUrlsApp extends CommonsApp {
-    constructor(domain) {
-        super('hydra-crawler');
-        this.domain = domain;
-    }
-    getAppName() {
-        return 'Hydra - Export Domain Urls';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            yield _super.init.call(this);
-        });
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set');
-            const validOnly = this.getArgs().hasAttribute('valid-only');
-            const pagesOnly = this.getArgs().hasAttribute('pages-only');
-            const mergeHttp = this.getArgs().hasAttribute('merge-http');
-            const matchPipeline = [
-                { $match: { domain: this.domain } }
-            ];
-            if (validOnly) {
-                matchPipeline.push({ $match: {
-                        status: { $in: [EStatus.DONE, EStatus.QUEUED, EStatus.ACTIVE] },
-                        statusCode: { $in: [200, 204] }
-                    } });
-            }
-            if (pagesOnly) {
-                matchPipeline.push(...[
-                    { $match: {
-                            url: { $not: /[?]/ }
-                        } },
-                    { $match: {
-                            url: { $not: /(gif|jpeg|jpg|png|webp|avif)$/i }
-                        } }
-                ]);
-            }
-            const cursor = this.databaseService.getUrls()
-                .aggregate([
-                ...matchPipeline,
-                { $project: { _id: false, url: true } }
-            ]);
-            // safer to do this directly rather than a call to listQueryResults
-            // ditto not using CommonsFile
-            commonsOutputDoing(`Exporting URLs for domain ${this.domain}`);
-            const urls = [];
-            while (true) {
-                if (urls.length % 1000 === 0)
-                    commonsOutputProgress(urls.length);
-                const row = yield cursor.next();
-                if (row === null)
-                    break;
-                let url = row.url
-                    .trim()
-                    .replace(/[?].*$/, '')
-                    .replace(/\/index\.(htm|html)$/i, '/')
-                    .replace(/\/$/, '')
-                    .trim();
-                if (mergeHttp) {
-                    url = url.replace(/^http(s?):/, '');
-                }
-                if (!urls.includes(url))
-                    urls.push(url);
-            }
-            commonsOutputResult(urls.length);
-            urls.sort();
-            for (const url of urls) {
-                console.log(url);
-            }
-        });
-    }
-}
-//# sourceMappingURL=export-domain-urls.js.map

package/dist/apps/export-domain-urls.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"export-domain-urls.js","sourceRoot":"","sources":["../../src/apps/export-domain-urls.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMhD,8EAA8E;AAE9E,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAGlD,YACU,MAAc;QAEvB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,WAAM,GAAN,MAAM,CAAQ;IAGxB,CAAC;IAEM,UAAU;QAChB,OAAO,4BAA4B,CAAC;IACrC,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,MAAM,OAAM,IAAI,WAAE,CAAC;QACpB,CAAC;KAAA;IAEY,GAAG;;YACf,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAEhF,MAAM,SAAS,GAAY,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,SAAS,GAAY,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,SAAS,GAAY,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YAErE,MAAM,aAAa,GAA4B;gBAC7C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;aACpC,CAAC;YACF,IAAI,SAAS,EAAE;gBACd,aAAa,CAAC,IAAI,CAChB,EAAE,MAAM,EAAE;wBACR,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE;wBACjE,UAAU,EAAE,EAAE,GAAG,EAAE,CAAE,GAAG,EAAE,GAAG,CAAE,EAAE;qBAClC,EAAE,CACJ,CAAC;aACF;YACD,IAAI,SAAS,EAAE;gBACd,aAAa,CAAC,IAAI,CAAC,GAAG;oBACpB,EAAE,MAAM,EAAE;4BACR,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;yBACrB,EAAE;oBACH,EAAE,MAAM,EAAE;4BACR,GAAG,EAAE,EAAE,IAAI,EAAE,gCAAgC,EAAE;yBAChD,EAAE;iBACJ,CAAC,CAAC;aACH;YAED,MAAM,MAAM,GAA+C,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE;iBACtF,SAAS,CAAC;gBACT,GAAG,aAAa;gBAChB,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;aACxC,CAAC,CAAC;YAEL,mEAAmE;YACnE,8BAA8B;YAE9B,kBAAkB,CAAC,6BAA6B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAE/D,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,OAAO,IAAI,EAAE;gBACZ,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC;oBAAE,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEjE,MAAM,GAAG,GAAyB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBACtD,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,IAAI,GAAG,GAAW,GAAG,CAAC,GAAG;qBACtB,IAAI,EAAE;qBACN,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;qBACrB,OAAO,CAAC,uBAAuB,EAAE,GAAG,CAAC;qBACrC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;qBAClB,IAAI,EAAE,CAAC;gBAEV,IAAI,SAAS,EAAE;oBACd,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;iBACpC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACxC;YACD,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACvB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;aACjB;QACF,CAAC;KAAA;CACD"}

package/dist/apps/extract-text.app.js

@@ -1,55 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import * as http from 'http';
-import * as https from 'https';
-import { commonsOutputDoing, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { CommonsApp } from 'nodecommons-es-app';
-import { Crawler } from '../classes/crawler';
-import { TextParser } from '../parsers/text.parser';
-// extract the text from the HTML of a URL
-class TextParserInstance extends TextParser {
-    // eslint-disable-next-line @typescript-eslint/require-await
-    parseText(_database, text) {
-        return __awaiter(this, void 0, void 0, function* () {
-            console.log(text);
-        });
-    }
-}
-export class ExtractTextApp extends CommonsApp {
-    constructor(url) {
-        super('hydra-crawler');
-        this.url = url;
-    }
-    getAppName() {
-        return 'Hydra - Extract Text';
-    }
-    run() {
-        return __awaiter(this, void 0, void 0, function* () {
-            commonsOutputDoing(`Downloading from URL at ${this.url}`);
-            const whatwg = new URL(this.url);
-            let handler;
-            switch (whatwg.protocol) {
-                case 'http:':
-                    handler = http;
-                    break;
-                case 'https:':
-                    handler = https;
-                    break;
-                default:
-                    throw new Error(`unable to handle protocol ${whatwg.protocol}`);
-            }
-            const outcome = yield Crawler.request(handler, this.url, 10000, 4096000);
-            commonsOutputSuccess();
-            const parser = new TextParserInstance(outcome);
-            yield parser.parse(undefined);
-        });
-    }
-}
-//# sourceMappingURL=extract-text.app.js.map

package/dist/apps/extract-text.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"extract-text.app.js","sourceRoot":"","sources":["../../src/apps/extract-text.app.ts"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAOpD,0CAA0C;AAE1C,MAAM,kBAAmB,SAAQ,UAAyB;IACzD,4DAA4D;IAC5C,SAAS,CACvB,SAA0B,EAC1B,IAAY;;YAEb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;KAAA;CACD;AAED,MAAM,OAAO,cAAe,SAAQ,UAAU;IAC7C,YACU,GAAW;QAEpB,KAAK,CAAC,eAAe,CAAC,CAAC;QAFd,QAAG,GAAH,GAAG,CAAQ;IAGrB,CAAC;IAEM,UAAU;QAChB,OAAO,sBAAsB,CAAC;IAC/B,CAAC;IAEY,GAAG;;YACf,kBAAkB,CAAC,2BAA2B,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEtC,IAAI,OAA2C,CAAC;YAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE;gBACxB,KAAK,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBACpC,KAAK,QAAQ;oBAAE,OAAO,GAAG,KAAK,CAAC;oBAAC,MAAM;gBACtC;oBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjE;YAED,MAAM,OAAO,GAAoB,MAAM,OAAO,CAAC,OAAO,CACpD,OAAO,EACP,IAAI,CAAC,GAAG,EACR,KAAK,EACL,OAAO,CACR,CAAC;YACF,oBAAoB,EAAE,CAAC;YAEvB,MAAM,MAAM,GAAuB,IAAI,kBAAkB,CACvD,OAAO,CACR,CAAC;YACF,MAAM,MAAM,CAAC,KAAK,CAAC,SAAuC,CAAC,CAAC;QAC7D,CAAC;KAAA;CACD"}

package/dist/apps/hydra.app.js

@@ -1,218 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { commonsStringSplitString, commonsTypeIsTKeyObject } from 'tscommons-es-core';
-import { commonsOutputDebug, commonsOutputDoing, commonsOutputInfo, commonsOutputSuccess } from 'nodecommons-es-cli';
-import { commonsExpressBuildDefaultServer, isICommonsExpressConfig } from 'nodecommons-es-express';
-import { CommonsSocketIoApp } from 'nodecommons-es-app-socket-io';
-import { AutocompleteApi } from '../apis/autocomplete.api';
-import { BugsApi } from '../apis/bugs.api';
-import { CrawlApi } from '../apis/crawl.api';
-import { DomainsApi } from '../apis/domains.api';
-import { ImagesApi } from '../apis/images.api';
-import { StatisticsApi } from '../apis/statistics.api';
-import { TestApi } from '../apis/test.api';
-import { UrlsApi } from '../apis/urls.api';
-import { Lists } from '../classes/lists';
-import { Expiry } from '../classes/expiry';
-import { Tracker } from '../classes/tracker';
-import { Crawler } from '../classes/crawler';
-import { SocketIoServer } from '../servers/socket-io.server';
-import { CrawlServer } from '../servers/crawl.server';
-import { MaintenanceServer } from '../servers/maintenance.server';
-import { ServerParser } from '../parsers/server.parser';
-import { HyperlinksParser } from '../parsers/hyperlinks.parser';
-import { ImageTagsParser } from '../parsers/image-tags.parser';
-import { JpegParser } from '../parsers/jpeg.parser';
-import { SpellingParser } from '../parsers/spelling.parser';
-import { BadWordsParser } from '../parsers/bad-words.parser';
-import { PhpErrorParser } from '../parsers/php-error.parser';
-import { AspErrorParser } from '../parsers/asp-error.parser';
-import { ComplexEnglishParser } from '../parsers/complex-english.parser';
-import { AccessibilityMetricsParser } from '../parsers/accessibility-metrics.parser';
-// import { OffenceParser } from '../parsers/offence.parser';
-// import { InterestParser } from '../parsers/interest.parser';
-import { LlamaGuardParser } from '../parsers/llama-guard.parser';
-import { isIParserConfig } from '../interfaces/iparser-config';
-import { isTHydraConfig } from '../types/thydra-config';
-import { isTCrawlConfig } from '../types/tcrawl-config';
-import { isTRobotsConfig } from '../types/trobots-config';
-export class HydraApp extends CommonsSocketIoApp {
-    constructor() {
-        super('hydra-crawler');
-        this.maintenanceTimes = [];
-        this.lists = new Lists();
-        this.expiry = new Expiry();
-        const hydraConfig = this.getConfigArea('hydra');
-        if (!isTHydraConfig(hydraConfig))
-            throw new Error('Hydra config is not valid');
-        this.hydraConfig = hydraConfig;
-        const crawlConfig = this.getConfigArea('crawl');
-        if (!isTCrawlConfig(crawlConfig))
-            throw new Error('Crawl config is not valid');
-        this.crawlConfig = crawlConfig;
-        const parsersConfig = this.getConfigArea('parsers');
-        if (!commonsTypeIsTKeyObject(parsersConfig, isIParserConfig))
-            throw new Error('Parsers config is not valid');
-        this.parsersConfig = parsersConfig;
-        const robotsConfig = this.getConfigArea('robots');
-        if (!isTRobotsConfig(robotsConfig))
-            throw new Error('Robots config is not valid');
-        this.robotsConfig = robotsConfig;
-    }
-    buildHttpServer() {
-        const expressConfig = this.getConfigArea('express');
-        if (!isICommonsExpressConfig(expressConfig))
-            throw new Error('Invalid express config');
-        return commonsExpressBuildDefaultServer(this.httpConfig.port, expressConfig.bodyParserLimit);
-    }
-    buildSocketIoServer(expressServer, expressConfig) {
-        return new SocketIoServer(expressServer, expressConfig);
-    }
-    getAppName() {
-        return 'Hydra';
-    }
-    setDatabaseService(databaseService) {
-        this.databaseService = databaseService;
-    }
-    addToList(list, entries) {
-        this.lists.add(list, entries);
-    }
-    addToExpiry(expiries) {
-        this.expiry.add(expiries);
-    }
-    setMaintenanceSchedule(times) {
-        this.maintenanceTimes = times;
-    }
-    init() {
-        const _super = Object.create(null, {
-            init: { get: () => super.init }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.databaseService)
-                throw new Error('Database service has not been set yet');
-            this.install((restServer, path) => {
-                if (!this.databaseService)
-                    throw new Error('Database service has not been defined yet');
-                commonsOutputDoing('Installing auto-complete API');
-                new AutocompleteApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing bugs API');
-                new BugsApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing crawl API');
-                new CrawlApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing domains API');
-                new DomainsApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing images API');
-                new ImagesApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing statistics API');
-                new StatisticsApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing test API');
-                new TestApi(restServer, path);
-                commonsOutputSuccess();
-                commonsOutputDoing('Installing urls API');
-                new UrlsApi(restServer, this.databaseService, path);
-                commonsOutputSuccess();
-            });
-            yield _super.init.call(this);
-            const socketIoServer = this.getSocketIoServer();
-            if (!socketIoServer)
-                throw new Error('SocketIo server has not been set yet');
-            this.tracker = new Tracker(this.databaseService, socketIoServer);
-            this.crawlServer = new CrawlServer(this.databaseService, this.hydraConfig, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.lists, this.tracker);
-            commonsOutputDoing('Connecting to database');
-            yield this.databaseService.init();
-            commonsOutputSuccess();
-            for (const parser of [
-                ServerParser,
-                HyperlinksParser,
-                ImageTagsParser,
-                JpegParser,
-                SpellingParser,
-                BadWordsParser,
-                PhpErrorParser,
-                AspErrorParser,
-                ComplexEnglishParser,
-                AccessibilityMetricsParser,
-                // OffenceParser,
-                // InterestParser,
-                LlamaGuardParser
-            ]) {
-                commonsOutputDebug(`Installing parser ${parser.name}`);
-                this.crawlServer.addParser(parser);
-                yield this.databaseService.initParser(parser);
-            }
-            const maintenanceHardLimit = this.getArgs().getNumberOrUndefined('maintenance-hard-limit');
-            if (maintenanceHardLimit)
-                commonsOutputDebug(`Setting maintenance hard limit to ${maintenanceHardLimit}`);
-            this.maintenanceServer = new MaintenanceServer(this.maintenanceTimes, this.expiry, this.lists, this.databaseService, this.crawlServer, maintenanceHardLimit // to prevent running out of heap on very large collections
-            );
-        });
-    }
-    listening() {
-        void (() => __awaiter(this, void 0, void 0, function* () {
-            if (!this.crawlServer)
-                throw new Error('No crawl server instantiated! This should not be possible');
-            yield this.crawlServer.start();
-            this.abort();
-        }))();
-    }
-    run() {
-        const _super = Object.create(null, {
-            run: { get: () => super.run }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            const singleUrl = this.getArgs().getStringOrUndefined('single-url');
-            if (singleUrl) {
-                if (!this.databaseService)
-                    throw new Error('Database service has not been set yet');
-                if (!this.crawlServer)
-                    throw new Error('Crawl server has not been set yet');
-                const whatwg = new URL(singleUrl);
-                const crawler = new Crawler(whatwg.hostname, this.databaseService, this.crawlConfig, this.parsersConfig, this.robotsConfig, this.crawlServer.listParsers(), this.lists);
-                commonsOutputInfo(`Running single crawl of: ${singleUrl}`);
-                const outcome = yield crawler.fetch(singleUrl, false);
-                commonsOutputInfo(`HTTP status code outcome was ${outcome}`);
-                return;
-            }
-            const domainRestrictTo = this.getArgs().getStringOrUndefined('domain-restrict-to');
-            if (domainRestrictTo) {
-                const domains = commonsStringSplitString(domainRestrictTo, ',', true, true);
-                commonsOutputDebug(`Restricting crawl domains to: ${domains.join(',')}`);
-                if (!this.crawlServer)
-                    throw new Error('CrawlServer has not been defined! This should not be possible');
-                this.crawlServer.restrictTo = domains;
-            }
-            if (!this.tracker)
-                throw new Error('Tracker has not been defined! This should not be possible');
-            if (!this.maintenanceServer)
-                throw new Error('Maintenance server has not been defined! This should not be possible');
-            this.maintenanceServer.start();
-            void this.tracker.start();
-            yield _super.run.call(this);
-        });
-    }
-    shutdown() {
-        const _super = Object.create(null, {
-            shutdown: { get: () => super.shutdown }
-        });
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.crawlServer)
-                throw new Error('No crawl server instantiated! This should not be possible');
-            yield this.crawlServer.shutdown();
-            yield _super.shutdown.call(this);
-        });
-    }
-}
-//# sourceMappingURL=hydra.app.js.map

package/dist/apps/hydra.app.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hydra.app.js","sourceRoot":"","sources":["../../src/apps/hydra.app.ts"],"names":[],"mappings":";;;;;;;;;AAEA,OAAO,EAAE,wBAAwB,EAAE,uBAAuB,EAAc,MAAM,mBAAmB,CAAC;AAGlG,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACrH,OAAO,EAAE,gCAAgC,EAA0D,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAG3J,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAIlE,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,0BAA0B,EAAE,MAAM,yCAAyC,CAAC;AACrF,6DAA6D;AAC7D,+DAA+D;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAEjE,OAAO,EAAiB,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAI9E,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAgB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACtE,OAAO,EAAiB,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAUzE,MAAM,OAAO,QAAS,SAAQ,kBAAkC;IAkB/D;QACC,KAAK,CAAC,eAAe,CAAC,CAAC;QANhB,qBAAgB,GAA2B,EAAE,CAAC;QAQrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,WAAW,GAAY,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/E,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAE/B,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAgB,aAAa,EAAE,eAAe,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC5H,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QAEnC,MAAM,YAAY,GAAY,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IAEkB,eAAe;QACjC,MAAM,aAAa,GAAY,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEvF,OAAO,gCAAgC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IAC9F,CAAC;IAES,mBAAmB,CAC3B,aAAyC,EACzC,aAAoC;QAErC,OAAO,IAAI,cAAc,CACvB,aAAa,EACb,aAAa,CACd,CAAC;IACH,CAAC;IAEM,UAAU;QAChB,OAAO,OAAO,CAAC;IAChB,CAAC;IAEM,kBAAkB,CACvB,eAAgC;QAEjC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACxC,CAAC;IAEM,SAAS,CACd,IAAW,EACX,OAAiB;QAElB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;IAEM,WAAW,CAChB,QAAmB;QAEpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IAEM,sBAAsB,CAC3B,KAA6B;QAE9B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEY,IAAI;;;;;YAChB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAEpF,IAAI,CAAC,OAAO,CACV,CAAC,UAAgF,EAAE,IAAY,EAAQ,EAAE;gBACxG,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAExF,kBAAkB,CAAC,8BAA8B,CAAC,CAAC;gBACnD,IAAI,eAAe,CACjB,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,sBAAsB,CAAC,CAAC;gBAC3C,IAAI,QAAQ,CACV,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;gBAC7C,IAAI,UAAU,CACZ,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,uBAAuB,CAAC,CAAC;gBAC5C,IAAI,SAAS,CACX,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,2BAA2B,CAAC,CAAC;gBAChD,IAAI,aAAa,CACf,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;gBAEvB,kBAAkB,CAAC,qBAAqB,CAAC,CAAC;gBAC1C,IAAI,OAAO,CACT,UAAU,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CACL,CAAC;gBACF,oBAAoB,EAAE,CAAC;YACxB,CAAC,CACF,CAAC;YAEF,MAAM,OAAM,IAAI,WAAE,CAAC;YAEnB,MAAM,cAAc,GAA6B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC1E,IAAI,CAAC,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE7E,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,IAAI,CAAC,eAAe,EACpB,cAAc,CACf,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAChC,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,CACb,CAAC;YAEF,kBAAkB,CAAC,wBAAwB,CAAC,CAAC;YAC7C,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YAClC,oBAAoB,EAAE,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI;gBACnB,YAAY;gBACZ,gBAAgB;gBAChB,eAAe;gBACf,UAAU;gBACV,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,cAAc;gBACd,oBAAoB;gBACpB,0BAA0B;gBAC1B,iBAAiB;gBACjB,kBAAkB;gBAClB,gBAAgB;aACjB,EAAE;gBACF,kBAAkB,CAAC,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvD,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC9C;YAED,MAAM,oBAAoB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,wBAAwB,CAAC,CAAC;YAC7G,IAAI,oBAAoB;gBAAE,kBAAkB,CAAC,qCAAqC,oBAAoB,EAAE,CAAC,CAAC;YAE1G,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAC5C,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,oBAAoB,CAAC,2DAA2D;aACjF,CAAC;QACH,CAAC;KAAA;IAES,SAAS;QAClB,KAAK,CAAC,GAAwB,EAAE;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAA,CAAC,EAAE,CAAC;IACN,CAAC;IAEY,GAAG;;;;;YACf,MAAM,SAAS,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,SAAS,EAAE;gBACd,IAAI,CAAC,IAAI,CAAC,eAAe;oBAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAE5E,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvC,MAAM,OAAO,GAAY,IAAI,OAAO,CAClC,MAAM,CAAC,QAAQ,EACf,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAC9B,IAAI,CAAC,KAAK,CACX,CAAC;gBAEF,iBAAiB,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC3D,MAAM,OAAO,GAAW,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC9D,iBAAiB,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAC;gBAE7D,OAAO;aACP;YAED,MAAM,gBAAgB,GAAqB,IAAI,CAAC,OAAO,EAAE,CAAC,oBAAoB,CAAC,oBAAoB,CAAC,CAAC;YACrG,IAAI,gBAAgB,EAAE;gBACrB,MAAM,OAAO,GAAa,wBAAwB,CAAC,gBAAgB,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;gBACtF,kBAAkB,CAAC,iCAAiC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAEzE,IAAI,CAAC,IAAI,CAAC,WAAW;oBAAE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBACxG,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC;aACtC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAChG,IAAI,CAAC,IAAI,CAAC,iBAAiB;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAErH,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAE1B,MAAM,OAAM,GAAG,WAAE,CAAC;QACnB,CAAC;KAAA;IAEe,QAAQ;;;;;YACvB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAEpG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAElC,MAAM,OAAM,QAAQ,WAAE,CAAC;QACxB,CAAC;KAAA;CACD"}