hydra-crawler 2.8.3 → 3.0.1

package/dist/classes/crawler.mjs

@@ -0,0 +1,418 @@
+import { URL } from 'url';
+import * as http from 'http';
+import * as https from 'https';
+import { commonsMapToObject, commonsTypeHasPropertyString } from 'tscommons-esm-core';
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-esm-async';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogAlert, commonsLogDebug, commonsLogError, commonsLogInfo } from 'nodecommons-esm-log';
+import { commonsGracefulAbortAddCallback, commonsGracefulAbortRemoveCallback } from 'nodecommons-esm-process';
+import { commonsHashMd5 } from 'nodecommons-esm-security';
+import { Robot } from '../classes/robot.mjs';
+import { Random } from '../helpers/random.mjs';
+import { EList } from '../enums/elist.mjs';
+import { Dns } from './dns.mjs';
+export class Crawler {
+    domain;
+    database;
+    crawlConfig;
+    parsersConfig;
+    robotsConfig;
+    parsers;
+    lists;
+    tracker;
+    static applyMasqueradeHeaders(request) {
+        request.setHeader('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8');
+        request.setHeader('Accept-Encoding', 'identity');
+        request.setHeader('Connection', 'close');
+        request.setHeader('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/68.0.3440.106 Chrome/68.0.3440.106 Safari/537.36 (UoM Hydra)');
+    }
+    static request(handler, url, connectTimeout, maxFileSize, tracker) {
+        return new Promise((resolve, reject) => {
+            const start = new Date().getTime();
+            let exceeded = false;
+            let completed = false;
+            let timedOut = false;
+            const timedOutMessage = `Timeout on crawl of ${url}`;
+            const secondaryTimeoutId = `crawler_${url}_secondary_timeout`;
+            commonsAsyncTimeout(connectTimeout + 60000, secondaryTimeoutId)
+                .then(() => {
+                if (timedOut || completed)
+                    return;
+                commonsLogDebug(`Secondary timeout for ${url}`);
+                timedOut = true;
+                reject(new Error(timedOutMessage));
+                return;
+            })
+                .catch((e) => {
+                if (e.message === 'abortTimeout called')
+                    return;
+                commonsLogDebug('debug position 10');
+                console.log(e);
+                throw e;
+            });
+            const req = handler.request(url, (res) => {
+                // likely unnecessary due to doing it below, but doesn't hurt to do it twice
+                req.setTimeout(connectTimeout);
+                res.setTimeout(connectTimeout);
+                const end = new Date().getTime();
+                const latency = end - start;
+                // always check for a body, even in 4xx 3xx etc.
+                let read = 0;
+                const data = [];
+                res.on('data', (chunk) => {
+                    if (read > maxFileSize) {
+                        if (!exceeded) {
+                            exceeded = true;
+                            res.destroy();
+                            commonsLogAlert(`Exceeded maxFileSize for URL: ${url}`);
+                        }
+                        return;
+                    }
+                    data.push(chunk);
+                    read += chunk.length;
+                    if (tracker)
+                        tracker.bandwidth(chunk.length);
+                });
+                res.on('end', () => {
+                    if (timedOut || completed)
+                        return;
+                    completed = true;
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
+                    resolve({
+                        latency: latency,
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+                        ip: res['remoteIp'],
+                        statusCode: res.statusCode || 599,
+                        headers: res.headers,
+                        data: Buffer.concat(data),
+                        exceeded: exceeded
+                    });
+                });
+                // not sure if this event actually occurs, as opposed to req error below
+                res.on('error', (err) => {
+                    if (timedOut || completed)
+                        return;
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
+                    reject(err);
+                });
+            });
+            req.setTimeout(connectTimeout);
+            Crawler.applyMasqueradeHeaders(req);
+            req.on('timeout', () => {
+                if (timedOut || completed)
+                    return;
+                timedOut = true;
+                req.destroy();
+                commonsAsyncAbortTimeout(secondaryTimeoutId);
+                reject(new Error(timedOutMessage));
+            });
+            req.on('response', (res) => {
+                res['remoteIp'] = res.connection.remoteAddress;
+            });
+            req.on('error', (err) => {
+                if (timedOut || completed)
+                    return;
+                commonsAsyncAbortTimeout(secondaryTimeoutId);
+                reject(err);
+            });
+            req.on('socket', (socket) => {
+                socket.setTimeout(connectTimeout);
+                socket.on('timeout', () => {
+                    if (timedOut || completed)
+                        return;
+                    commonsAsyncAbortTimeout(secondaryTimeoutId);
+                    timedOut = true;
+                    req.destroy();
+                    reject(new Error(timedOutMessage));
+                });
+            });
+            req.end();
+        });
+    }
+    static pruneHeaders(headers, keepHeaders) {
+        const map = new Map();
+        const keys = [];
+        for (const k of keepHeaders) {
+            const kt = k.split(':');
+            keys.push({ key: kt[0], type: kt[1] });
+        }
+        for (const key of keys) {
+            if (!Object.keys(headers).includes(key.key))
+                continue;
+            if (!commonsTypeHasPropertyString(headers, key.key))
+                continue;
+            const value = headers[key.key];
+            switch (key.type) {
+                case 'string':
+                    map.set(key.key, value.toString());
+                    break;
+                case 'number':
+                    map.set(key.key, parseInt(value, 10));
+                    break;
+                case 'date':
+                    map.set(key.key, new Date(value));
+                    break;
+                default: throw new Error('unknown header key type');
+            }
+        }
+        return map;
+    }
+    isPaused = false;
+    isAborted = false;
+    constructor(domain, database, crawlConfig, parsersConfig, robotsConfig, parsers, lists, tracker) {
+        this.domain = domain;
+        this.database = database;
+        this.crawlConfig = crawlConfig;
+        this.parsersConfig = parsersConfig;
+        this.robotsConfig = robotsConfig;
+        this.parsers = parsers;
+        this.lists = lists;
+        this.tracker = tracker;
+    }
+    abort() {
+        this.isAborted = true;
+        commonsAsyncAbortTimeout(`crawl_${this.domain}`);
+    }
+    pause() {
+        commonsLogAlert(`Pausing crawler for ${this.domain}`);
+        this.isPaused = true;
+    }
+    resume() {
+        commonsLogInfo(`Resuming crawler for ${this.domain}`);
+        this.isPaused = false;
+    }
+    async fetch(url, setDomainIp) {
+        commonsLogInfo(`Fetching ${url}`);
+        const whatwg = new URL(url);
+        let handler;
+        switch (whatwg.protocol) {
+            case 'http:':
+                handler = http;
+                break;
+            case 'https:':
+                handler = https;
+                break;
+            default:
+                throw new Error(`unable to handle protocol ${whatwg.protocol}`);
+        }
+        let outcome;
+        // eslint-disable-next-line no-useless-catch
+        try {
+            outcome = await Crawler.request(handler, url, this.crawlConfig.connectTimeout, this.crawlConfig.maxFileSize, this.tracker);
+        }
+        catch (ex) {
+            //if (!/^Timeout on crawl of/.test(ex.message)) console.error(ex);
+            throw ex;
+        }
+        if (setDomainIp) {
+            await this.database.domain(whatwg.hostname, outcome.ip);
+            if (this.tracker)
+                this.tracker.domain();
+        }
+        await this.database.setStatusCode(url, outcome.statusCode);
+        const headers = Crawler.pruneHeaders(outcome.headers, this.crawlConfig.keepHeaders);
+        if (!headers.has('content-length') && outcome.data !== undefined)
+            headers.set('content-length', outcome.data.length);
+        await this.database.setHeaders(url, commonsMapToObject(headers));
+        const outgoing = [];
+        if (outcome.statusCode >= 300
+            && outcome.statusCode < 400
+            && headers.has('location')
+            && this.lists.match(EList.ALLOW, url) // only queue redirects for allowlist urls
+        ) {
+            // redirect codes
+            const location = new URL(headers.get('location'), url);
+            if (location.protocol.match(/^http(s?):$/)) {
+                const added = await this.database.queue(location.toString(), this.lists.match(EList.DENY, location.toString()));
+                if (added && this.tracker)
+                    this.tracker.delta(this.lists.match(EList.DENY, location.toString()) ? EStatus.DENY : EStatus.QUEUED, 1);
+                if (location.toString() !== url)
+                    outgoing.push(location.toString());
+            }
+        }
+        if (outcome.data) {
+            const hash = commonsHashMd5(outcome.data);
+            const existing = await this.database.getHash(url);
+            if (hash !== existing)
+                await this.database.setHash(url, hash);
+        }
+        if (outcome.data && outcome.data.length && headers.has('content-type')) {
+            const contentType = headers.get('content-type');
+            const isAllow = this.lists.match(EList.ALLOW, url);
+            for (const ctor of this.parsers) {
+                const parser = new ctor(url, outcome, this.parsersConfig);
+                if (!parser.isEnabled())
+                    continue;
+                if (!parser.supports(contentType, isAllow))
+                    continue;
+                try {
+                    await parser.parse(this.database);
+                }
+                catch (ex) {
+                    commonsLogDebug('debug position 3');
+                    console.log(ex);
+                }
+                if (isAllow) {
+                    try {
+                        const links = await parser.links();
+                        if (links.length === 0)
+                            continue;
+                        for (const link of links) {
+                            if (link === url)
+                                continue;
+                            try {
+                                outgoing.push(link);
+                            }
+                            catch (_ex) { /* do nothing */ }
+                        }
+                    }
+                    catch (ex) {
+                        commonsLogDebug('debug position 2');
+                        console.log(ex);
+                    }
+                }
+            }
+        }
+        for (const link of outgoing) {
+            const added = await this.database.queue(link, this.lists.match(EList.DENY, link));
+            if (added && this.tracker)
+                this.tracker.delta(this.lists.match(EList.DENY, link) ? EStatus.DENY : EStatus.QUEUED, 1);
+        }
+        await this.database.link(url, outgoing);
+        if (this.tracker)
+            this.tracker.link(outgoing.length);
+        return outcome.statusCode;
+    }
+    async crawl() {
+        const gracefulAbortId = commonsGracefulAbortAddCallback(() => {
+            commonsLogAlert(`SIGINT abort flag is set. Aborting crawler ${this.domain}.`);
+            this.abort();
+        });
+        try {
+            try {
+                await Dns.resolve(this.domain, this.crawlConfig.connectTimeout);
+            }
+            catch (ex) {
+                if (ex.message === 'DNS timeout' && !this.crawlConfig.treatTimeoutDnsAsDead) {
+                    commonsLogError(`DNS timed out for ${this.domain}. Will allow reattempts`);
+                    return;
+                }
+                commonsLogError(`Domain ${this.domain} is dead.`);
+                await this.database.markDead(this.domain);
+                return;
+            }
+            const robots = new Robot(this.domain, this.crawlConfig, this.robotsConfig, this.tracker);
+            await robots.load();
+            let tally = 0;
+            while (true) {
+                if (this.isAborted)
+                    break;
+                if (this.isPaused) {
+                    await commonsAsyncTimeout(1000);
+                    continue;
+                }
+                const url = await this.database.next(this.domain);
+                if (!url) {
+                    commonsLogInfo(`Crawl for ${this.domain} completed.`);
+                    break;
+                }
+                await this.database.unsetFailReason(url);
+                if (this.tracker)
+                    this.tracker.delta(EStatus.QUEUED, -1);
+                if (this.lists.match(EList.DENY, url)) {
+                    await this.database.setStatus(url, EStatus.DENY);
+                    await this.database.unsetTtl(url);
+                    if (this.tracker)
+                        this.tracker.delta(EStatus.DENY, 1);
+                    // To get here, a url was queued but is now denylisted.
+                    // It may have had outgoings, so remove any that were there.
+                    await this.database.link(url, []);
+                    continue;
+                }
+                if (robots.isDisallowed(url)) {
+                    await this.database.setStatus(url, EStatus.DISALLOWED);
+                    await this.database.unsetTtl(url);
+                    if (this.tracker)
+                        this.tracker.delta(EStatus.DISALLOWED, 1);
+                    // To get here, a url was queued but is now disallowed.
+                    // It may have had outgoings, so remove any that were there.
+                    await this.database.link(url, []);
+                    continue;
+                }
+                if (!await this.database.setStatus(url, EStatus.ACTIVE)) {
+                    commonsLogError(`Unable to set status to active for ${url}. Aborting crawler.`);
+                    break;
+                }
+                if (this.tracker)
+                    this.tracker.delta(EStatus.ACTIVE, 1);
+                if (tally > 0) {
+                    try {
+                        const timeout = Random.randomise(this.crawlConfig.betweenFetchDelay, this.crawlConfig);
+                        await commonsAsyncTimeout(timeout, `crawl_${this.domain}`);
+                        while (this.isPaused && !this.isAborted) {
+                            await commonsAsyncTimeout(1000);
+                        }
+                    }
+                    catch (_ex) { /* ignore */ }
+                }
+                if (this.isAborted)
+                    break;
+                if (this.tracker)
+                    await this.tracker.fetching(url);
+                try {
+                    const statusCode = await this.fetch(url, tally === 0);
+                    if (this.tracker)
+                        await this.tracker.done(url, statusCode);
+                    await this.database.setStatus(url, EStatus.DONE);
+                    await this.database.unsetTtl(url);
+                    if (this.tracker)
+                        this.tracker.delta(EStatus.DONE, 1);
+                }
+                catch (ex) {
+                    if (this.tracker)
+                        await this.tracker.failed(url);
+                    const ttl = await this.database.getTtl(url) || 0;
+                    if (ttl < this.crawlConfig.maxFailedTtl) {
+                        await this.database.setTtl(url, ttl + 1);
+                        await this.database.setStatus(url, EStatus.QUEUED);
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.QUEUED, 1);
+                    }
+                    else {
+                        await this.database.setStatus(url, EStatus.FAILED);
+                        if (this.tracker)
+                            this.tracker.delta(EStatus.FAILED, 1);
+                        if (commonsTypeHasPropertyString(ex, 'code')) {
+                            await this.database.setFailReason(url, ex.code);
+                        }
+                        else if (commonsTypeHasPropertyString(ex, 'message')) {
+                            if (/^Timeout on crawl/.test(ex.message)) {
+                                await this.database.setFailReason(url, 'TIMEOUT');
+                            }
+                            else {
+                                await this.database.setFailReason(url, ex.message);
+                            }
+                        }
+                    }
+                }
+                finally {
+                    if (this.tracker)
+                        this.tracker.delta(EStatus.ACTIVE, -1);
+                }
+                tally++;
+                if (tally >= this.crawlConfig.maxFetchesPerCrawl) {
+                    commonsLogAlert(`Crawl for ${this.domain} reached the maximum permitted fetches for this session.`);
+                    break;
+                }
+            }
+        }
+        catch (e) {
+            commonsLogDebug('debug position 1');
+            console.log(e);
+        }
+        finally {
+            commonsGracefulAbortRemoveCallback(gracefulAbortId);
+        }
+    }
+}
+//# sourceMappingURL=crawler.mjs.map

package/dist/classes/crawler.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.mjs","sourceRoot":"","sources":["../../src/classes/crawler.mts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAG/B,OAAO,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAEtF,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACxG,OAAO,EAAE,+BAA+B,EAAE,kCAAkC,EAAE,MAAM,yBAAyB,CAAC;AAC9G,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAG1D,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAK7C,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAU/C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAEhC,MAAM,OAAO,OAAO;IAsMT;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IA5MH,MAAM,CAAC,sBAAsB,CAAC,OAA2B;QAC/D,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,uFAAuF,CAAC,CAAC;QACrH,OAAO,CAAC,SAAS,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QACjD,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,CAAC,SAAS,CAAC,YAAY,EAAE,qJAAqJ,CAAC,CAAC;IACxL,CAAC;IAEM,MAAM,CAAC,OAAO,CACnB,OAAiC,EACjC,GAAW,EACX,cAAsB,EACtB,WAAmB,EACnB,OAAiB;QAElB,OAAO,IAAI,OAAO,CAAkB,CAAC,OAAqC,EAAE,MAA0B,EAAQ,EAAE;YAC/G,MAAM,KAAK,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YAC3C,IAAI,QAAQ,GAAY,KAAK,CAAC;YAE9B,IAAI,SAAS,GAAY,KAAK,CAAC;YAC/B,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,eAAe,GAAW,uBAAuB,GAAG,EAAE,CAAC;YAE7D,MAAM,kBAAkB,GAAW,WAAW,GAAG,oBAAoB,CAAC;YACtE,mBAAmB,CACjB,cAAc,GAAG,KAAK,EACtB,kBAAkB,CACnB;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,eAAe,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBAEhD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACnC,OAAO;YACR,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,eAAe,CAAC,mBAAmB,CAAC,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC;YACT,CAAC,CAAC,CAAC;YAEL,MAAM,GAAG,GAAuB,OAAO,CAAC,OAAO,CAC7C,GAAG,EACH,CAAC,GAAyB,EAAQ,EAAE;gBACnC,4EAA4E;gBAC5E,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAE/B,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAW,GAAG,GAAG,KAAK,CAAC;gBAEpC,gDAAgD;gBAEhD,IAAI,IAAI,GAAW,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,GAAG,CAAC,EAAE,CACJ,MAAM,EACN,CAAC,KAAa,EAAQ,EAAE;oBACvB,IAAI,IAAI,GAAG,WAAW,EAAE,CAAC;wBACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,QAAQ,GAAG,IAAI,CAAC;4BAChB,GAAG,CAAC,OAAO,EAAE,CAAC;4BAEd,eAAe,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;wBACzD,CAAC;wBAED,OAAO;oBACR,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACjB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,OAAO;wBAAE,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC9C,CAAC,CACF,CAAC;gBACF,GAAG,CAAC,EAAE,CACJ,KAAK,EACL,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,SAAS,GAAG,IAAI,CAAC;oBAEjB,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,OAAO,CAAC;wBACN,OAAO,EAAE,OAAO;wBAChB,mEAAmE;wBACnE,EAAE,EAAG,GAAuB,CAAC,UAAU,CAAC;wBACxC,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG;wBACjC,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC,CAAC;gBACJ,CAAC,CACF,CAAC;gBAEF,wEAAwE;gBACxE,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;oBACpB,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;gBACb,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAEpC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAS,EAAE;gBAC5B,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,QAAQ,GAAG,IAAI,CAAC;gBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YACpC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CACJ,UAAU,EACV,CAAC,GAAyB,EAAQ,EAAE;gBAClC,GAAuB,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACrE,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,OAAO,EACP,CAAC,GAAU,EAAQ,EAAE;gBACpB,IAAI,QAAQ,IAAI,SAAS;oBAAE,OAAO;gBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAE7C,MAAM,CAAC,GAAG,CAAC,CAAC;YACb,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,EAAE,CACJ,QAAQ,EACR,CAAC,MAAc,EAAQ,EAAE;gBACxB,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;gBAClC,MAAM,CAAC,EAAE,CACP,SAAS,EACT,GAAS,EAAE;oBACV,IAAI,QAAQ,IAAI,SAAS;wBAAE,OAAO;oBAElC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;oBAE7C,QAAQ,GAAG,IAAI,CAAC;oBAChB,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACpC,CAAC,CACF,CAAC;YACH,CAAC,CACF,CAAC;YAEF,GAAG,CAAC,GAAG,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,OAA4B,EAAE,WAAqB;QAC9E,MAAM,GAAG,GAAoC,IAAI,GAAG,EAA8B,CAAC;QAEnF,MAAM,IAAI,GAAoC,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,EAAE,GAAa,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACtD,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE9D,MAAM,KAAK,GAAW,OAAO,CAAC,GAAG,CAAC,GAAG,CAAW,CAAC;YACjD,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;gBAClB,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAAC,MAAM;gBACzD,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBAAC,MAAM;gBAC5D,KAAK,MAAM;oBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBAAC,MAAM;gBACtD,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACrD,CAAC;QACF,CAAC;QAED,OAAO,GAAG,CAAC;IACZ,CAAC;IAEO,QAAQ,GAAY,KAAK,CAAC;IAC1B,SAAS,GAAY,KAAK,CAAC;IAEnC,YACU,MAAc,EACd,QAAyB,EACzB,WAAyB,EACzB,aAA6C,EAC7C,YAA2B,EAC3B,OAAsB,EACtB,KAAY,EACZ,OAAiB;QAPjB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;QACzB,gBAAW,GAAX,WAAW,CAAc;QACzB,kBAAa,GAAb,aAAa,CAAgC;QAC7C,iBAAY,GAAZ,YAAY,CAAe;QAC3B,YAAO,GAAP,OAAO,CAAe;QACtB,UAAK,GAAL,KAAK,CAAO;QACZ,YAAO,GAAP,OAAO,CAAU;IACxB,CAAC;IAEI,KAAK;QACZ,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,wBAAwB,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAEM,KAAK;QACX,eAAe,CAAC,uBAAuB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtB,CAAC;IAEM,MAAM;QACZ,cAAc,CAAC,wBAAwB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,WAAoB;QACnD,cAAc,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;QAElC,MAAM,MAAM,GAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjC,IAAI,OAA2C,CAAC;QAChD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,KAAK,OAAO;gBAAE,OAAO,GAAG,IAAI,CAAC;gBAAC,MAAM;YACpC,KAAK,QAAQ;gBAAE,OAAO,GAAG,KAAK,CAAC;gBAAC,MAAM;YACtC;gBACC,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,OAAwB,CAAC;QAC7B,4CAA4C;QAC5C,IAAI,CAAC;YACJ,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAC7B,OAAO,EACP,GAAG,EACH,IAAI,CAAC,WAAW,CAAC,cAAc,EAC/B,IAAI,CAAC,WAAW,CAAC,WAAW,EAC5B,IAAI,CAAC,OAAO,CACb,CAAC;QACH,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACb,kEAAkE;YAClE,MAAM,EAAE,CAAC;QACV,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAoC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;QACrH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrH,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,IACC,OAAO,CAAC,UAAU,IAAI,GAAG;eACtB,OAAO,CAAC,UAAU,GAAG,GAAG;eACxB,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;eACvB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,0CAA0C;UAC/E,CAAC;YACF,iBAAiB;YACjB,MAAM,QAAQ,GAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAY,EAAE,GAAG,CAAC,CAAC;YAEvE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBACzH,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAEpI,IAAI,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG;oBAAE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,CAAC;QACF,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,GAAW,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,QAAQ,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpE,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACxE,MAAM,WAAW,GAAW,OAAO,CAAC,GAAG,CAAC,cAAc,CAAY,CAAC;YACnE,MAAM,OAAO,GAAY,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAE5D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAY,IAAI,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAAE,SAAS;gBAElC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;oBAAE,SAAS;gBAErD,IAAI,CAAC;oBACJ,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACnC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;oBACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACjB,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACb,IAAI,CAAC;wBACJ,MAAM,KAAK,GAAa,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;wBAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;4BAAE,SAAS;wBAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;4BAC1B,IAAI,IAAI,KAAK,GAAG;gCAAE,SAAS;4BAC3B,IAAI,CAAC;gCACJ,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BACrB,CAAC;4BAAC,OAAO,GAAG,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;wBACnC,CAAC;oBACF,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC;wBACb,eAAe,CAAC,kBAAkB,CAAC,CAAC;wBACpC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjB,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAY,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3F,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtH,CAAC;QACD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAErD,OAAO,OAAO,CAAC,UAAU,CAAC;IAC3B,CAAC;IAEM,KAAK,CAAC,KAAK;QACjB,MAAM,eAAe,GAAW,+BAA+B,CAAC,GAAS,EAAE;YAC1E,eAAe,CAAC,8CAA8C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9E,IAAI,CAAC,KAAK,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACJ,IAAI,CAAC;gBACJ,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACb,IAAK,EAAY,CAAC,OAAO,KAAK,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC;oBACxF,eAAe,CAAC,qBAAqB,IAAI,CAAC,MAAM,yBAAyB,CAAC,CAAC;oBAC3E,OAAO;gBACR,CAAC;gBAED,eAAe,CAAC,UAAU,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC1C,OAAO;YACR,CAAC;YAED,MAAM,MAAM,GAAU,IAAI,KAAK,CAC7B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CACb,CAAC;YACF,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAEpB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS;oBAAE,MAAM;gBAE1B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnB,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;oBAChC,SAAS;gBACV,CAAC;gBAED,MAAM,GAAG,GAAqB,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpE,IAAI,CAAC,GAAG,EAAE,CAAC;oBACV,cAAc,CAAC,aAAa,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC;oBACtD,MAAM;gBACP,CAAC;gBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAEzC,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;gBAEzD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;oBACvC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;oBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;oBAEtD,uDAAuD;oBACvD,4DAA4D;oBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClC,SAAS;gBACV,CAAC;gBAED,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;oBACvD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;oBAE5D,uDAAuD;oBACvD,4DAA4D;oBAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAClC,SAAS;gBACV,CAAC;gBAED,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzD,eAAe,CAAC,sCAAsC,GAAG,qBAAqB,CAAC,CAAC;oBAChF,MAAM;gBACP,CAAC;gBACD,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAExD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACf,IAAI,CAAC;wBACJ,MAAM,OAAO,GAAW,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;wBAE/F,MAAM,mBAAmB,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;wBAE3D,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;4BACzC,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBACjC,CAAC;oBACF,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;gBAC/B,CAAC;gBACD,IAAI,IAAI,CAAC,SAAS;oBAAE,MAAM;gBAE1B,IAAI,IAAI,CAAC,OAAO;oBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACnD,IAAI,CAAC;oBACJ,MAAM,UAAU,GAAW,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;oBAE9D,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAC3D,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;oBACjD,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAClC,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACb,IAAI,IAAI,CAAC,OAAO;wBAAE,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAEjD,MAAM,GAAG,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACzD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;wBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;wBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;wBACnD,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;oBACzD,CAAC;yBAAM,CAAC;wBACP,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;wBACnD,IAAI,IAAI,CAAC,OAAO;4BAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;wBAExD,IAAI,4BAA4B,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;4BAC9C,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAAuB,CAAC,IAAI,CAAC,CAAC;wBACvE,CAAC;6BAAM,IAAI,4BAA4B,CAAC,EAAE,EAAE,SAAS,CAAC,EAAE,CAAC;4BACxD,IAAI,mBAAmB,CAAC,IAAI,CAAE,EAA0B,CAAC,OAAO,CAAC,EAAE,CAAC;gCACnE,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;4BACnD,CAAC;iCAAM,CAAC;gCACP,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAG,EAA0B,CAAC,OAAO,CAAC,CAAC;4BAC7E,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;wBAAS,CAAC;oBACV,IAAI,IAAI,CAAC,OAAO;wBAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBAED,KAAK,EAAE,CAAC;gBACR,IAAI,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;oBAClD,eAAe,CAAC,aAAa,IAAI,CAAC,MAAM,0DAA0D,CAAC,CAAC;oBACpG,MAAM;gBACP,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,eAAe,CAAC,kBAAkB,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;gBAAS,CAAC;YACV,kCAAkC,CAAC,eAAe,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;CACD"}

package/dist/classes/{dns.js → dns.mjs}

@@ -1,6 +1,6 @@
 import * as dns from 'dns';
-import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-es-async';
-import { commonsOutputDebug } from 'nodecommons-es-cli';
+import { commonsAsyncAbortTimeout, commonsAsyncTimeout } from 'tscommons-esm-async';
+import { commonsLogDebug } from 'nodecommons-esm-log';
 export class Dns {
     static resolve(domain, connectTimeout) {
         // try IP4 first, then IP6
@@ -15,7 +15,7 @@ export class Dns {
                 .catch((e) => {
                 if (e.message === 'abortTimeout called')
                     return;
-                commonsOutputDebug('debug position 4');
+                commonsLogDebug('debug position 4');
                 console.log(e);
                 reject(e);
             });
@@ -44,4 +44,4 @@ export class Dns {
         });
     }
 }
-//# sourceMappingURL=dns.js.map
+//# sourceMappingURL=dns.mjs.map

package/dist/classes/dns.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dns.mjs","sourceRoot":"","sources":["../../src/classes/dns.mts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAE3B,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAEpF,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,OAAgB,GAAG;IACjB,MAAM,CAAC,OAAO,CACnB,MAAc,EACd,cAAsB;QAEvB,0BAA0B;QAE1B,OAAO,IAAI,OAAO,CAAW,CAAC,OAA8B,EAAE,MAA0B,EAAQ,EAAE;YACjG,IAAI,QAAQ,GAAY,KAAK,CAAC;YAC9B,MAAM,SAAS,GAAW,OAAO,MAAM,UAAU,CAAC;YAElD,mBAAmB,CACjB,cAAc,EACd,SAAS,CACV;iBACE,IAAI,CAAC,GAAS,EAAE;gBAChB,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAQ,EAAQ,EAAE;gBACzB,IAAI,CAAC,CAAC,OAAO,KAAK,qBAAqB;oBAAE,OAAO;gBAChD,eAAe,CAAC,kBAAkB,CAAC,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,MAAM,CAAC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;YAEL,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAe,EAAE,OAAiB,EAAQ,EAAE;gBACjE,IAAI,GAAG,EAAE,CAAC;oBACT,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAgB,EAAE,QAAkB,EAAQ,EAAE;wBACnE,IAAI,IAAI,EAAE,CAAC;4BACV,IAAI,CAAC,QAAQ;gCAAE,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACnD,MAAM,CAAC,IAAI,CAAC,CAAC;4BACb,OAAO;wBACR,CAAC;wBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,wBAAwB,CAAC,SAAS,CAAC,CAAC;4BACpC,OAAO,CAAC,QAAQ,CAAC,CAAC;wBACnB,CAAC;wBACD,OAAO;oBACR,CAAC,CAAC,CAAC;oBACH,OAAO;gBACR,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBACpC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}

package/dist/classes/{expirer.d.ts → expirer.d.mts}

@@ -1,5 +1,5 @@
-import { Expiry } from '../classes/expiry';
-import { DatabaseService } from '../services/database.service';
+import { Expiry } from '../classes/expiry.mjs';
+import { DatabaseService } from '../services/database.service.mjs';
 export declare class Expirer {
     private expiry;
     private database;

package/dist/classes/expirer.mjs

@@ -0,0 +1,117 @@
+import mongodb from 'mongodb';
+import { commonsArrayChunk, commonsDateDateToYmdHis, commonsTypeHasPropertyDate } from 'tscommons-esm-core';
+import { EStatus } from 'hydra-crawler-esm-assets';
+import { commonsLogAlert, commonsLogDoing } from 'nodecommons-esm-log';
+import { isTMongoIdRow } from '../services/database.service.mjs';
+export class Expirer {
+    expiry;
+    database;
+    constructor(expiry, database) {
+        this.expiry = expiry;
+        this.database = database;
+    }
+    async listQueuedAndActive() {
+        const log = commonsLogDoing('expirer', 'Enumering queued and active domains');
+        const results = this.database.getUrls().aggregate([
+            { $match: { status: { $in: [EStatus.QUEUED, EStatus.ACTIVE] } } },
+            { $group: { _id: '$domain' } }
+        ], { allowDiskUse: true });
+        const queued = await this.database.listQueryResults(results, isTMongoIdRow);
+        log.success(queued.length);
+        return queued
+            // eslint-disable-next-line no-underscore-dangle
+            .map((q) => q._id);
+    }
+    async expire(limit) {
+        commonsLogAlert('Running expiry thread');
+        const queued = await this.listQueuedAndActive();
+        const expired = [];
+        { // scope
+            const log = commonsLogDoing('expirer', 'Searching for expired URLs');
+            const results = this.database.getUrls().find({ $and: [
+                    { status: { $nin: [
+                                EStatus.ARCHIVED,
+                                EStatus.QUEUED,
+                                EStatus.ACTIVE
+                            ] } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } }
+                ] }, {});
+            const now = new Date().getTime() / 1000;
+            let tally = 0;
+            let found = 0;
+            while (true) {
+                tally++;
+                if ((tally % 1000) === 0)
+                    log.progress(`${tally}, ${found}`);
+                if (limit !== undefined && tally > limit)
+                    break;
+                const row = await results.next();
+                if (row === null)
+                    break;
+                const interval = this.expiry.getBestExpiry(row.url);
+                if (!commonsTypeHasPropertyDate(row, 'attempted'))
+                    continue;
+                const attempted = row['attempted'].getTime() / 1000;
+                if ((attempted + interval) < now) {
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+                    expired.push(row['_id']);
+                    found++;
+                }
+            }
+            log.success(found);
+            if (found === 0)
+                return false;
+        }
+        const batches = commonsArrayChunk(expired, 100);
+        { // scope
+            const log = commonsLogDoing('expirer', 'Re-queuing expired');
+            let tally = 0;
+            for (const batch of batches) {
+                const batchIds = batch
+                    .map((id) => new mongodb.ObjectId(id));
+                await this.database.getUrls().updateMany({ _id: { $in: batchIds } }, {
+                    $set: { status: EStatus.QUEUED },
+                    $unset: { ttl: true }
+                });
+                tally += 100;
+                log.progress(tally);
+            }
+            log.success();
+        }
+        return true;
+    }
+    async expireFixed(threshold) {
+        commonsLogAlert('Running fixed date expiry thread');
+        const queued = await this.listQueuedAndActive();
+        { // scope
+            const log = commonsLogDoing('expirer', `Searching attempted before date ${commonsDateDateToYmdHis(threshold)}`);
+            const count = await this.database.getUrls().find({ $and: [
+                    { status: { $ne: EStatus.ARCHIVED } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } },
+                    { attempted: { $lt: threshold } }
+                ] }, {}).count();
+            log.success(count);
+            if (count === 0)
+                return false;
+        }
+        { // scope
+            const log = commonsLogDoing('expirer', 'Re-queuing expired');
+            await this.database.getUrls().updateMany({ $and: [
+                    { status: { $ne: EStatus.ARCHIVED } },
+                    { domain: { $nin: queued } },
+                    { attempted: { $exists: true } },
+                    { attempted: { $lt: threshold } }
+                ] }, {
+                $set: { status: EStatus.QUEUED },
+                $unset: { ttl: true }
+            });
+            log.success();
+        }
+        // ?
+        // const log: CommonsSubLoggable = commonsLogDoing('expirer', 'Searching for expired URLs');
+        return true;
+    }
+}
+//# sourceMappingURL=expirer.mjs.map

package/dist/classes/expirer.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"expirer.mjs","sourceRoot":"","sources":["../../src/classes/expirer.mts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,0BAA0B,EAAmB,MAAM,oBAAoB,CAAC;AAE7H,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAGnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAsB,MAAM,qBAAqB,CAAC;AAI3F,OAAO,EAAgC,aAAa,EAAE,MAAM,kCAAkC,CAAC;AAE/F,MAAM,OAAO,OAAO;IAET;IACA;IAFV,YACU,MAAc,EACd,QAAyB;QADzB,WAAM,GAAN,MAAM,CAAQ;QACd,aAAQ,GAAR,QAAQ,CAAiB;IAChC,CAAC;IAEI,KAAK,CAAC,mBAAmB;QAChC,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;QAClG,MAAM,OAAO,GAA2C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,SAAS,CAAc;YACrG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE;YACnE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;SAC/B,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3B,MAAM,MAAM,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAE3F,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE3B,OAAO,MAAM;YACX,gDAAgD;aAC/C,GAAG,CAAC,CAAC,CAAc,EAAU,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,KAAc;QACjC,eAAe,CAAC,uBAAuB,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAEzF,MAAM,OAAO,GAA6B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACpE,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE;gCAChB,OAAO,CAAC,QAAQ;gCAChB,OAAO,CAAC,MAAM;gCACd,OAAO,CAAC,MAAM;6BACf,EAAE,EAAE;oBACL,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;iBACjC,EAAC,EACF,EAAE,CACH,CAAC;YAEF,MAAM,GAAG,GAAW,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YAEhD,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,OAAO,IAAI,EAAE,CAAC;gBACb,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC7D,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,KAAK;oBAAE,MAAM;gBAEhD,MAAM,GAAG,GAAc,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,GAAG,KAAK,IAAI;oBAAE,MAAM;gBAExB,MAAM,QAAQ,GAAW,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE5D,IAAI,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,CAAC;oBAAE,SAAS;gBAE5D,MAAM,SAAS,GAAa,GAAuB,CAAC,WAAW,CAAU,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;gBAC3F,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC;oBAClC,iEAAiE;oBACjE,OAAO,CAAC,IAAI,CAAE,GAAuB,CAAC,KAAK,CAAC,CAAC,CAAC;oBAC9C,KAAK,EAAE,CAAC;gBACT,CAAC;YACF,CAAC;YACD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,MAAM,OAAO,GAAe,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAE5D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,IAAI,KAAK,GAAW,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAuB,KAAK;qBACvC,GAAG,CAAC,CAAC,EAAU,EAAoB,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAC1B;oBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;oBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;iBACtB,CACF,CAAC;gBAEF,KAAK,IAAI,GAAG,CAAC;gBACb,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YACD,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,SAAe;QACvC,eAAe,CAAC,kCAAkC,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAa,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAE1D,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,mCAAmC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpI,MAAM,KAAK,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CACtD,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF,EAAE,CACH,CAAC,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAEnB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC/B,CAAC;QAED,CAAC,CAAC,QAAQ;YACT,MAAM,GAAG,GAAuB,eAAe,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;YACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,UAAU,CACtC,EAAE,IAAI,EAAE;oBACN,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;oBACrC,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;oBAC5B,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;oBAChC,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE;iBAClC,EAAC,EACF;gBACE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;aACtB,CACF,CAAC;YACF,GAAG,CAAC,OAAO,EAAE,CAAC;QACf,CAAC;QAED,IAAI;QACJ,4FAA4F;QAE5F,OAAO,IAAI,CAAC;IACb,CAAC;CACD"}

package/dist/classes/{expiry.d.ts → expiry.d.mts}

@@ -1,4 +1,4 @@
-import { IExpiry } from '../interfaces/iexpiry';
+import { IExpiry } from '../interfaces/iexpiry.mjs';
 export declare class Expiry {
     static loadFromFile(file: string): IExpiry[];
     private static getScore;

package/dist/classes/{expiry.js → expiry.mjs}

@@ -1,12 +1,9 @@
 import { URL } from 'url';
-import { commonsTypeHasPropertyString, commonsTypeIsArray, commonsTypeIsTArray } from 'tscommons-es-core';
-import { commonsFileReadJsonFile } from 'nodecommons-es-file';
-import { Matcher } from '../helpers/matcher';
-import { isIExpiry, toIExpiry } from '../interfaces/iexpiry';
+import { commonsTypeHasPropertyString, commonsTypeIsArray, commonsTypeIsTArray } from 'tscommons-esm-core';
+import { commonsFileReadJsonFile } from 'nodecommons-esm-file';
+import { Matcher } from '../helpers/matcher.mjs';
+import { isIExpiry, toIExpiry } from '../interfaces/iexpiry.mjs';
 export class Expiry {
-    constructor() {
-        this.expiries = [];
-    }
     static loadFromFile(file) {
         const json = commonsFileReadJsonFile(file);
         if (!commonsTypeIsArray(json))
@@ -16,7 +13,7 @@ export class Expiry {
             try {
                 return toIExpiry(entry);
             }
-            catch (e) {
+            catch (_e) {
                 return undefined;
             }
         });
@@ -37,6 +34,7 @@ export class Expiry {
         }
         return total;
     }
+    expiries = [];
     add(expiries) {
         for (const expiry of expiries) {
             this.expiries.push(expiry);
@@ -60,4 +58,4 @@ export class Expiry {
         return best.expiry;
     }
 }
-//# sourceMappingURL=expiry.js.map
+//# sourceMappingURL=expiry.mjs.map