@crawlee/core 4.0.0-beta.23 → 4.0.0-beta.231

package/crawlers/statistics.js

@@ -1,349 +0,0 @@
-import ow from 'ow';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
-import { ErrorTracker } from './error_tracker.js';
-/**
- * @ignore
- */
-class Job {
-    lastRunAt = null;
-    durationMillis;
-    run() {
-        this.lastRunAt = Date.now();
-    }
-    finish() {
-        this.durationMillis = Date.now() - this.lastRunAt;
-        return this.durationMillis;
-    }
-}
-const errorTrackerConfig = {
-    showErrorCode: true,
-    showErrorName: true,
-    showStackTrace: true,
-    showFullStack: false,
-    showErrorMessage: true,
-    showFullMessage: false,
-};
-/**
- * The statistics class provides an interface to collecting and logging run
- * statistics for requests.
- *
- * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
- * migrations and abort/resurrect
- *
- * @category Crawlers
- */
-export class Statistics {
-    static id = 0;
-    /**
-     * An error tracker for final retry errors.
-     */
-    errorTracker;
-    /**
-     * An error tracker for retry errors prior to the final retry.
-     */
-    errorTrackerRetry;
-    /**
-     * Statistic instance id.
-     */
-    id;
-    /**
-     * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
-     */
-    state;
-    /**
-     * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
-     */
-    requestRetryHistogram = [];
-    /**
-     * Contains the associated Configuration instance
-     */
-    config;
-    keyValueStore = undefined;
-    persistStateKey;
-    logIntervalMillis;
-    logMessage;
-    listener;
-    requestsInProgress = new Map();
-    log;
-    instanceStart;
-    logInterval;
-    events;
-    persistenceOptions;
-    /**
-     * @internal
-     */
-    constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            logIntervalSecs: ow.optional.number,
-            logMessage: ow.optional.string,
-            log: ow.optional.object,
-            keyValueStore: ow.optional.object,
-            config: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-            saveErrorSnapshots: ow.optional.boolean,
-            id: ow.optional.any(ow.number, ow.string),
-        }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = Configuration.getGlobalConfig(), persistenceOptions = {
-            enable: true,
-        }, saveErrorSnapshots = false, id, } = options;
-        this.id = id ?? String(Statistics.id++);
-        this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
-        this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
-        this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
-        this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
-        this.logIntervalMillis = logIntervalSecs * 1000;
-        this.logMessage = logMessage;
-        this.keyValueStore = keyValueStore;
-        this.listener = this.persistState.bind(this);
-        this.events = config.getEventManager();
-        this.config = config;
-        this.persistenceOptions = persistenceOptions;
-        // initialize by "resetting"
-        this.reset();
-    }
-    /**
-     * Set the current statistic instance to pristine values
-     */
-    reset() {
-        this.errorTracker.reset();
-        this.errorTrackerRetry.reset();
-        this.state = {
-            requestsFinished: 0,
-            requestsFailed: 0,
-            requestsRetries: 0,
-            requestsFailedPerMinute: 0,
-            requestsFinishedPerMinute: 0,
-            requestMinDurationMillis: Infinity,
-            requestMaxDurationMillis: 0,
-            requestTotalFailedDurationMillis: 0,
-            requestTotalFinishedDurationMillis: 0,
-            crawlerStartedAt: null,
-            crawlerFinishedAt: null,
-            statsPersistedAt: null,
-            crawlerRuntimeMillis: 0,
-            requestsWithStatusCode: {},
-            errors: this.errorTracker.result,
-            retryErrors: this.errorTrackerRetry.result,
-        };
-        this.requestRetryHistogram.length = 0;
-        this.requestsInProgress.clear();
-        this.instanceStart = Date.now();
-        this._teardown();
-    }
-    /**
-     * @param options - Override the persistence options provided in the constructor
-     */
-    async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        if (!this.keyValueStore) {
-            return;
-        }
-        await this.keyValueStore.setValue(this.persistStateKey, null);
-    }
-    /**
-     * Increments the status code counter.
-     */
-    registerStatusCode(code) {
-        const s = String(code);
-        if (this.state.requestsWithStatusCode[s] === undefined) {
-            this.state.requestsWithStatusCode[s] = 0;
-        }
-        this.state.requestsWithStatusCode[s]++;
-    }
-    /**
-     * Starts a job
-     * @ignore
-     */
-    startJob(id) {
-        let job = this.requestsInProgress.get(id);
-        if (!job)
-            job = new Job();
-        job.run();
-        this.requestsInProgress.set(id, job);
-    }
-    /**
-     * Mark job as finished and sets the state
-     * @ignore
-     */
-    finishJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
-        if (!job)
-            return;
-        const jobDurationMillis = job.finish();
-        this.state.requestsFinished++;
-        this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
-        this._saveRetryCountForJob(retryCount);
-        if (jobDurationMillis < this.state.requestMinDurationMillis)
-            this.state.requestMinDurationMillis = jobDurationMillis;
-        if (jobDurationMillis > this.state.requestMaxDurationMillis)
-            this.state.requestMaxDurationMillis = jobDurationMillis;
-        this.requestsInProgress.delete(id);
-    }
-    /**
-     * Mark job as failed and sets the state
-     * @ignore
-     */
-    failJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
-        if (!job)
-            return;
-        this.state.requestTotalFailedDurationMillis += job.finish();
-        this.state.requestsFailed++;
-        this._saveRetryCountForJob(retryCount);
-        this.requestsInProgress.delete(id);
-    }
-    /**
-     * Calculate the current statistics
-     */
-    calculate() {
-        const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
-        const totalMillis = Date.now() - this.instanceStart;
-        const totalMinutes = totalMillis / 1000 / 60;
-        return {
-            requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
-            requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
-            requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
-            requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
-            requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
-            requestsTotal: requestsFailed + requestsFinished,
-            crawlerRuntimeMillis: totalMillis,
-        };
-    }
-    /**
-     * Initializes the key value store for persisting the statistics,
-     * displaying the current state in predefined intervals
-     */
-    async startCapturing() {
-        this.keyValueStore ??= await KeyValueStore.open(null, { config: this.config });
-        if (this.state.crawlerStartedAt === null) {
-            this.state.crawlerStartedAt = new Date();
-        }
-        if (this.persistenceOptions.enable) {
-            await this._maybeLoadStatistics();
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        }
-        this.logInterval = setInterval(() => {
-            this.log.info(this.logMessage, {
-                ...this.calculate(),
-                retryHistogram: this.requestRetryHistogram,
-            });
-        }, this.logIntervalMillis);
-    }
-    /**
-     * Stops logging and remove event listeners, then persist
-     */
-    async stopCapturing() {
-        this._teardown();
-        this.state.crawlerFinishedAt = new Date();
-        await this.persistState();
-    }
-    _saveRetryCountForJob(retryCount) {
-        if (retryCount > 0)
-            this.state.requestsRetries++;
-        this.requestRetryHistogram[retryCount] ??= 0;
-        this.requestRetryHistogram[retryCount]++;
-    }
-    /**
-     * Persist internal state to the key value store
-     * @param options - Override the persistence options provided in the constructor
-     */
-    async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
-            return;
-        }
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
-        // use half the interval of `persistState` to avoid race conditions
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
-        const timeoutSecs = persistStateIntervalMillis / 2_000;
-        await this.keyValueStore
-            .setValue(this.persistStateKey, this.toJSON(), {
-            timeoutSecs,
-            doNotRetryTimeouts: true,
-        })
-            .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
-    }
-    /**
-     * Loads the current statistic from the key value store if any
-     */
-    async _maybeLoadStatistics() {
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        const savedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (!savedState)
-            return;
-        // We saw a run where the requestRetryHistogram was not iterable and crashed
-        // the crawler. Adding some logging to monitor this problem in the future.
-        if (!Array.isArray(savedState.requestRetryHistogram)) {
-            this.log.warning('Received invalid state from Key-value store.', {
-                persistStateKey: this.persistStateKey,
-                state: savedState,
-            });
-        }
-        this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });
-        // the `requestRetryHistogram` array might be very large, we could end up with
-        // `RangeError: Maximum call stack size exceeded` if we use `a.push(...b)`
-        savedState.requestRetryHistogram.forEach((idx) => this.requestRetryHistogram.push(idx));
-        this.state.requestsFinished = savedState.requestsFinished;
-        this.state.requestsFailed = savedState.requestsFailed;
-        this.state.requestsRetries = savedState.requestsRetries;
-        this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;
-        this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;
-        this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;
-        this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;
-        // persisted state uses ISO date strings
-        this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;
-        this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
-        this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
-        this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
-        this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
-        this.log.debug('Loaded from KeyValueStore');
-    }
-    _teardown() {
-        // this can be called before a call to startCapturing happens (or in a 'finally' block)
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        if (this.logInterval) {
-            clearInterval(this.logInterval);
-            this.logInterval = null;
-        }
-    }
-    /**
-     * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
-     * or through `keyValueStore.setValue('KEY', statsInstance)`
-     */
-    toJSON() {
-        // merge all the current state information that can be used from the outside
-        // without the need to reconstruct for the sake of stats.calculate()
-        // omit duplicated information
-        const result = {
-            ...this.state,
-            crawlerLastStartTimestamp: this.instanceStart,
-            crawlerFinishedAt: this.state.crawlerFinishedAt
-                ? new Date(this.state.crawlerFinishedAt).toISOString()
-                : null,
-            crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
-            requestRetryHistogram: this.requestRetryHistogram,
-            statsId: this.id,
-            statsPersistedAt: new Date().toISOString(),
-            ...this.calculate(),
-        };
-        Reflect.deleteProperty(result, 'requestsWithStatusCode');
-        Reflect.deleteProperty(result, 'errors');
-        Reflect.deleteProperty(result, 'retryErrors');
-        result.requestsWithStatusCode = this.state.requestsWithStatusCode;
-        result.errors = this.state.errors;
-        result.retryErrors = this.state.retryErrors;
-        return result;
-    }
-}
-//# sourceMappingURL=statistics.js.map

package/crawlers/statistics.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;GAEG;AACH,MAAM,GAAG;IACG,SAAS,GAAkB,IAAI,CAAC;IAChC,cAAc,CAAU;IAEhC,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAM,OAAO,UAAU;IACX,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAEtB;;OAEG;IACH,YAAY,CAAe;IAE3B;;OAEG;IACH,iBAAiB,CAAe;IAEhC;;OAEG;IACM,EAAE,CAAS;IAEpB;;OAEG;IACH,KAAK,CAAkB;IAEvB;;OAEG;IACM,qBAAqB,GAAa,EAAE,CAAC;IAE9C;;OAEG;IACc,MAAM,CAAgB;IAE7B,aAAa,GAAmB,SAAS,CAAC;IAC1C,eAAe,CAAS;IAC1B,iBAAiB,CAAS;IAC1B,UAAU,CAAS;IACnB,QAAQ,CAAsB;IAC9B,kBAAkB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5C,GAAG,CAAM;IAClB,aAAa,CAAU;IACvB,WAAW,CAAU;IACrB,MAAM,CAAe;IACrB,kBAAkB,CAAqB;IAE/C;;OAEG;IACH,YAAY,UAA6B,EAAE;QACvC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YACvC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;SAC5C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,EAC1B,EAAE,GACL,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,0BAA0B,IAAI,CAAC,EAAE,EAAE,CAAC;QAE3D,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB,EAAE,UAAkB;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB,EAAE,UAAkB;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAK,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,UAAkB;QAC9C,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;YAC3C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uCAAuC,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7F,CAAC;IACV,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC"}

package/enqueue_links/enqueue_links.d.ts

@@ -1,249 +0,0 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import type { Request } from '../request.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
-    /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
-    limit?: number;
-    /** An array of URLs to enqueue. */
-    urls?: readonly string[];
-    /** A request queue to which the URLs will be enqueued. */
-    requestQueue?: RequestProvider;
-    /** A CSS selector matching links to be enqueued. */
-    selector?: string;
-    /** Sets {@link Request.userData} for newly enqueued requests. */
-    userData?: Dictionary;
-    /**
-     * Sets {@link Request.label} for newly enqueued requests.
-     *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this option.
-     */
-    label?: string;
-    /**
-     * If set to `true`, tells the crawler to skip navigation and process the request directly.
-     * @default false
-     */
-    skipNavigation?: boolean;
-    /**
-     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
-     * since the relative URL resolution is done inside the browser automatically.
-     */
-    baseUrl?: string;
-    /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
-     * enqueues the links with the same subdomain.
-     */
-    globs?: readonly GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
-     *
-     * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
-     * enqueues the links with the same subdomain.
-     */
-    regexps?: readonly RegExpInput[];
-    /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues the links with the same subdomain.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
-     */
-    pseudoUrls?: readonly PseudoUrlInput[];
-    /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
-     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
-     * or to dynamically update or create `userData`.
-     *
-     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
-     * when `uniqueKey` is computed.
-     *
-     * **Example:**
-     * ```javascript
-     * {
-     *     transformRequestFunction: (request) => {
-     *         request.userData.foo = 'bar';
-     *         request.keepUrlFragment = true;
-     *         return request;
-     *     }
-     * }
-     * ```
-     *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this function. Some request options returned by `transformRequestFunction` may be overwritten by pattern-based options from `globs`, `regexps`, or `pseudoUrls`.
-     */
-    transformRequestFunction?: RequestTransform;
-    /**
-     * The strategy to use when enqueueing the urls.
-     *
-     * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
-     *
-     * ```md
-     * Protocol          Domain
-     * ┌────┐          ┌─────────┐
-     * https://example.crawlee.dev/...
-     * │       └─────────────────┤
-     * │             Hostname    │
-     * │                         │
-     * └─────────────────────────┘
-     *          Origin
-     *```
-     *
-     * @default EnqueueStrategy.SameHostname
-     */
-    strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
-    /**
-     * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
-     * You can use this option to wait for adding all of them.
-     */
-    waitForAllRequestsToBeAdded?: boolean;
-    /**
-     * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
-     * If provided, disallowed URLs will be ignored.
-     */
-    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
-    /**
-     * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired for requests skipped
-     * 1. based on robots.txt file,
-     * 2. because they don't match enqueueLinks filters,
-     * 3. or because the maxRequestsPerCrawl limit has been reached
-     */
-    onSkippedRequest?: SkippedRequestCallback;
-}
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   globs: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestQueue'>, 'urls'> & {
-    requestQueue: {
-        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
-    };
-}): Promise<BatchAddRequestsResult>;
-/**
- * @internal
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
- * - If a user provides a base url, we always return it
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
- *   request domain, or a redirected one
- * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
- */
-export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
-/**
- * @internal
- */
-export interface ResolveBaseUrl {
-    userProvidedBaseUrl?: string;
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-}
-//# sourceMappingURL=enqueue_links.d.ts.map

package/enqueue_links/enqueue_links.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAI7C,OAAO,KAAK,EAAE,OAAO,EAAkB,MAAM,eAAe,CAAC;AAC7D,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,4BAA4B,EAC/B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,sBAAsB,EAGzB,MAAM,aAAa,CAAC;AAUrB,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEjD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,GAAG;IACtE,YAAY,EAAE;QACV,kBAAkB,EAAE,CAChB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,KACjC,OAAO,CAAC,wBAAwB,CAAC,CAAC;KAC1C,CAAC;CACL,GACF,OAAO,CAAC,sBAAsB,CAAC,CA4NjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}