@crawlee/core 4.0.0-beta.5 → 4.0.0-beta.51

package/proxy_configuration.js

@@ -1,55 +1,5 @@
 import ow from 'ow';
-import log from '@apify/log';
-import { cryptoRandomObjectId } from '@apify/utilities';
-/**
- * Internal class for tracking the proxy tier history for a specific domain.
- *
- * Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
- */
-class ProxyTierTracker {
-    histogram;
-    currentTier;
-    constructor(tieredProxyUrls) {
-        this.histogram = tieredProxyUrls.map(() => 0);
-        this.currentTier = 0;
-    }
-    /**
-     * Processes a single step of the algorithm and updates the current tier prediction based on the error history.
-     */
-    processStep() {
-        this.histogram.forEach((x, i) => {
-            if (this.currentTier === i)
-                return;
-            if (x > 0)
-                this.histogram[i]--;
-        });
-        const left = this.currentTier > 0 ? this.histogram[this.currentTier - 1] : Infinity;
-        const right = this.currentTier < this.histogram.length - 1 ? this.histogram[this.currentTier + 1] : Infinity;
-        if (this.histogram[this.currentTier] > Math.min(left, right)) {
-            this.currentTier = left <= right ? this.currentTier - 1 : this.currentTier + 1;
-        }
-        else if (this.histogram[this.currentTier] === left) {
-            this.currentTier--;
-        }
-    }
-    /**
-     * Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
-     *
-     * The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
-     * @param tier The proxy tier to mark as problematic.
-     */
-    addError(tier) {
-        this.histogram[tier] += 10;
-    }
-    /**
-     * Returns the best proxy tier for the next request based on the error history for different proxy tiers.
-     * @returns The proxy tier prediction
-     */
-    predictTier() {
-        this.processStep();
-        return this.currentTier;
-    }
-}
+import { serviceLocator } from './service_locator.js';
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
  * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -82,11 +32,9 @@ export class ProxyConfiguration {
     isManInTheMiddle = false;
     nextCustomUrlIndex = 0;
     proxyUrls;
-    tieredProxyUrls;
     usedProxyUrls = new Map();
     newUrlFunction;
-    log = log.child({ prefix: 'ProxyConfiguration' });
-    domainTiers = new Map();
+    log = serviceLocator.getLogger().child({ prefix: 'ProxyConfiguration' });
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -109,19 +57,21 @@ export class ProxyConfiguration {
      */
     constructor(options = {}) {
         const { validateRequired, ...rest } = options;
+        if ('tieredProxyUrls' in rest) {
+            throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
+                'See the v4 upgrading guide for the recommended migration to named sessions.');
+        }
         ow(rest, ow.object.exactShape({
-            proxyUrls: ow.optional.array.nonEmpty.ofType(ow.string.url),
+            proxyUrls: ow.optional.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null)),
             newUrlFunction: ow.optional.function,
-            tieredProxyUrls: ow.optional.array.nonEmpty.ofType(ow.array.nonEmpty.ofType(ow.any(ow.string.url, ow.null))),
         }));
-        const { proxyUrls, newUrlFunction, tieredProxyUrls } = options;
-        if ([proxyUrls, newUrlFunction, tieredProxyUrls].filter((x) => x).length > 1)
+        const { proxyUrls, newUrlFunction } = options;
+        if (proxyUrls && newUrlFunction)
             this._throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
             this._throwNoOptionsProvided();
         this.proxyUrls = proxyUrls;
         this.newUrlFunction = newUrlFunction;
-        this.tieredProxyUrls = tieredProxyUrls;
     }
     /**
      * This function creates a new {@link ProxyInfo} info object.
@@ -129,140 +79,42 @@ export class ProxyConfiguration {
      * the currently used proxy via the requestHandler parameter `proxyInfo`.
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
      *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return Represents information about used proxy and its configuration.
      */
-    async newProxyInfo(sessionId, options) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
-        let url;
-        let tier;
-        if (this.tieredProxyUrls) {
-            const { proxyUrl, proxyTier } = this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options);
-            url = proxyUrl ?? undefined;
-            tier = proxyTier;
-        }
-        else {
-            url = await this.newUrl(sessionId, options);
-        }
+    async newProxyInfo(options) {
+        const url = await this.newUrl(options);
         if (!url)
             return undefined;
         const { username, password, port, hostname } = new URL(url);
         return {
-            sessionId,
             url,
             username: decodeURIComponent(username),
             password: decodeURIComponent(password),
             hostname,
             port: port,
-            proxyTier: tier,
         };
     }
     /**
-     * Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
-     * @param _sessionId Session identifier
-     * @param options Options for the tiered proxy rotation
-     * @returns An object with the proxy URL and the proxy tier used.
-     */
-    _handleTieredUrl(_sessionId, options) {
-        if (!this.tieredProxyUrls)
-            throw new Error('Tiered proxy URLs are not set');
-        if (!options || (!options?.request && options?.proxyTier === undefined)) {
-            const allProxyUrls = this.tieredProxyUrls.flat();
-            return {
-                proxyUrl: allProxyUrls[this.nextCustomUrlIndex++ % allProxyUrls.length],
-            };
-        }
-        let tierPrediction = options.proxyTier;
-        if (typeof tierPrediction !== 'number') {
-            tierPrediction = this.predictProxyTier(options.request);
-        }
-        const proxyTier = this.tieredProxyUrls[tierPrediction];
-        return {
-            proxyUrl: proxyTier[this.nextCustomUrlIndex++ % proxyTier.length],
-            proxyTier: tierPrediction,
-        };
-    }
-    /**
-     * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
-     *
-     * This returns `null` if `tieredProxyUrls` option is not set.
-     */
-    predictProxyTier(request) {
-        if (!this.tieredProxyUrls)
-            return null;
-        const domain = new URL(request.url).hostname;
-        if (!this.domainTiers.has(domain)) {
-            this.domainTiers.set(domain, new ProxyTierTracker(this.tieredProxyUrls));
-        }
-        request.userData.__crawlee ??= {};
-        const tracker = this.domainTiers.get(domain);
-        if (typeof request.userData.__crawlee.lastProxyTier === 'number') {
-            tracker.addError(request.userData.__crawlee.lastProxyTier);
-        }
-        const tierPrediction = tracker.predictTier();
-        if (typeof request.userData.__crawlee.lastProxyTier === 'number' &&
-            request.userData.__crawlee.lastProxyTier !== tierPrediction) {
-            log.debug(`Changing proxy tier for domain "${domain}" from ${request.userData.__crawlee.lastProxyTier} to ${tierPrediction}.`);
-        }
-        request.userData.__crawlee.lastProxyTier = tierPrediction;
-        request.userData.__crawlee.forefront = true;
-        return tierPrediction;
-    }
-    /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
+     * Returns a new proxy URL based on provided configuration options.
      *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    async newUrl(sessionId, options) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
+    async newUrl(options) {
         if (this.newUrlFunction) {
-            return (await this._callNewUrlFunction(sessionId, { request: options?.request })) ?? undefined;
-        }
-        if (this.tieredProxyUrls) {
-            return this._handleTieredUrl(sessionId ?? cryptoRandomObjectId(6), options).proxyUrl ?? undefined;
+            return (await this._callNewUrlFunction({ request: options?.request })) ?? undefined;
         }
-        return this._handleCustomUrl(sessionId);
+        return this._handleProxyUrlsList() ?? undefined;
     }
-    /**
-     * Handles custom url rotation with session
-     */
-    _handleCustomUrl(sessionId) {
-        let customUrlToUse;
-        if (!sessionId) {
-            return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
-        }
-        if (this.usedProxyUrls.has(sessionId)) {
-            customUrlToUse = this.usedProxyUrls.get(sessionId);
-        }
-        else {
-            customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
-            this.usedProxyUrls.set(sessionId, customUrlToUse);
-        }
-        return customUrlToUse;
+    _handleProxyUrlsList() {
+        return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    async _callNewUrlFunction(sessionId, options) {
-        const proxyUrl = await this.newUrlFunction(sessionId, options);
+    async _callNewUrlFunction(options) {
+        const proxyUrl = await this.newUrlFunction(options);
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new

package/proxy_configuration.js.map

@@ -1 +1 @@
-{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAmHxD;;;;GAIG;AACH,MAAM,gBAAgB;IACV,SAAS,CAAW;IACpB,WAAW,CAAS;IAE5B,YAAY,eAAoC;QAC5C,IAAI,CAAC,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,WAAW;QACf,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5B,IAAI,IAAI,CAAC,WAAW,KAAK,CAAC;gBAAE,OAAO;YACnC,IAAI,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpF,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAE7G,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,WAAW,GAAG,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACnF,CAAC;aAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,CAAC;YACnD,IAAI,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,IAAY;QACjB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,WAAW;QACP,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,GAAG,KAAK,CAAC;IACf,kBAAkB,GAAG,CAAC,CAAC;IACvB,SAAS,CAAY;IACrB,eAAe,CAAuB;IACtC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,cAAc,CAA8B;IAC5C,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAClD,WAAW,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE5D;;;;;;;;;;;;;;;;;;;OAmBG;IACH,YAAY,UAAqC,EAAE;QAC/C,MAAM,EAAE,gBAAgB,EAAE,GAAG,IAAI,EAAE,GAAG,OAAqB,CAAC;QAC5D,EAAE,CACE,IAAI,EACJ,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YACpC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAC9C,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAC3D;SACJ,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;QAE/D,IAAI,CAAC,SAAS,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC;YACxE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QAC5C,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,YAAY,CAAC,SAA2B,EAAE,OAA4B;QACxE,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,GAAuB,CAAC;QAC5B,IAAI,IAAwB,CAAC;QAC7B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,IAAI,oBAAoB,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACrG,GAAG,GAAG,QAAQ,IAAI,SAAS,CAAC;YAC5B,IAAI,GAAG,SAAS,CAAC;QACrB,CAAC;aAAM,CAAC;YACJ,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAE3B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC;YACtC,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC;YACtC,QAAQ;YACR,IAAI,EAAE,IAAK;YACX,SAAS,EAAE,IAAI;SAClB,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACO,gBAAgB,CAAC,UAAkB,EAAE,OAA4B;QACvE,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAE5E,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;YACtE,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YACjD,OAAO;gBACH,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC;aAC1E,CAAC;QACN,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,CAAC,SAAU,CAAC;QAExC,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;YACrC,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,OAAQ,CAAE,CAAC;QAC9D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,eAAgB,CAAC,cAAc,CAAC,CAAC;QAExD,OAAO;YACH,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;YACjE,SAAS,EAAE,cAAc;SAC5B,CAAC;IACN,CAAC;IAED;;;;OAIG;IACO,gBAAgB,CAAC,OAAgB;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QAEvC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,CAAC,QAAQ,CAAC,SAAS,KAAK,EAAE,CAAC;QAElC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;QAE9C,IAAI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YAC/D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,cAAc,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAE7C,IACI,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,KAAK,QAAQ;YAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,KAAK,cAAc,EAC7D,CAAC;YACC,GAAG,CAAC,KAAK,CACL,mCAAmC,MAAM,UAAU,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,OAAO,cAAc,GAAG,CACtH,CAAC;QACN,CAAC;QAED,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,GAAG,cAAc,CAAC;QAC1D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,GAAG,IAAI,CAAC;QAE5C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,MAAM,CAAC,SAA2B,EAAE,OAA4B;QAClE,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC;QACnG,CAAC;QAED,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,IAAI,oBAAoB,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC;QACtG,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QACxD,CAAC;aAAM,CAAC;YACJ,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,SAAkB,EAAE,OAA+B;QACnF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,SAAU,EAAE,OAAO,CAAC,CAAC;QACjE,IAAI,CAAC;YACD,IAAI,QAAQ,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YACpD,CAAC;YACD,OAAO,QAAQ,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACX,mEAAoE,GAAa,CAAC,OAAO,EAAE,CAC9F,CAAC;QACN,CAAC;IACL,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CACX,6GAA6G,CAChH,CAAC;IACN,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ"}
+{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA6BtD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,GAAG,KAAK,CAAC;IACf,kBAAkB,GAAG,CAAC,CAAC;IACvB,SAAS,CAAW;IACpB,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,cAAc,CAA8B;IAC5C,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAEnF;;;;;;;;;;;;;;;;;;;OAmBG;IACH,YAAY,UAAqC,EAAE;QAC/C,MAAM,EAAE,gBAAgB,EAAE,GAAG,IAAI,EAAE,GAAG,OAAqB,CAAC;QAE5D,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACX,+DAA+D;gBAC3D,6EAA6E,CACpF,CAAC;QACN,CAAC;QAED,EAAE,CACE,IAAI,EACJ,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAC5E,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CAAC,OAAuB;QACtC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAE3B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,GAAG;YACH,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC;YACtC,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,CAAC;YACtC,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CAAC,OAAuB;QAChC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC;QACxF,CAAC;QAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,IAAI,SAAS,CAAC;IACpD,CAAC;IAES,oBAAoB;QAC1B,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,OAA+B;QAC/D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAe,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC;YACD,IAAI,QAAQ,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;YACpD,CAAC;YACD,OAAO,QAAQ,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACX,mEAAoE,GAAa,CAAC,OAAO,EAAE,CAC9F,CAAC;QACN,CAAC;IACL,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CACX,6GAA6G,CAChH,CAAC;IACN,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ"}

package/recoverable_state.d.ts

@@ -0,0 +1,121 @@
+import type { Configuration, CrawleeLogger } from '@crawlee/core';
+export interface RecoverableStatePersistenceOptions {
+    /**
+     * The key under which the state is stored in the KeyValueStore
+     */
+    persistStateKey: string;
+    /**
+     * Flag to enable or disable state persistence
+     */
+    persistenceEnabled?: boolean;
+    /**
+     * The name of the KeyValueStore to use for persistence.
+     * If neither a name nor an id are supplied, the default store will be used.
+     */
+    persistStateKvsName?: string;
+    /**
+     * The identifier of the KeyValueStore to use for persistence.
+     * If neither a name nor an id are supplied, the default store will be used.
+     */
+    persistStateKvsId?: string;
+}
+/**
+ * Options for configuring the RecoverableState
+ */
+export interface RecoverableStateOptions<TStateModel = Record<string, unknown>> extends RecoverableStatePersistenceOptions {
+    /**
+     * The default state used if no persisted state is found.
+     * A deep copy is made each time the state is used.
+     */
+    defaultState: TStateModel;
+    /**
+     * A logger instance for logging operations related to state persistence
+     */
+    logger?: CrawleeLogger;
+    /**
+     * Configuration instance to use
+     */
+    config?: Configuration;
+    /**
+     * Optional function to transform the state to a JSON string before persistence.
+     * If not provided, JSON.stringify will be used.
+     */
+    serialize?: (state: TStateModel) => string;
+    /**
+     * Optional function to transform a JSON-serialized object back to the state model.
+     * If not provided, JSON.parse is used.
+     * It is advisable to perform validation in this function and to throw an exception if it fails.
+     */
+    deserialize?: (serializedState: string) => TStateModel;
+}
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export declare class RecoverableState<TStateModel = Record<string, unknown>> {
+    private readonly defaultState;
+    private state;
+    private readonly persistenceEnabled;
+    private readonly persistStateKey;
+    private readonly persistStateKvsName?;
+    private readonly persistStateKvsId?;
+    private keyValueStore;
+    private readonly log;
+    private readonly serialize;
+    private readonly deserialize;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options: RecoverableStateOptions<TStateModel>);
+    /**
+     * Initialize the recoverable state.
+     *
+     * This method must be called before using the recoverable state. It loads the saved state
+     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     *
+     * @returns The loaded state object
+     */
+    initialize(): Promise<TStateModel>;
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time.
+     */
+    teardown(): Promise<void>;
+    /**
+     * Get the current state.
+     */
+    get currentValue(): TStateModel;
+    /**
+     * Reset the state to the default values and clear any persisted state.
+     *
+     * Resets the current state to the default state and, if persistence is enabled,
+     * clears the persisted state from the KeyValueStore.
+     */
+    reset(): Promise<void>;
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    persistState(eventData?: {
+        isMigrating: boolean;
+    }): Promise<void>;
+    /**
+     * Load the saved state from the KeyValueStore
+     */
+    private loadSavedState;
+}
+//# sourceMappingURL=recoverable_state.d.ts.map

package/recoverable_state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recoverable_state.d.ts","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGlE,MAAM,WAAW,kCAAkC;IAC/C;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB,CACpC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CACvC,SAAQ,kCAAkC;IACxC;;;OAGG;IACH,YAAY,EAAE,WAAW,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,MAAM,CAAC;IAE3C;;;;OAIG;IACH,WAAW,CAAC,EAAE,CAAC,eAAe,EAAE,MAAM,KAAK,WAAW,CAAC;CAC1D;AAED;;;;;;;;;GASG;AACH,qBAAa,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC/D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAc;IAC3C,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;IAC5C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAgB;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiC;IAC3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA2C;IAEvE;;;;OAIG;gBACS,OAAO,EAAE,uBAAuB,CAAC,WAAW,CAAC;IAazD;;;;;;;OAOG;IACG,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IA6BxC;;;;;OAKG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B;;OAEG;IACH,IAAI,YAAY,IAAI,WAAW,CAM9B;IAED;;;;;OAKG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAY5B;;;;;;;OAOG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE;QAAE,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAcvE;;OAEG;YACW,cAAc;CAY/B"}

package/recoverable_state.js

@@ -0,0 +1,142 @@
+import { KeyValueStore, serviceLocator } from '@crawlee/core';
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export class RecoverableState {
+    defaultState;
+    state = null;
+    persistenceEnabled;
+    persistStateKey;
+    persistStateKvsName;
+    persistStateKvsId;
+    keyValueStore = null;
+    log;
+    serialize;
+    deserialize;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options) {
+        this.defaultState = options.defaultState;
+        this.persistStateKey = options.persistStateKey;
+        this.persistenceEnabled = options.persistenceEnabled ?? false;
+        this.persistStateKvsName = options.persistStateKvsName;
+        this.persistStateKvsId = options.persistStateKvsId;
+        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.serialize = options.serialize ?? JSON.stringify;
+        this.deserialize = options.deserialize ?? JSON.parse;
+        this.persistState = this.persistState.bind(this);
+    }
+    /**
+     * Initialize the recoverable state.
+     *
+     * This method must be called before using the recoverable state. It loads the saved state
+     * if persistence is enabled and registers the object to listen for PERSIST_STATE events.
+     *
+     * @returns The loaded state object
+     */
+    async initialize() {
+        if (this.state !== null && this.state !== undefined) {
+            return this.currentValue;
+        }
+        if (!this.persistenceEnabled) {
+            this.state = this.deserialize(this.serialize(this.defaultState));
+            return this.currentValue;
+        }
+        let kvsIdentifier = null;
+        if (this.persistStateKvsName) {
+            kvsIdentifier = { name: this.persistStateKvsName };
+        }
+        else if (this.persistStateKvsId) {
+            kvsIdentifier = { id: this.persistStateKvsId };
+        }
+        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, { config: serviceLocator.getConfiguration() });
+        await this.loadSavedState();
+        // Register for persist state events
+        const eventManager = serviceLocator.getEventManager();
+        eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        return this.currentValue;
+    }
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time.
+     */
+    async teardown() {
+        if (!this.persistenceEnabled || !this.persistState) {
+            return;
+        }
+        const eventManager = serviceLocator.getEventManager();
+        eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        await this.persistState();
+    }
+    /**
+     * Get the current state.
+     */
+    get currentValue() {
+        if (this.state === null) {
+            throw new Error('Recoverable state has not yet been loaded');
+        }
+        return this.state;
+    }
+    /**
+     * Reset the state to the default values and clear any persisted state.
+     *
+     * Resets the current state to the default state and, if persistence is enabled,
+     * clears the persisted state from the KeyValueStore.
+     */
+    async reset() {
+        this.state = this.deserialize(this.serialize(this.defaultState));
+        if (this.persistenceEnabled) {
+            if (this.keyValueStore === null) {
+                throw new Error('Recoverable state has not yet been initialized');
+            }
+            await this.keyValueStore.setValue(this.persistStateKey, null);
+        }
+    }
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    async persistState(eventData) {
+        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        if (this.keyValueStore === null || this.state === null) {
+            throw new Error('Recoverable state has not yet been initialized');
+        }
+        if (this.persistenceEnabled) {
+            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
+                contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
+            });
+        }
+    }
+    /**
+     * Load the saved state from the KeyValueStore
+     */
+    async loadSavedState() {
+        if (this.keyValueStore === null) {
+            throw new Error('Recoverable state has not yet been initialized');
+        }
+        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
+        if (storedState === null || storedState === undefined) {
+            this.state = this.deserialize(this.serialize(this.defaultState));
+        }
+        else {
+            this.state = this.deserialize(storedState);
+        }
+    }
+}
+//# sourceMappingURL=recoverable_state.js.map

package/recoverable_state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AACA,OAAO,EAAa,aAAa,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AA8DzE;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACR,YAAY,CAAc;IACnC,KAAK,GAAuB,IAAI,CAAC;IACxB,kBAAkB,CAAU;IAC5B,eAAe,CAAS;IACxB,mBAAmB,CAAU;IAC7B,iBAAiB,CAAU;IACpC,aAAa,GAAyB,IAAI,CAAC;IAClC,GAAG,CAAgB;IACnB,SAAS,CAAiC;IAC1C,WAAW,CAA2C;IAEvE;;;;OAIG;IACH,YAAY,OAA6C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC9F,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,aAAa,GAA6C,IAAI,CAAC;QAEnE,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,aAAa,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACvD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAChC,aAAa,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAE5G,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,oCAAoC;QACpC,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACtD,YAAY,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,SAAoC;QACnD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChF,WAAW,EAAE,YAAY,EAAE,wHAAwH;aACtJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5E,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,WAAqB,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;CACJ"}