@crawlee/core 4.0.0-beta.3 → 4.0.0-beta.31

package/configuration.js

@@ -1,11 +1,9 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { MemoryStorage } from '@crawlee/memory-storage';
 import { pathExistsSync } from 'fs-extra/esm';
-import log, { LogLevel } from '@apify/log';
-import { LocalEventManager } from './events/local_event_manager.js';
+import { log, LogLevel } from './log.js';
+import { serviceLocator } from './service_locator.js';
 import { entries } from './typedefs.js';
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
@@ -35,7 +33,7 @@ import { entries } from './typedefs.js';
  * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
  * // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ ... }, config);
+ * const crawler = new BasicCrawler({ configuration: config, ... });
  * ```
  *
  * The configuration provided via environment variables always takes precedence. We can also
@@ -70,7 +68,6 @@ import { entries } from './typedefs.js';
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
  * `containerized | `CRAWLEE_CONTAINERIZED | -
  */
 export class Configuration {
@@ -93,7 +90,6 @@ export class Configuration {
         CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
         CRAWLEE_LOG_LEVEL: 'logLevel',
         CRAWLEE_PERSIST_STORAGE: 'persistStorage',
-        CRAWLEE_SYSTEM_INFO_V2: 'systemInfoV2',
         CRAWLEE_CONTAINERIZED: 'containerized',
     };
     static BOOLEAN_VARS = [
@@ -102,7 +98,6 @@ export class Configuration {
         'xvfb',
         'disableBrowserSandbox',
         'persistStorage',
-        'systemInfoV2',
         'containerized',
     ];
     static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
@@ -120,18 +115,8 @@ export class Configuration {
         persistStateIntervalMillis: 60_000,
         systemInfoIntervalMillis: 1_000,
         persistStorage: true,
-        systemInfoV2: false,
     };
-    /**
-     * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
-     * @internal
-     */
-    static storage = new AsyncLocalStorage();
     options;
-    services = new Map();
-    /** @internal */
-    static globalConfig;
-    storageManagers = new Map();
     /**
      * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
      */
@@ -198,94 +183,13 @@ export class Configuration {
     set(key, value) {
         this.options.set(key, value);
     }
-    /**
-     * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
-     */
-    static set(key, value) {
-        this.getGlobalConfig().set(key, value);
-    }
-    /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
-     * return the same client instance.
-     *
-     * Caching works based on the `storageClientOptions`, so calling this method with different options will return
-     * multiple instances, one for each variant of the options.
-     * @internal
-     */
-    getStorageClient() {
-        if (this.options.has('storageClient')) {
-            return this.options.get('storageClient');
-        }
-        const options = this.options.get('storageClientOptions');
-        return this.createMemoryStorage(options);
-    }
-    getEventManager() {
-        if (this.options.has('eventManager')) {
-            return this.options.get('eventManager');
-        }
-        if (this.services.has('eventManager')) {
-            return this.services.get('eventManager');
-        }
-        const eventManager = new LocalEventManager(this);
-        this.services.set('eventManager', eventManager);
-        return eventManager;
-    }
-    /**
-     * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
-     * @internal
-     */
-    createMemoryStorage(options = {}) {
-        const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
-        if (this.services.has(cacheKey)) {
-            return this.services.get(cacheKey);
-        }
-        const storage = new MemoryStorage({
-            persistStorage: this.get('persistStorage'),
-            // Override persistStorage if user provides it via storageClientOptions
-            ...options,
-        });
-        this.services.set(cacheKey, storage);
-        return storage;
-    }
-    useStorageClient(client) {
-        this.options.set('storageClient', client);
-    }
-    static useStorageClient(client) {
-        this.getGlobalConfig().useStorageClient(client);
-    }
-    useEventManager(events) {
-        this.options.set('eventManager', events);
-    }
     /**
      * Returns the global configuration instance. It will respect the environment variables.
+     *
+     * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
     static getGlobalConfig() {
-        if (Configuration.storage.getStore()) {
-            return Configuration.storage.getStore();
-        }
-        Configuration.globalConfig ??= new Configuration();
-        return Configuration.globalConfig;
-    }
-    /**
-     * Gets default {@link StorageClient} instance.
-     */
-    static getStorageClient() {
-        return this.getGlobalConfig().getStorageClient();
-    }
-    /**
-     * Gets default {@link EventManager} instance.
-     */
-    static getEventManager() {
-        return this.getGlobalConfig().getEventManager();
-    }
-    /**
-     * Resets global configuration instance. The default instance holds configuration based on env vars,
-     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
-     */
-    static resetGlobalState() {
-        delete this.globalConfig;
+        return serviceLocator.getConfiguration();
     }
     buildOptions(options) {
         // try to load configuration from crawlee.json as the baseline

package/configuration.js.map

@@ -1 +1 @@
-{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAoB,OAAO,EAAE,MAAM,eAAe,CAAC;AAuK1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,MAAM,OAAO,aAAa;IACtB;;OAEG;IACO,MAAM,CAAC,OAAO,GAAe;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,gBAAgB;QACzC,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,eAAe;KACzC,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG;QAC5B,cAAc;QACd,UAAU;QACV,MAAM;QACN,uBAAuB;QACvB,gBAAgB;QAChB,cAAc;QACd,eAAe;KAClB,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;IAEjG,MAAM,CAAC,yBAAyB,GAAa,EAAE,CAAC;IAEhD,MAAM,CAAC,QAAQ,GAAe;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,MAAM;QAClC,wBAAwB,EAAE,KAAK;QAC/B,cAAc,EAAE,IAAI;QACpB,YAAY,EAAE,KAAK;KACtB,CAAC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,GAAG,IAAI,iBAAiB,EAAiB,CAAC;IAE9C,OAAO,CAAqF;IAC5F,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAiB;IAEpB,eAAe,GAAG,IAAI,GAAG,EAA+B,CAAC;IAEzE;;OAEG;IACH,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,YAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;gBACpC,CAAC,CAAC,CAAC,QAAQ;gBACX,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACtE,GAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACZ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;QAClD,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACtC,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,KAAK,CAAC;QAClB,CAAC;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,aAAa,CAAC,yBAAyB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,KAAK;gBAAE,OAAO,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,KAAK,CAAC;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAA+B,EAAE,KAAW;QACnD,IAAI,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;QAC9D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC7D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;YAC9B,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAC1C,uEAAuE;YACvE,GAAG,OAAO;SACb,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,MAAqB;QACzC,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;QAC7C,CAAC;QAED,aAAa,CAAC,YAAY,KAAK,IAAI,aAAa,EAAE,CAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;QACL,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC"}
+{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AA8IxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,aAAa;IACtB;;OAEG;IACO,MAAM,CAAC,OAAO,GAAe;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,gBAAgB;QACzC,qBAAqB,EAAE,eAAe;KACzC,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG;QAC5B,cAAc;QACd,UAAU;QACV,MAAM;QACN,uBAAuB;QACvB,gBAAgB;QAChB,eAAe;KAClB,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;IAEjG,MAAM,CAAC,yBAAyB,GAAa,EAAE,CAAC;IAEhD,MAAM,CAAC,QAAQ,GAAe;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,MAAM;QAClC,wBAAwB,EAAE,KAAK;QAC/B,cAAc,EAAE,IAAI;KACvB,CAAC;IAEQ,OAAO,CAAqF;IAEtG;;OAEG;IACH,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,YAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;gBACpC,CAAC,CAAC,CAAC,QAAQ;gBACX,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACtE,GAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACZ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;QAClD,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACtC,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,KAAK,CAAC;QAClB,CAAC;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,aAAa,CAAC,yBAAyB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,KAAK;gBAAE,OAAO,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,KAAK,CAAC;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,cAAc,CAAC,gBAAgB,EAAE,CAAC;IAC7C,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;QACL,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC"}

package/cookie_utils.d.ts

@@ -7,7 +7,7 @@ export interface ResponseLike {
 /**
  * @internal
  */
-export declare function getCookiesFromResponse(response: ResponseLike): Cookie[];
+export declare function getCookiesFromResponse(response: Response): Cookie[];
 /**
  * Calculate cookie expiration date
  * @param maxAgeSecs

package/cookie_utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,YAAY;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,EAAE,CAWvE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAehF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAiB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA8BzE"}
+{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,YAAY;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE,CASnE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAehF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAiB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAgCzE"}

package/cookie_utils.js

@@ -1,19 +1,17 @@
 import { Cookie, CookieJar } from 'tough-cookie';
-import { log } from './log.js';
+import { serviceLocator } from './service_locator.js';
 import { CookieParseError } from './session_pool/errors.js';
 /**
  * @internal
  */
 export function getCookiesFromResponse(response) {
-    const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
-    const cookieHeader = headers?.['set-cookie'] || '';
+    const headers = response.headers;
+    const cookieHeaders = headers.getSetCookie();
     try {
-        return Array.isArray(cookieHeader)
-            ? cookieHeader.map((cookie) => Cookie.parse(cookie))
-            : [Cookie.parse(cookieHeader)];
+        return cookieHeaders.map((cookie) => Cookie.parse(cookie));
     }
     catch (e) {
-        throw new CookieParseError(cookieHeader);
+        throw new CookieParseError(cookieHeaders);
     }
 }
 /**
@@ -103,7 +101,9 @@ export function mergeCookies(url, sourceCookies) {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });
             if (similarKeyCookie) {
-                log.warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
             }
             jar.setCookieSync(cookie, url);
         }

package/cookie_utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAO5D;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAsB;IACzD,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEnD,IAAI,CAAC;QACD,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EACH,WAAW,CAAC,OAAO,IAAI,IAAI,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU;YAC7D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI;QAC9C,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;QAClG,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc;QAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAC3F,OAAO,IAAI,MAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,QAAQ,EAAE,CAAC,mBAAmB;KACjC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;QAC7C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACnB,GAAG,CAAC,WAAW,CACX,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CACzG,CAAC;YACN,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC"}
+{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAO5D;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAkB;IACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IACjC,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,IAAI,CAAC;QACD,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EACH,WAAW,CAAC,OAAO,IAAI,IAAI,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU;YAC7D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI;QAC9C,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;QAClG,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc;QAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAC3F,OAAO,IAAI,MAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,QAAQ,EAAE,CAAC,mBAAmB;KACjC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;QAC7C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACnB,cAAc;qBACT,SAAS,EAAE;qBACX,WAAW,CACR,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CACzG,CAAC;YACV,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC"}

package/crawlers/context_pipeline.d.ts

@@ -0,0 +1,61 @@
+import type { Awaitable } from '@crawlee/types';
+/**
+ * Represents a middleware step in the context pipeline.
+ *
+ * @template TCrawlingContext - The input context type for this middleware
+ * @template TCrawlingContextExtension - The enhanced output context type
+ */
+export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
+    /** The main middleware function that enhances the context */
+    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
+    /** Optional cleanup function called after the consumer finishes or fails */
+    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
+}
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
+    /**
+     * Adds a middleware to the pipeline, creating a new pipeline instance.
+     *
+     * This method provides a fluent interface for building context transformation pipelines.
+     * Each middleware can enhance the context with additional properties or utilities.
+     *
+     * @template TCrawlingContextExtension - The enhanced context type produced by this middleware
+     * @param middleware - The middleware to add to the pipeline
+     * @returns A new ContextPipeline instance with the added middleware
+     */
+    abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+    /**
+     * Executes the middleware pipeline and passes the final context to a consumer function.
+     *
+     * This method runs the crawling context through the entire middleware chain, enhancing it
+     * at each step, and then passes the final enhanced context to the provided consumer function.
+     * Proper cleanup is performed even if exceptions occur during processing.
+     *
+     * @param crawlingContext - The initial context to process through the pipeline
+     * @param finalContextConsumer - The function that will receive the final enhanced context
+     *
+     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
+     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
+     * @throws {RequestHandlerError} When the final context consumer throws an exception
+     * @throws {ContextPipelineCleanupError} When cleanup operations fail
+     * @throws {SessionError} Session errors are re-thrown as-is for special handling
+     */
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+}
+//# sourceMappingURL=context_pipeline.d.ts.map

package/crawlers/context_pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context_pipeline.d.ts","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAUhD;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB;IAC1E,6DAA6D;IAC7D,MAAM,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,SAAS,CAAC,yBAAyB,CAAC,CAAC;IAC5E,4EAA4E;IAC5E,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,GAAG,yBAAyB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;CACzG;AAED;;;;;;;;;GASG;AACH,8BAAsB,eAAe,CAAC,YAAY,EAAE,gBAAgB,SAAS,YAAY;IACrF;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,YAAY,KAAK,eAAe,CAAC,YAAY,EAAE,YAAY,CAAC;IAI1E;;;;;;;;;OASG;IACH,QAAQ,CAAC,OAAO,CAAC,yBAAyB,EACtC,UAAU,EAAE,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB,CAAC,GAC3E,eAAe,CAAC,YAAY,EAAE,gBAAgB,GAAG,yBAAyB,CAAC;IAE9E;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,IAAI,CACT,eAAe,EAAE,YAAY,EAC7B,oBAAoB,EAAE,CAAC,YAAY,EAAE,gBAAgB,KAAK,SAAS,CAAC,OAAO,CAAC,GAC7E,OAAO,CAAC,IAAI,CAAC;CACnB"}

package/crawlers/context_pipeline.js

@@ -0,0 +1,99 @@
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export class ContextPipeline {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create() {
+        return new ContextPipelineImpl({ action: async (context) => context });
+    }
+}
+/**
+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
+ * properties from the `ContextPipeline` interface, making type checking more reliable.
+ */
+class ContextPipelineImpl extends ContextPipeline {
+    middleware;
+    parent;
+    constructor(middleware, parent) {
+        super();
+        this.middleware = middleware;
+        this.parent = parent;
+    }
+    /**
+     * @inheritdoc
+     */
+    compose(middleware) {
+        return new ContextPipelineImpl(middleware, this);
+    }
+    *middlewareChain() {
+        let step = this;
+        while (step !== undefined) {
+            yield step.middleware;
+            step = step.parent;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async call(crawlingContext, finalContextConsumer) {
+        const middlewares = Array.from(this.middlewareChain()).reverse();
+        const cleanupStack = [];
+        let consumerException;
+        try {
+            for (const { action, cleanup } of middlewares) {
+                try {
+                    const contextExtension = await action(crawlingContext);
+                    Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));
+                    if (cleanup) {
+                        cleanupStack.push(cleanup);
+                    }
+                }
+                catch (exception) {
+                    if (exception instanceof SessionError) {
+                        throw exception; // Session errors are re-thrown as-is
+                    }
+                    if (exception instanceof ContextPipelineInterruptedError) {
+                        throw exception;
+                    }
+                    throw new ContextPipelineInitializationError(exception);
+                }
+            }
+            try {
+                await finalContextConsumer(crawlingContext);
+            }
+            catch (exception) {
+                if (exception instanceof SessionError) {
+                    consumerException = exception;
+                    throw exception; // Session errors are re-thrown as-is
+                }
+                consumerException = exception;
+                throw new RequestHandlerError(exception);
+            }
+        }
+        finally {
+            try {
+                for (const cleanup of cleanupStack.reverse()) {
+                    await cleanup(crawlingContext, consumerException);
+                }
+            }
+            catch (exception) {
+                // eslint-disable-next-line no-unsafe-finally
+                throw new ContextPipelineCleanupError(exception);
+            }
+        }
+    }
+}
+//# sourceMappingURL=context_pipeline.js.map

package/crawlers/context_pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context_pipeline.js","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,2BAA2B,EAC3B,kCAAkC,EAClC,+BAA+B,EAC/B,mBAAmB,EACnB,YAAY,GACf,MAAM,cAAc,CAAC;AAetB;;;;;;;;;GASG;AACH,MAAM,OAAgB,eAAe;IACjC;;;;;OAKG;IACH,MAAM,CAAC,MAAM;QACT,OAAO,IAAI,mBAAmB,CAA6B,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;CAoCJ;AAED;;;GAGG;AACH,MAAM,mBAAyE,SAAQ,eAGtF;IAEe;IACA;IAFZ,YACY,UAA6D,EAC7D,MAAwD;QAEhE,KAAK,EAAE,CAAC;QAHA,eAAU,GAAV,UAAU,CAAmD;QAC7D,WAAM,GAAN,MAAM,CAAkD;IAGpE,CAAC;IAED;;OAEG;IACH,OAAO,CACH,UAA0E;QAE1E,OAAO,IAAI,mBAAmB,CAC1B,UAAiB,EACjB,IAAW,CACd,CAAC;IACN,CAAC;IAEO,CAAC,eAAe;QACpB,IAAI,IAAI,GAAgE,IAAW,CAAC;QAEpF,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACN,eAA6B,EAC7B,oBAA0E;QAE1E,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QACjE,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,iBAAsC,CAAC;QAE3C,IAAI,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACD,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;oBACvD,MAAM,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,CAAC;oBAE7F,IAAI,OAAO,EAAE,CAAC;wBACV,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC/B,CAAC;gBACL,CAAC;gBAAC,OAAO,SAAkB,EAAE,CAAC;oBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;wBACpC,MAAM,SAAS,CAAC,CAAC,qCAAqC;oBAC1D,CAAC;oBACD,IAAI,SAAS,YAAY,+BAA+B,EAAE,CAAC;wBACvD,MAAM,SAAS,CAAC;oBACpB,CAAC;oBAED,MAAM,IAAI,kCAAkC,CAAC,SAAS,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,oBAAoB,CAAC,eAAmC,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;oBACpC,iBAAiB,GAAG,SAAS,CAAC;oBAC9B,MAAM,SAAS,CAAC,CAAC,qCAAqC;gBAC1D,CAAC;gBACD,iBAAiB,GAAG,SAAS,CAAC;gBAC9B,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAC7C,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC;gBACD,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC3C,MAAM,OAAO,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,6CAA6C;gBAC7C,MAAM,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;IACL,CAAC;CACJ"}

package/crawlers/crawler_commons.d.ts

@@ -1,11 +1,8 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { OptionsInit, Response as GotResponse } from 'got-scraping';
-import type { ReadonlyDeep } from 'type-fest';
+import type { Dictionary, HttpRequestOptions, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
-import type { Log } from '../log.js';
-import type { ProxyInfo } from '../proxy_configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import type { Request, Source } from '../request.js';
 import type { Session } from '../session_pool/session.js';
 import type { Dataset } from '../storages/dataset.js';
@@ -22,7 +19,7 @@ export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl
 export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
     request: LoadedRequest<Context['request']>;
 } & Omit<Context, 'request'>;
-export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
     id: string;
     session?: Session;
     /**
@@ -66,7 +63,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      *
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      */
-    enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;
+    enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>>) => Promise<unknown>;
     /**
      * Add requests directly to the request queue.
      *
@@ -85,10 +82,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
     /**
      * A preconfigured logger for the request handler.
      */
-    log: Log;
+    log: CrawleeLogger;
 }
-export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    crawler: Crawler;
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     /**
      * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
      * currently used by the crawler.
@@ -114,14 +110,9 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * @param [options] All `enqueueLinks()` parameters are passed via an options object.
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestQueue' | 'robotsTxtFile'>): Promise<unknown>;
     /**
-     * Get a key-value store with given name or id, or the default one for the crawler.
-     */
-    getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;
-    /**
-     * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
-     * options on the fly.
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *
      * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
      * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
@@ -136,7 +127,11 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
      * },
      * ```
      */
-    sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+    /**
+     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     */
+    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
 }
 /**
  * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
@@ -149,7 +144,6 @@ export declare class RequestHandlerResult {
     private _keyValueStoreChanges;
     private pushDataCalls;
     private addRequestsCalls;
-    private enqueueLinksCalls;
     constructor(config: Configuration, crawleeStateKey: string);
     /**
      * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
@@ -157,7 +151,6 @@ export declare class RequestHandlerResult {
     get calls(): ReadonlyDeep<{
         pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
         addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-        enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
     }>;
     /**
      * A record of changes made to key-value stores by a request handler.
@@ -188,7 +181,6 @@ export declare class RequestHandlerResult {
         label?: string;
     }[]>;
     pushData: RestrictedCrawlingContext['pushData'];
-    enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
     addRequests: RestrictedCrawlingContext['addRequests'];
     useState: RestrictedCrawlingContext['useState'];
     getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];

package/crawlers/crawler_commons.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAEpF,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAE/E,SAAQ,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,OAAO,CAAC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtG;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CACxF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAC9G,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAWzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAX3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;IAEtF,OAAO,CAAC,iBAAiB,CAA+D;gBAG5E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,YAAY,EAAE,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAMD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAsBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,YAAY,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAErD;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}
+{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAEpF,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IAC/E,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,KACtG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtB;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,eAAe,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAClH;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,GACnG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,eAAe,CAAC,GAChE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB;;;;;;;;;;;;;;;OAeG;IACH,WAAW,EAAE,CACT,gBAAgB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,EAC9C,gBAAgB,CAAC,EAAE,kBAAkB,KACpC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;CAClE;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IASzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAT3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;gBAG1E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC,CAKD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAkBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}

package/crawlers/crawler_commons.js

@@ -10,7 +10,6 @@ export class RequestHandlerResult {
     _keyValueStoreChanges = {};
     pushDataCalls = [];
     addRequestsCalls = [];
-    enqueueLinksCalls = [];
     constructor(config, crawleeStateKey) {
         this.config = config;
         this.crawleeStateKey = crawleeStateKey;
@@ -22,7 +21,6 @@ export class RequestHandlerResult {
         return {
             pushData: this.pushDataCalls,
             addRequests: this.addRequestsCalls,
-            enqueueLinks: this.enqueueLinksCalls,
         };
     }
     /**
@@ -42,9 +40,6 @@ export class RequestHandlerResult {
      */
     get enqueuedUrls() {
         const result = [];
-        for (const [options] of this.enqueueLinksCalls) {
-            result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
-        }
         for (const [requests] of this.addRequestsCalls) {
             for (const request of requests) {
                 if (typeof request === 'object' &&
@@ -78,9 +73,6 @@ export class RequestHandlerResult {
     pushData = async (data, datasetIdOrName) => {
         this.pushDataCalls.push([data, datasetIdOrName]);
     };
-    enqueueLinks = async (options) => {
-        this.enqueueLinksCalls.push([options]);
-    };
     addRequests = async (requests, options = {}) => {
         this.addRequestsCalls.push([requests, options]);
     };

package/crawlers/crawler_commons.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;AA6JnF;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAWjB;IACA;IAXJ,qBAAqB,GACzB,EAAE,CAAC;IAEC,aAAa,GAAwD,EAAE,CAAC;IAExE,gBAAgB,GAA2D,EAAE,CAAC;IAE9E,iBAAiB,GAA4D,EAAE,CAAC;IAExF,YACY,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAKL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;YAClC,YAAY,EAAE,IAAI,CAAC,iBAAiB;SACvC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,QAAQ,GAA0C,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;QAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,YAAY,GAA8C,KAAK,EAAE,OAAO,EAAE,EAAE;QACxE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC;IAEF,WAAW,GAA6C,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;QACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,QAAQ,GAA0C,KAAK,EAAE,YAAY,EAAE,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7E,CAAC,CAAC;IAEF,gBAAgB,GAAkD,KAAK,EAAE,QAAQ,EAAE,EAAE;QACjF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1E,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAC9B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;YACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/C,CAAC;IACN,CAAC,CAAC;IAEM,WAAW,GAAG,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEnG,4BAA4B,GAAG,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;QACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;IACpE,CAAC,CAAC;IAEM,4BAA4B,GAAG,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;QACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;CACL"}
+{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;AA6JnF;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IASjB;IACA;IATJ,qBAAqB,GACzB,EAAE,CAAC;IAEC,aAAa,GAAwD,EAAE,CAAC;IAExE,gBAAgB,GAA2D,EAAE,CAAC;IAEtF,YACY,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAIL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;SACrC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,QAAQ,GAA0C,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;QAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,WAAW,GAA6C,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;QACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,QAAQ,GAA0C,KAAK,EAAE,YAAY,EAAE,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7E,CAAC,CAAC;IAEF,gBAAgB,GAAkD,KAAK,EAAE,QAAQ,EAAE,EAAE;QACjF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1E,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAC9B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;YACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/C,CAAC;IACN,CAAC,CAAC;IAEM,WAAW,GAAG,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEnG,4BAA4B,GAAG,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;QACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;IACpE,CAAC,CAAC;IAEM,4BAA4B,GAAG,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;QACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;CACL"}

package/crawlers/error_snapshotter.d.ts

@@ -1,6 +1,7 @@
 import type { CrawlingContext } from '../crawlers/crawler_commons.js';
 import type { KeyValueStore } from '../storages/key_value_store.js';
 import type { ErrnoException } from './error_tracker.js';
+import type { SnapshottableProperties } from './internals/types.js';
 interface BrowserCrawlingContext {
     saveSnapshot: (options: {
         key: string;
@@ -39,7 +40,7 @@ export declare class ErrorSnapshotter {
     /**
      * Capture a snapshot of the error context.
      */
-    captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot>;
+    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
     /**
      * Captures a snapshot of the current page using the context.saveSnapshot function.
      * This function is applicable for browser contexts only.
@@ -49,7 +50,7 @@ export declare class ErrorSnapshotter {
     /**
      * Save the HTML snapshot of the page, and return the fileName with the extension.
      */
-    saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
+    saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
     /**
      * Generate a unique fileName for each error snapshot.
      */