@crawlee/core 4.0.0-beta.4 → 4.0.0-beta.41

package/events/local_event_manager.js

@@ -1,12 +1,25 @@
-import os from 'node:os';
-import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
-import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service_locator.js';
 import { EventManager } from './event_manager.js';
 export class LocalEventManager extends EventManager {
-    previousTicks = { idle: 0, total: 0 };
+    systemInfoIntervalMillis;
+    constructor(options) {
+        super(options);
+        this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+    }
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfig(config) {
+        const resolvedConfig = config ?? serviceLocator.getConfiguration();
+        return new LocalEventManager({
+            persistStateIntervalMillis: resolvedConfig.get('persistStateIntervalMillis'),
+            systemInfoIntervalMillis: resolvedConfig.get('systemInfoIntervalMillis'),
+        });
+    }
     /**
-     * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     async init() {
@@ -14,9 +27,8 @@ export class LocalEventManager extends EventManager {
             return;
         }
         await super.init();
-        const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
         this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
-        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
     }
     /**
      * @inheritDoc
@@ -33,7 +45,7 @@ export class LocalEventManager extends EventManager {
      */
     async emitSystemInfoEvent(intervalCallback) {
         const info = await this.createSystemInfo({
-            maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
+            maxUsedCpuRatio: serviceLocator.getConfiguration().get('maxUsedCpuRatio'),
         });
         this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
         intervalCallback();
@@ -42,17 +54,8 @@ export class LocalEventManager extends EventManager {
      * @internal
      */
     async isContainerizedWrapper() {
-        return this.config.get('containerized', await isContainerized());
-    }
-    getCurrentCpuTicks() {
-        const cpus = os.cpus();
-        return cpus.reduce((acc, cpu) => {
-            const cpuTimes = Object.values(cpu.times);
-            return {
-                idle: acc.idle + cpu.times.idle,
-                total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
-            };
-        }, { idle: 0, total: 0 });
+        const { isContainerized } = await import('@crawlee/utils');
+        return serviceLocator.getConfiguration().get('containerized', await isContainerized());
     }
     /**
      * Creates a SystemInfo object based on local metrics.
@@ -65,18 +68,11 @@ export class LocalEventManager extends EventManager {
         };
     }
     async createCpuInfo(options) {
-        if (this.config.get('systemInfoV2')) {
-            const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
-            return {
-                cpuCurrentUsage: usedCpuRatio * 100,
-                isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
-            };
-        }
-        const ticks = this.getCurrentCpuTicks();
-        const idleTicksDelta = ticks.idle - this.previousTicks.idle;
-        const totalTicksDelta = ticks.total - this.previousTicks.total;
-        const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
-        Object.assign(this.previousTicks, ticks);
+        const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
+        const usedCpuRatio = await getCurrentCpuTicksV2({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
         return {
             cpuCurrentUsage: usedCpuRatio * 100,
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -84,19 +80,17 @@ export class LocalEventManager extends EventManager {
     }
     async createMemoryInfo() {
         try {
-            if (this.config.get('systemInfoV2')) {
-                const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
-                return {
-                    memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
-                };
-            }
-            const memInfo = await getMemoryInfo();
+            const { getMemoryInfo } = await import('@crawlee/utils');
+            const memInfo = await getMemoryInfo({
+                containerized: await this.isContainerizedWrapper(),
+                logger: serviceLocator.getLogger(),
+            });
             return {
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
             };
         }
         catch (err) {
-            log.exception(err, 'Memory snapshot failed.');
+            this.log.exception(err, 'Memory snapshot failed.');
             return {};
         }
     }

package/events/local_event_manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvG,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC3E,OAAO;oBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;iBAC1E,CAAC;YACN,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YACtC,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAuC,MAAM,oBAAoB,CAAC;AAOvF,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,wBAAwB,CAAS;IAEzC,YAAY,OAAiC;QACzC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAsB;QACpC,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAEnE,OAAO,IAAI,iBAAiB,CAAC;YACzB,0BAA0B,EAAE,cAAc,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAC5E,wBAAwB,EAAE,cAAc,CAAC,GAAG,CAAC,0BAA0B,CAAC;SAC3E,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CACzC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EACnC,IAAI,CAAC,wBAAwB,CAChC,CAAC;IACN,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,iBAAiB,CAAC;SAC5E,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC;YAC5C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,EAAE;YAClD,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE;SACrC,CAAC,CAAC;QACH,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;gBAChC,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,EAAE;gBAClD,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE;aACrC,CAAC,CAAC;YACH,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YAC5D,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/index.d.ts

@@ -1,10 +1,10 @@
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
@@ -14,6 +14,7 @@ export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './recoverable_state.js';
 export { PseudoUrl } from '@apify/pseudo_url';
 export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}

package/index.js

@@ -1,10 +1,10 @@
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
@@ -14,5 +14,6 @@ export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './recoverable_state.js';
 export { PseudoUrl } from '@apify/pseudo_url';
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}

package/log.d.ts

@@ -1,3 +1,83 @@
-import log, { Log, Logger, LoggerJson, LoggerOptions, LoggerText, LogLevel } from '@apify/log';
-export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText };
+import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+import type { LoggerOptions } from '@apify/log';
+import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+export type { CrawleeLogger, CrawleeLoggerOptions };
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
+    private options;
+    private readonly warningsLogged;
+    constructor(options?: Partial<CrawleeLoggerOptions>);
+    /**
+     * Core logging method. Subclasses must implement this to dispatch log messages
+     * to the underlying logger (Winston, Pino, console, etc.).
+     *
+     * Level filtering is the responsibility of the underlying library — this method
+     * is called for every message regardless of the current level.
+     *
+     * @param level Crawlee log level (use {@link LogLevel} constants)
+     * @param message The log message
+     * @param data Optional structured data to attach to the log entry
+     */
+    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    /**
+     * Creates a child logger instance. Subclasses must implement this to define
+     * how child loggers are created for the underlying logger.
+     */
+    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    getOptions(): CrawleeLoggerOptions;
+    setOptions(options: Partial<CrawleeLoggerOptions>): void;
+    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    error(message: string, data?: Record<string, unknown>): void;
+    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>): void;
+    warningOnce(message: string): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>): void;
+    deprecated(message: string): void;
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export declare class ApifyLogAdapter extends BaseCrawleeLogger {
+    private readonly apifyLog;
+    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
+    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+}
+export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
+export type { LoggerOptions };
 //# sourceMappingURL=log.d.ts.map

package/log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhF,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,8BAAsB,iBAAkB,YAAW,aAAa;IAC5D,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;gBAExC,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM;IAIvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAE3F;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAErF,UAAU,IAAI,oBAAoB;IAIlC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAIxD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAI5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAQlF,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI/D,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI9D,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAOlC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAGpC;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,iBAAiB;IAE9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,GAAG,EAC9B,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC;IAK3C,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAIlF,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;CAM/E;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAC9D,YAAY,EAAE,aAAa,EAAE,CAAC"}

package/log.js

@@ -1,3 +1,105 @@
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export class BaseCrawleeLogger {
+    options;
+    warningsLogged = new Set();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    getOptions() {
+        return this.options;
+    }
+    setOptions(options) {
+        this.options = { ...this.options, ...options };
+    }
+    child(options) {
+        return this.createChild(options);
+    }
+    error(message, data) {
+        this.logWithLevel(LogLevel.ERROR, message, data);
+    }
+    exception(exception, message, data) {
+        this.logWithLevel(LogLevel.ERROR, `${message}: ${exception.message}`, {
+            ...data,
+            stack: exception.stack,
+            exception,
+        });
+    }
+    softFail(message, data) {
+        this.logWithLevel(LogLevel.SOFT_FAIL, message, data);
+    }
+    warning(message, data) {
+        this.logWithLevel(LogLevel.WARNING, message, data);
+    }
+    warningOnce(message) {
+        if (!this.warningsLogged.has(message)) {
+            this.warningsLogged.add(message);
+            this.warning(message);
+        }
+    }
+    info(message, data) {
+        this.logWithLevel(LogLevel.INFO, message, data);
+    }
+    debug(message, data) {
+        this.logWithLevel(LogLevel.DEBUG, message, data);
+    }
+    perf(message, data) {
+        this.logWithLevel(LogLevel.PERF, `[PERF] ${message}`, data);
+    }
+    deprecated(message) {
+        this.warningOnce(`[DEPRECATED] ${message}`);
+    }
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export class ApifyLogAdapter extends BaseCrawleeLogger {
+    apifyLog;
+    constructor(apifyLog, options) {
+        super(options ?? {});
+        this.apifyLog = apifyLog;
+    }
+    logWithLevel(level, message, data) {
+        this.apifyLog.internal(level, message, data);
+    }
+    createChild(options) {
+        return new ApifyLogAdapter(this.apifyLog.child({ prefix: options.prefix ?? null }), {
+            ...this.getOptions(),
+            ...options,
+        });
+    }
+}
 export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
 //# sourceMappingURL=log.js.map

package/log.js.map

@@ -1 +1 @@
-{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAiB,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAiB,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAGA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAIhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAgB,iBAAiB;IAC3B,OAAO,CAAuB;IACrB,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpD,YAAY,UAAyC,EAAE;QACnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAqBD,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,UAAU,CAAC,OAAsC;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAsC;QACxC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,SAAS,CAAC,SAAgB,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,KAAK,SAAS,CAAC,OAAO,EAAE,EAAE;YAClE,GAAG,IAAI;YACP,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,SAAS;SACZ,CAAC,CAAC;IACP,CAAC;IAED,QAAQ,CAAC,OAAe,EAAE,IAA8B;QACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,OAAe,EAAE,IAA8B;QACnD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,WAAW,CAAC,OAAe;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,UAAU,CAAC,OAAe;QACtB,IAAI,CAAC,WAAW,CAAC,gBAAgB,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;CACJ;AAED;;;;;GAKG;AACH,MAAM,OAAO,eAAgB,SAAQ,iBAAiB;IAE7B;IADrB,YACqB,QAAa,EAC9B,OAAuC;QAEvC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAHJ,aAAQ,GAAR,QAAQ,CAAK;IAIlC,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7D,CAAC;IAES,WAAW,CAAC,OAAsC;QACxD,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,EAAE;YAChF,GAAG,IAAI,CAAC,UAAU,EAAE;YACpB,GAAG,OAAO;SACb,CAAC,CAAC;IACP,CAAC;CACJ;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.4",
+    "version": "4.0.0-beta.41",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,20 +53,19 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.4",
-        "@crawlee/types": "4.0.0-beta.4",
-        "@crawlee/utils": "4.0.0-beta.4",
+        "@crawlee/memory-storage": "4.0.0-beta.41",
+        "@crawlee/types": "4.0.0-beta.41",
+        "@crawlee/utils": "4.0.0-beta.41",
         "@sapphire/async-queue": "^1.5.5",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "json5": "^2.2.3",
         "minimatch": "^10.0.1",
         "ow": "^2.0.0",
         "stream-json": "^1.9.1",
         "tldts": "^7.0.6",
-        "tough-cookie": "^5.1.2",
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
     },
@@ -77,5 +76,5 @@
             }
         }
     },
-    "gitHead": "3880cdebbcddf9e2a242bc89751e36e2ef5ae897"
+    "gitHead": "0f3d1a433e1a27647d99ad5dc81a605e82505d1a"
 }

package/proxy_configuration.d.ts

@@ -1,18 +1,20 @@
+import type { ProxyInfo } from '@crawlee/types';
 import type { Request } from './request.js';
 export interface ProxyConfigurationFunction {
-    (sessionId: string | number, options?: {
+    (options?: {
         request?: Request;
     }): string | null | Promise<string | null>;
 }
+type UrlList = (string | null)[];
 export interface ProxyConfigurationOptions {
     /**
      * An array of custom proxy URLs to be rotated.
      * Custom proxies are not compatible with Apify Proxy and an attempt to use both
      * configuration options will cause an error to be thrown on initialize.
      */
-    proxyUrls?: string[];
+    proxyUrls?: UrlList;
     /**
-     * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter and an optional parameter with the `Request` object when applicable.
+     * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
      * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
      *
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
@@ -29,72 +31,12 @@ export interface ProxyConfigurationOptions {
      *
      * Use `null` as a proxy URL to disable the proxy for the given tier.
      */
-    tieredProxyUrls?: (string | null)[][];
+    tieredProxyUrls?: UrlList[];
 }
 export interface TieredProxy {
     proxyUrl: string | null;
     proxyTier?: number;
 }
-/**
- * The main purpose of the ProxyInfo object is to provide information
- * about the current proxy connection used by the crawler for the request.
- * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
- *
- * **Example usage:**
- *
- * ```javascript
- * const proxyConfiguration = new ProxyConfiguration({
- *   proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
- * });
- *
- * // Getting proxyInfo object by calling class method directly
- * const proxyInfo = await proxyConfiguration.newProxyInfo();
- *
- * // In crawler
- * const crawler = new CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   requestHandler({ proxyInfo }) {
- *      // Getting used proxy URL
- *       const proxyUrl = proxyInfo.url;
- *
- *      // Getting ID of used Session
- *       const sessionIdentifier = proxyInfo.sessionId;
- *   }
- * })
- *
- * ```
- */
-export interface ProxyInfo {
-    /**
-     * The identifier of used {@link Session}, if used.
-     */
-    sessionId?: string;
-    /**
-     * The URL of the proxy.
-     */
-    url: string;
-    /**
-     * Username for the proxy.
-     */
-    username?: string;
-    /**
-     * User's password for the proxy.
-     */
-    password: string;
-    /**
-     * Hostname of your proxy.
-     */
-    hostname: string;
-    /**
-     * Proxy port.
-     */
-    port: number | string;
-    /**
-     * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
-     */
-    proxyTier?: number;
-}
 interface TieredProxyOptions {
     request?: Request;
     proxyTier?: number;
@@ -156,12 +98,12 @@ declare class ProxyTierTracker {
 export declare class ProxyConfiguration {
     isManInTheMiddle: boolean;
     protected nextCustomUrlIndex: number;
-    protected proxyUrls?: string[];
-    protected tieredProxyUrls?: (string | null)[][];
-    protected usedProxyUrls: Map<string, string>;
+    protected proxyUrls?: UrlList;
+    protected tieredProxyUrls?: UrlList[];
+    protected usedProxyUrls: Map<string, string | null>;
     protected newUrlFunction?: ProxyConfigurationFunction;
 // @ts-ignore optional peer dependency or compatibility with es2022
-    protected log: import("@apify/log").Log;
+    protected log: import("@crawlee/types").CrawleeLogger;
     protected domainTiers: Map<string, ProxyTierTracker>;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
@@ -190,25 +132,16 @@ export declare class ProxyConfiguration {
      * the currently used proxy via the requestHandler parameter `proxyInfo`.
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
      *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(options?: TieredProxyOptions): Promise<ProxyInfo | undefined>;
     /**
-     * Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
-     * @param _sessionId Session identifier
+     * Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
      * @param options Options for the tiered proxy rotation
      * @returns An object with the proxy URL and the proxy tier used.
      */
-    protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): TieredProxy;
+    protected _handleTieredUrl(options?: TieredProxyOptions): TieredProxy;
     /**
      * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
      *
@@ -216,27 +149,17 @@ export declare class ProxyConfiguration {
      */
     protected predictProxyTier(request: Request): number | null;
     /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
+     * Returns a new proxy URL based on provided configuration options.
      *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string | undefined>;
-    /**
-     * Handles custom url rotation with session
-     */
-    protected _handleCustomUrl(sessionId?: string): string;
+    newUrl(options?: TieredProxyOptions): Promise<string | undefined>;
+    protected _handleProxyUrlsList(): string | null;
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    protected _callNewUrlFunction(sessionId?: string, options?: {
+    protected _callNewUrlFunction(options?: {
         request?: Request;
     }): Promise<string | null>;
     protected _throwCannotCombineCustomMethods(): never;

package/proxy_configuration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAE5C;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;CACzC;AAED,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,kBAAkB;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,cAAM,gBAAgB;IAClB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,WAAW,CAAS;gBAEhB,eAAe,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE;IAKhD;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB;;;OAGG;IACH,WAAW;CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IAC3B,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAChD,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAC5D,SAAS,CAAC,WAAW,gCAAuC;IAE5D;;;;;;;;;;;;;;;;;;;OAmBG;gBACS,OAAO,GAAE,yBAA8B;IAwBnD;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IA4B7G;;;;;OAKG;IACH,SAAS,CAAC,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,WAAW;IAwBzF;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAiC3D;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAcpG;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE;IAcvF,SAAS,CAAC,gCAAgC,IAAI,KAAK;IAMnD,SAAS,CAAC,uBAAuB,IAAI,KAAK;CAG7C"}
+{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,WAAW,0BAA0B;IACvC,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7E;AAED,KAAK,OAAO,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;AAEjC,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAE5C;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,OAAO,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,kBAAkB;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,cAAM,gBAAgB;IAClB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,WAAW,CAAS;gBAEhB,eAAe,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE;IAKhD;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB;;;OAGG;IACH,WAAW;CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IAC3B,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,eAAe,CAAC,EAAE,OAAO,EAAE,CAAC;IACtC,SAAS,CAAC,aAAa,6BAAoC;IAC3D,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,yCAAsE;IACnF,SAAS,CAAC,WAAW,gCAAuC;IAE5D;;;;;;;;;;;;;;;;;;;OAmBG;gBACS,OAAO,GAAE,yBAA8B;IAwBnD;;;;;;;;OAQG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAyBhF;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,WAAW;IAwBrE;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAiC3D;;;;;OAKG;IACG,MAAM,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAYvE,SAAS,CAAC,oBAAoB,IAAI,MAAM,GAAG,IAAI;IAI/C;;OAEG;cACa,mBAAmB,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE;IAcnE,SAAS,CAAC,gCAAgC,IAAI,KAAK;IAMnD,SAAS,CAAC,uBAAuB,IAAI,KAAK;CAG7C"}