@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.60

package/events/local_event_manager.js

@@ -1,12 +1,25 @@
-import os from 'node:os';
-import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
-import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service_locator.js';
 import { EventManager } from './event_manager.js';
 export class LocalEventManager extends EventManager {
-    previousTicks = { idle: 0, total: 0 };
+    systemInfoIntervalMillis;
+    constructor(options) {
+        super(options);
+        this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+    }
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfig(config) {
+        const resolvedConfig = config ?? serviceLocator.getConfiguration();
+        return new LocalEventManager({
+            persistStateIntervalMillis: resolvedConfig.persistStateIntervalMillis,
+            systemInfoIntervalMillis: resolvedConfig.systemInfoIntervalMillis,
+        });
+    }
     /**
-     * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     async init() {
@@ -14,9 +27,8 @@ export class LocalEventManager extends EventManager {
             return;
         }
         await super.init();
-        const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
         this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
-        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
     }
     /**
      * @inheritDoc
@@ -33,7 +45,7 @@ export class LocalEventManager extends EventManager {
      */
     async emitSystemInfoEvent(intervalCallback) {
         const info = await this.createSystemInfo({
-            maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
+            maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
         });
         this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
         intervalCallback();
@@ -42,17 +54,8 @@ export class LocalEventManager extends EventManager {
      * @internal
      */
     async isContainerizedWrapper() {
-        return this.config.get('containerized', await isContainerized());
-    }
-    getCurrentCpuTicks() {
-        const cpus = os.cpus();
-        return cpus.reduce((acc, cpu) => {
-            const cpuTimes = Object.values(cpu.times);
-            return {
-                idle: acc.idle + cpu.times.idle,
-                total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
-            };
-        }, { idle: 0, total: 0 });
+        const { isContainerized } = await import('@crawlee/utils');
+        return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
     }
     /**
      * Creates a SystemInfo object based on local metrics.
@@ -65,18 +68,11 @@ export class LocalEventManager extends EventManager {
         };
     }
     async createCpuInfo(options) {
-        if (this.config.get('systemInfoV2')) {
-            const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
-            return {
-                cpuCurrentUsage: usedCpuRatio * 100,
-                isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
-            };
-        }
-        const ticks = this.getCurrentCpuTicks();
-        const idleTicksDelta = ticks.idle - this.previousTicks.idle;
-        const totalTicksDelta = ticks.total - this.previousTicks.total;
-        const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
-        Object.assign(this.previousTicks, ticks);
+        const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
+        const usedCpuRatio = await getCurrentCpuTicksV2({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
         return {
             cpuCurrentUsage: usedCpuRatio * 100,
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -84,19 +80,17 @@ export class LocalEventManager extends EventManager {
     }
     async createMemoryInfo() {
         try {
-            if (this.config.get('systemInfoV2')) {
-                const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
-                return {
-                    memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
-                };
-            }
-            const memInfo = await getMemoryInfo();
+            const { getMemoryInfo } = await import('@crawlee/utils');
+            const memInfo = await getMemoryInfo({
+                containerized: await this.isContainerizedWrapper(),
+                logger: serviceLocator.getLogger(),
+            });
             return {
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
             };
         }
         catch (err) {
-            log.exception(err, 'Memory snapshot failed.');
+            this.log.exception(err, 'Memory snapshot failed.');
             return {};
         }
     }

package/events/local_event_manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEvG,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,OAAO,EAAE,YAAY,EAAa,MAAM,oBAAoB,CAAC;AAE7D,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,aAAa,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAE9C;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC3E,OAAO;oBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;iBAC1E,CAAC;YACN,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YACtC,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAuC,MAAM,oBAAoB,CAAC;AAOvF,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,wBAAwB,CAAS;IAEzC,YAAY,OAAiC;QACzC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAsB;QACpC,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAEnE,OAAO,IAAI,iBAAiB,CAAC;YACzB,0BAA0B,EAAE,cAAc,CAAC,0BAA0B;YACrE,wBAAwB,EAAE,cAAc,CAAC,wBAAwB;SACpE,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CACzC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EACnC,IAAI,CAAC,wBAAwB,CAChC,CAAC;IACN,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,eAAe;SACrE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,gBAAgB,EAAE,CAAC,aAAa,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC;YAC5C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,EAAE;YAClD,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE;SACrC,CAAC,CAAC;QACH,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;gBAChC,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,EAAE;gBAClD,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE;aACrC,CAAC,CAAC;YACH,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YAC5D,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/index.d.ts

@@ -1,10 +1,10 @@
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
@@ -14,6 +14,7 @@ export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './recoverable_state.js';
 export { PseudoUrl } from '@apify/pseudo_url';
-export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
+export type { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
 //# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}

package/index.js

@@ -1,10 +1,10 @@
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
@@ -14,5 +14,6 @@ export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
+export * from './recoverable_state.js';
 export { PseudoUrl } from '@apify/pseudo_url';
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,yBAAyB,CAAC;AACxC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,0BAA0B,CAAC;AACzC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC"}

package/log.d.ts

@@ -1,3 +1,83 @@
-import log, { Log, Logger, LoggerJson, LoggerOptions, LoggerText, LogLevel } from '@apify/log';
-export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText };
+import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+import type { LoggerOptions } from '@apify/log';
+import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+export type { CrawleeLogger, CrawleeLoggerOptions };
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
+    private options;
+    private readonly warningsLogged;
+    constructor(options?: Partial<CrawleeLoggerOptions>);
+    /**
+     * Core logging method. Subclasses must implement this to dispatch log messages
+     * to the underlying logger (Winston, Pino, console, etc.).
+     *
+     * Level filtering is the responsibility of the underlying library — this method
+     * is called for every message regardless of the current level.
+     *
+     * @param level Crawlee log level (use {@link LogLevel} constants)
+     * @param message The log message
+     * @param data Optional structured data to attach to the log entry
+     */
+    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    /**
+     * Creates a child logger instance. Subclasses must implement this to define
+     * how child loggers are created for the underlying logger.
+     */
+    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    getOptions(): CrawleeLoggerOptions;
+    setOptions(options: Partial<CrawleeLoggerOptions>): void;
+    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    error(message: string, data?: Record<string, unknown>): void;
+    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>): void;
+    warningOnce(message: string): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>): void;
+    deprecated(message: string): void;
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export declare class ApifyLogAdapter extends BaseCrawleeLogger {
+    private readonly apifyLog;
+    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
+    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+}
+export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
+export type { LoggerOptions };
 //# sourceMappingURL=log.d.ts.map

package/log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhF,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,8BAAsB,iBAAkB,YAAW,aAAa;IAC5D,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;gBAExC,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM;IAIvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAE3F;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAErF,UAAU,IAAI,oBAAoB;IAIlC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAIxD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAI5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAQlF,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI/D,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI9D,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAOlC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAGpC;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,iBAAiB;IAE9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,GAAG,EAC9B,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC;IAK3C,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAIlF,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;CAM/E;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAC9D,YAAY,EAAE,aAAa,EAAE,CAAC"}

package/log.js

@@ -1,3 +1,105 @@
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export class BaseCrawleeLogger {
+    options;
+    warningsLogged = new Set();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    getOptions() {
+        return this.options;
+    }
+    setOptions(options) {
+        this.options = { ...this.options, ...options };
+    }
+    child(options) {
+        return this.createChild(options);
+    }
+    error(message, data) {
+        this.logWithLevel(LogLevel.ERROR, message, data);
+    }
+    exception(exception, message, data) {
+        this.logWithLevel(LogLevel.ERROR, `${message}: ${exception.message}`, {
+            ...data,
+            stack: exception.stack,
+            exception,
+        });
+    }
+    softFail(message, data) {
+        this.logWithLevel(LogLevel.SOFT_FAIL, message, data);
+    }
+    warning(message, data) {
+        this.logWithLevel(LogLevel.WARNING, message, data);
+    }
+    warningOnce(message) {
+        if (!this.warningsLogged.has(message)) {
+            this.warningsLogged.add(message);
+            this.warning(message);
+        }
+    }
+    info(message, data) {
+        this.logWithLevel(LogLevel.INFO, message, data);
+    }
+    debug(message, data) {
+        this.logWithLevel(LogLevel.DEBUG, message, data);
+    }
+    perf(message, data) {
+        this.logWithLevel(LogLevel.PERF, `[PERF] ${message}`, data);
+    }
+    deprecated(message) {
+        this.warningOnce(`[DEPRECATED] ${message}`);
+    }
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export class ApifyLogAdapter extends BaseCrawleeLogger {
+    apifyLog;
+    constructor(apifyLog, options) {
+        super(options ?? {});
+        this.apifyLog = apifyLog;
+    }
+    logWithLevel(level, message, data) {
+        this.apifyLog.internal(level, message, data);
+    }
+    createChild(options) {
+        return new ApifyLogAdapter(this.apifyLog.child({ prefix: options.prefix ?? null }), {
+            ...this.getOptions(),
+            ...options,
+        });
+    }
+}
 export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
 //# sourceMappingURL=log.js.map

package/log.js.map

@@ -1 +1 @@
-{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAiB,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAiB,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAGA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAIhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAgB,iBAAiB;IAC3B,OAAO,CAAuB;IACrB,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpD,YAAY,UAAyC,EAAE;QACnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAqBD,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,UAAU,CAAC,OAAsC;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAsC;QACxC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,SAAS,CAAC,SAAgB,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,KAAK,SAAS,CAAC,OAAO,EAAE,EAAE;YAClE,GAAG,IAAI;YACP,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,SAAS;SACZ,CAAC,CAAC;IACP,CAAC;IAED,QAAQ,CAAC,OAAe,EAAE,IAA8B;QACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,OAAe,EAAE,IAA8B;QACnD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,WAAW,CAAC,OAAe;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,UAAU,CAAC,OAAe;QACtB,IAAI,CAAC,WAAW,CAAC,gBAAgB,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;CACJ;AAED;;;;;GAKG;AACH,MAAM,OAAO,eAAgB,SAAQ,iBAAiB;IAE7B;IADrB,YACqB,QAAa,EAC9B,OAAuC;QAEvC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAHJ,aAAQ,GAAR,QAAQ,CAAK;IAIlC,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7D,CAAC;IAES,WAAW,CAAC,OAAsC;QACxD,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,EAAE;YAChF,GAAG,IAAI,CAAC,UAAU,EAAE;YACpB,GAAG,OAAO;SACb,CAAC,CAAC;IACP,CAAC;CACJ;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.6",
+    "version": "4.0.0-beta.60",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -53,22 +53,21 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.6",
-        "@crawlee/types": "4.0.0-beta.6",
-        "@crawlee/utils": "4.0.0-beta.6",
+        "@crawlee/memory-storage": "4.0.0-beta.60",
+        "@crawlee/types": "4.0.0-beta.60",
+        "@crawlee/utils": "4.0.0-beta.60",
         "@sapphire/async-queue": "^1.5.5",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "csv-stringify": "^6.5.2",
-        "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "json5": "^2.2.3",
         "minimatch": "^10.0.1",
         "ow": "^2.0.0",
         "stream-json": "^1.9.1",
         "tldts": "^7.0.6",
-        "tough-cookie": "^5.1.2",
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "type-fest": "^4.41.0",
+        "zod": "^3.24.0 || ^4.0.0"
     },
     "lerna": {
         "command": {
@@ -77,5 +76,5 @@
             }
         }
     },
-    "gitHead": "e82777976478391d607a30206afb8e863e8cc993"
+    "gitHead": "20de5276ab66f520d92b58a40741ee3200913f85"
 }