@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/storages/utils.js

@@ -1,21 +1,31 @@
 import crypto from 'node:crypto';
 import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
-export async function purgeDefaultStorages(configOrOptions, client) {
-    const options = configOrOptions instanceof Configuration
+export async function purgeDefaultStorages(configurationOrOptions, storageBackend) {
+    const options = configurationOrOptions instanceof Configuration
         ? {
-            client,
-            config: configOrOptions,
+            storageBackend,
+            configuration: configurationOrOptions,
         }
-        : (configOrOptions ?? {});
-    const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
-    ({ client = config.getStorageClient() } = options);
-    const casted = client;
+        : (configurationOrOptions ?? {});
+    const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
+    ({ storageBackend = serviceLocator.getStorageBackend() } = options);
+    const casted = storageBackend;
+    const runPurge = async () => {
+        try {
+            await casted.purge?.();
+        }
+        catch (e) {
+            casted.__purged = undefined;
+            throw e;
+        }
+    };
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
-    if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
-        casted.__purged = true;
-        await casted.purge?.();
+    if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
+        casted.__purged = runPurge();
     }
+    await casted.__purged;
 }
 /**
  * Easily create and manage state values. All state values are automatically persisted.
@@ -24,11 +34,11 @@ export async function purgeDefaultStorages(configOrOptions, client) {
  *
  * @param name The name of the store to use.
  * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
- * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
+ * @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
  */
 export async function useState(name, defaultValue = {}, options) {
-    const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
-        config: options?.config || Configuration.getGlobalConfig(),
+    const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
+        configuration: options?.configuration || serviceLocator.getConfiguration(),
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
@@ -69,4 +79,54 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
  * @internal
  */
 export const MAX_QUERIES_FOR_CONSISTENCY = 6;
-//# sourceMappingURL=utils.js.map
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export function createDualIterable(options) {
+    const { createPages, extractItems } = options;
+    let cached = null;
+    function getOrCreate() {
+        if (!cached) {
+            cached = (async () => {
+                const items = [];
+                for await (const page of createPages()) {
+                    items.push(...extractItems(page));
+                }
+                return items;
+            })();
+        }
+        return cached;
+    }
+    async function* iterateAll() {
+        for await (const page of createPages()) {
+            yield* extractItems(page);
+        }
+    }
+    const result = {
+        [Symbol.asyncIterator]() {
+            return iterateAll();
+        },
+        then(onfulfilled, onrejected) {
+            return getOrCreate().then(onfulfilled, onrejected);
+        },
+        catch(onrejected) {
+            return getOrCreate().catch(onrejected);
+        },
+        finally(onfinally) {
+            return getOrCreate().finally(onfinally);
+        },
+        [Symbol.toStringTag]: 'DualIterable',
+    };
+    return result;
+}

package/system-info/cpu-info.d.ts

@@ -0,0 +1,67 @@
+import type { CrawleeLogger } from '@crawlee/types';
+/**
+ * Gets the "bare metal" cpu load.
+ * Used in
+ *  - AWS Lambda
+ *  - Containers without a cGroup quota
+ *  - Uncontainerized environments
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicks(): number;
+/**
+ * Reads the cgroup cpu quota.
+ * In V1, a quota of -1 means “unlimited.”
+ * In V2, a first field of "max" means unlimited.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu Quota
+ * @internal
+ */
+export declare function getCpuQuota(cgroupsVersion: string): Promise<number | null>;
+/**
+ * Reads the cgroup cpu period.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu quota period
+ * @internal
+ */
+export declare function getCpuPeriod(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the container
+ *
+ * @param cgroupsVersion the cGroup version
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getContainerCpuUsage(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the system from cgroup
+ *
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getSystemCpuUsage(): Promise<number>;
+/**
+ * a cpu sample with the container usage and system usage
+ */
+export interface CpuSample {
+    containerUsage: number;
+    systemUsage: number;
+}
+/**
+ * Takes a CPU usage sample for both the container and the system.
+ *
+ * @returns An object containing the container and system CPU usage.
+ * @internal
+ */
+export declare function sampleCpuUsage(cGroupsVersion: string): Promise<CpuSample>;
+/**
+ * Gets the cpu usage of the system.
+ * If the crawler is running in a containerized environment, crawlee will check for a cgroup enforced cpu limit.
+ * If a cgroup limit is found, it will be taken as the maximum load against which the current load will be gauged.
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicksV2(options?: {
+    containerized?: boolean;
+    logger?: CrawleeLogger;
+}): Promise<number>;

package/system-info/cpu-info.js

@@ -0,0 +1,216 @@
+import { execSync } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import os from 'node:os';
+import { getCgroupsVersion } from './runtime.js';
+const CPU_FILE_PATHS = {
+    STAT: {
+        V1: '/sys/fs/cgroup/cpuacct/cpuacct.usage',
+        V2: '/sys/fs/cgroup/cpu.stat',
+    },
+    QUOTA: {
+        V1: '/sys/fs/cgroup/cpu/cpu.cfs_quota_us',
+        V2: '/sys/fs/cgroup/cpu.max',
+    },
+    PERIOD: {
+        V1: '/sys/fs/cgroup/cpu/cpu.cfs_period_us',
+        V2: '/sys/fs/cgroup/cpu.max',
+    },
+};
+let CLOCK_TICKS_PER_SECOND = 100;
+let CLOCK_TICKS_CHECKED = false;
+const NANOSECONDS_PER_SECOND = 1e9;
+const previousTicks = { idle: 0, total: 0 };
+/**
+ * Gets the "bare metal" cpu load.
+ * Used in
+ *  - AWS Lambda
+ *  - Containers without a cGroup quota
+ *  - Uncontainerized environments
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export function getCurrentCpuTicks() {
+    const cpusCores = os.cpus();
+    const ticks = cpusCores.reduce((acc, cpu) => {
+        const cpuTimes = Object.values(cpu.times);
+        return {
+            idle: acc.idle + cpu.times.idle,
+            total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
+        };
+    }, { idle: 0, total: 0 });
+    const idleTicksDelta = ticks.idle - previousTicks.idle;
+    const totalTicksDelta = ticks.total - previousTicks.total;
+    Object.assign(previousTicks, ticks);
+    return totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
+}
+/**
+ * Reads the linux tick rate.
+ * @returns the number of ticks per second, or `null` if detection failed
+ */
+function getClockTicks() {
+    try {
+        return parseInt(execSync('getconf CLK_TCK').toString().trim(), 10);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Reads the cgroup cpu quota.
+ * In V1, a quota of -1 means “unlimited.”
+ * In V2, a first field of "max" means unlimited.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu Quota
+ * @internal
+ */
+export async function getCpuQuota(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const quotaStr = await readFile(CPU_FILE_PATHS.QUOTA.V1, 'utf8');
+        const quota = parseInt(quotaStr.trim(), 10);
+        return quota === -1 ? null : quota;
+    }
+    // cgroup v2
+    const maxStr = await readFile(CPU_FILE_PATHS.QUOTA.V2, 'utf8');
+    const parts = maxStr.trim().split(/\s+/);
+    if (parts[0] === 'max') {
+        return null;
+    }
+    return parseInt(parts[0], 10);
+}
+/**
+ * Reads the cgroup cpu period.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu quota period
+ * @internal
+ */
+export async function getCpuPeriod(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const quotaStr = await readFile(CPU_FILE_PATHS.PERIOD.V1, 'utf8');
+        const quota = parseInt(quotaStr.trim(), 10);
+        return quota;
+    }
+    // cgroup v2
+    const maxStr = await readFile(CPU_FILE_PATHS.PERIOD.V2, 'utf8');
+    const parts = maxStr.trim().split(/\s+/);
+    return parseInt(parts[1], 10);
+}
+/**
+ * Reads the cgroup cpu usage of the container
+ *
+ * @param cgroupsVersion the cGroup version
+ * @returns the cpu usage
+ * @internal
+ */
+export async function getContainerCpuUsage(cgroupsVersion) {
+    if (cgroupsVersion === 'V1') {
+        const data = await readFile(CPU_FILE_PATHS.STAT.V1, 'utf8');
+        return Number(data.trim());
+    }
+    // cgroup v2
+    const data = await readFile(CPU_FILE_PATHS.STAT.V2, 'utf8');
+    const lines = data.split('\n');
+    let usageUsec = 0;
+    for (const line of lines) {
+        const parts = line.trim().split(/\s+/);
+        if (parts[0] === 'usage_usec') {
+            usageUsec = Number(parts[1]);
+            break;
+        }
+    }
+    // Convert microseconds to nanoseconds.
+    return usageUsec * 1000;
+}
+/**
+ * Reads the cgroup cpu usage of the system from cgroup
+ *
+ * @returns the cpu usage
+ * @internal
+ */
+export async function getSystemCpuUsage() {
+    const statData = await readFile('/proc/stat', 'utf8');
+    const lines = statData.split('\n');
+    for (const line of lines) {
+        if (line.startsWith('cpu ')) {
+            // Split the line and extract the first seven numeric fields:
+            // user, nice, system, idle, iowait, irq, softirq
+            const parts = line.split(/\s+/).slice(1, 8);
+            let totalTicks = 0;
+            for (const part of parts) {
+                totalTicks += Number(part);
+            }
+            // Convert clock ticks to nanoseconds.
+            return (totalTicks * NANOSECONDS_PER_SECOND) / CLOCK_TICKS_PER_SECOND;
+        }
+    }
+    throw new Error('no cpu line'); // shouldnt ever happen
+}
+/**
+ * Takes a CPU usage sample for both the container and the system.
+ *
+ * @returns An object containing the container and system CPU usage.
+ * @internal
+ */
+export async function sampleCpuUsage(cGroupsVersion) {
+    const [containerUsage, systemUsage] = await Promise.all([
+        getContainerCpuUsage(cGroupsVersion),
+        getSystemCpuUsage(),
+    ]);
+    return { containerUsage, systemUsage };
+}
+let previousSample = { containerUsage: 0, systemUsage: 0 };
+/**
+ * Gets the cpu usage of the system.
+ * If the crawler is running in a containerized environment, crawlee will check for a cgroup enforced cpu limit.
+ * If a cgroup limit is found, it will be taken as the maximum load against which the current load will be gauged.
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export async function getCurrentCpuTicksV2(options = {}) {
+    const { containerized = false, logger } = options;
+    try {
+        // if not containerized
+        if (!containerized) {
+            // bare metal cpu limit
+            return getCurrentCpuTicks();
+        }
+        if (!CLOCK_TICKS_CHECKED) {
+            const ticks = getClockTicks();
+            if (ticks === null) {
+                CLOCK_TICKS_PER_SECOND = 100;
+                logger?.warning('Failed to get clock ticks; defaulting to 100. CPU metrics may be inaccurate.');
+            }
+            else {
+                CLOCK_TICKS_PER_SECOND = ticks;
+            }
+            CLOCK_TICKS_CHECKED = true;
+        }
+        const cgroupsVersion = await getCgroupsVersion();
+        // if cgroup is not detected, return bare metal cpu limit
+        if (cgroupsVersion === null) {
+            logger?.warningOnce('Your environment is containerized, but your system does not support cgroups.\n' +
+                "If you're running containers with limited cpu, cpu auto-scaling will not work properly.");
+            return getCurrentCpuTicks();
+        }
+        // cgroup aware cpu limit. If no limits are set, default to returning getCurrentCpuTicks.
+        const quota = await getCpuQuota(cgroupsVersion);
+        if (quota === null) {
+            // no cgroup limit, return host cpu load
+            return getCurrentCpuTicks();
+        }
+        const period = await getCpuPeriod(cgroupsVersion);
+        // eg. having a 200000us quots per 100000us means the cGroup can fully use 2 cores
+        const cpuAllowance = quota / period;
+        const sample = await sampleCpuUsage(cgroupsVersion);
+        const containerDelta = sample.containerUsage - previousSample.containerUsage;
+        const systemDelta = sample.systemUsage - previousSample.systemUsage;
+        previousSample = sample;
+        const numCpus = os.cpus().length;
+        // Calculate the CPU usage percentage.
+        return ((containerDelta / systemDelta) * numCpus) / cpuAllowance;
+    }
+    catch (err) {
+        // if anything fails, default to bare metal metrics
+        logger?.warning('Cpu snapshot failed, falling back to bare-metal metrics.', { error: err });
+        return getCurrentCpuTicks();
+    }
+}

package/system-info/memory-info.d.ts

@@ -0,0 +1,31 @@
+import type { CrawleeLogger } from '@crawlee/types';
+/**
+ * Describes memory usage of the process.
+ */
+export interface MemoryInfo {
+    /** Total memory available in the system or container */
+    totalBytes: number;
+    /** Amount of free memory in the system or container */
+    freeBytes: number;
+    /** Amount of memory used (= totalBytes - freeBytes) */
+    usedBytes: number;
+    /** Amount of memory used the current Node.js process */
+    mainProcessBytes: number;
+    /** Amount of memory used by child processes of the current Node.js process */
+    childProcessesBytes: number;
+}
+/**
+ * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
+ *
+ * If the process runs inside of a container, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ * @returns An object containing the free and used memory metrics.
+ * @internal
+ */
+export declare function getMemoryInfo(options?: {
+    containerized?: boolean;
+    logger?: CrawleeLogger;
+}): Promise<MemoryInfo>;

package/system-info/memory-info.js

@@ -0,0 +1,115 @@
+import { execSync } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import { freemem, totalmem } from 'node:os';
+import { getCgroupsVersion, isLambda } from './runtime.js';
+import { psTree } from './ps-tree.js';
+const MEMORY_FILE_PATHS = {
+    TOTAL: {
+        V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
+        V2: '/sys/fs/cgroup/memory.max',
+    },
+    USED: {
+        V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
+        V2: '/sys/fs/cgroup/memory.current',
+    },
+};
+/**
+ * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
+ *
+ * If the process runs inside of a container, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ * @returns An object containing the free and used memory metrics.
+ * @internal
+ */
+export async function getMemoryInfo(options = {}) {
+    const { containerized = false, logger } = options;
+    let mainProcessBytes = -1;
+    let childProcessesBytes = 0;
+    // lambda does *not* have `ps` and other command line tools
+    // required to extract memory usage.
+    if (isLambda()) {
+        // reported in bytes
+        mainProcessBytes = process.memoryUsage().rss;
+        // https://stackoverflow.com/a/55914335/129415
+        const memInfo = execSync('cat /proc/meminfo').toString();
+        const values = memInfo.split(/[\n: ]/).filter((val) => val.trim());
+        // /proc/meminfo reports in kb, not bytes, the total used memory is reported by meminfo
+        // subtract memory used by the main node process in order to infer memory used by any child processes
+        childProcessesBytes = +values[19] * 1000 - mainProcessBytes;
+    }
+    else {
+        // Query both root and child processes
+        const processes = await psTree(process.pid, true);
+        processes.forEach((rec) => {
+            // Obtain main process' memory separately
+            if (rec.PID === `${process.pid}`) {
+                mainProcessBytes = rec.RSS;
+                return;
+            }
+            childProcessesBytes += rec.RSS;
+        });
+    }
+    let totalBytes;
+    let usedBytes;
+    let freeBytes;
+    if (isLambda()) {
+        // memory size is defined in megabytes
+        totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
+        usedBytes = mainProcessBytes + childProcessesBytes;
+        freeBytes = totalBytes - usedBytes;
+    }
+    else if (containerized) {
+        // When running inside a container, use container memory limits
+        const cgroupsVersion = await getCgroupsVersion();
+        try {
+            if (cgroupsVersion === null) {
+                throw new Error('cgroup not available');
+            }
+            let [totalBytesStr, usedBytesStr] = await Promise.all([
+                readFile(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], 'utf8'),
+                readFile(MEMORY_FILE_PATHS.USED[cgroupsVersion], 'utf8'),
+            ]);
+            // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
+            totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            // Cgroups V2 contains 'max' string if memory is not limited
+            // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
+            if (totalBytesStr === 'max') {
+                totalBytes = totalmem();
+                // Cgroups V1 is set to number related to platform and page size if memory is not limited
+                // See https://unix.stackexchange.com/q/420906
+            }
+            else {
+                totalBytes = parseInt(totalBytesStr, 10);
+                const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
+                if (containerRunsWithUnlimitedMemory)
+                    totalBytes = totalmem();
+            }
+            usedBytes = parseInt(usedBytesStr, 10);
+            freeBytes = totalBytes - usedBytes;
+        }
+        catch (err) {
+            logger?.warningOnce('Your environment is containerized, but your system does not support memory cgroups. ' +
+                "If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" +
+                `Cause: ${err.message}`);
+            totalBytes = totalmem();
+            freeBytes = freemem();
+            usedBytes = totalBytes - freeBytes;
+        }
+    }
+    else {
+        totalBytes = totalmem();
+        freeBytes = freemem();
+        usedBytes = totalBytes - freeBytes;
+    }
+    return {
+        totalBytes,
+        freeBytes,
+        usedBytes,
+        mainProcessBytes,
+        childProcessesBytes,
+    };
+}

package/system-info/ps-tree.d.ts

@@ -0,0 +1,17 @@
+export interface ProcessInfo {
+    PPID: string;
+    PID: string;
+    STAT: string | null;
+    RSS: number;
+    COMMAND: string;
+}
+/**
+ * Returns a promise that resolves with an array of ProcessInfo objects representing
+ * the children of the given PID.
+ *
+ * @param pid - The PID (number or string) for which to list child processes.
+ * @param includeRoot - Optional flag. When true, include the process with the given PID if found.
+ *                      Defaults to false.
+ * @internal
+ */
+export declare function psTree(pid: number | string, includeRoot?: boolean): Promise<ProcessInfo[]>;