@crawlee/core 4.0.0-beta.17 → 4.0.0-beta.171

package/storages/transaction.js

@@ -0,0 +1,287 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
+import { serviceLocator } from '../service_locator.js';
+import { AfterCommitError, NonRetryableError } from '../errors.js';
+const DEFAULT_STORAGE_WRITE_POLICY = { requestQueue: 'writeThrough' };
+const DEFAULT_COMMIT_TIMEOUT_MILLIS = 300_000;
+const transactionStorage = new AsyncLocalStorage();
+const COMMIT_ORDER = ['keyValueStore', 'requestQueue', 'dataset'];
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export class StorageTransaction {
+    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    journal = [];
+    /** Per-storage-type write policy. */
+    policy;
+    #commitTimeoutMillis;
+    #state = 'open';
+    #disposed = false;
+    /** Callbacks registered via `afterCommit()`, run in registration order once a commit has been attempted. */
+    #afterCommitCallbacks = [];
+    /** @internal */
+    constructor(options = {}) {
+        this.policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };
+        this.#commitTimeoutMillis = options.commitTimeoutMillis ?? DEFAULT_COMMIT_TIMEOUT_MILLIS;
+    }
+    get state() {
+        return this.#state;
+    }
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive() {
+        return this.#state === 'open';
+    }
+    /** Runs `callback` with this transaction installed in the async context. */
+    async run(callback) {
+        return transactionStorage.run(this, async () => callback());
+    }
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry) {
+        if (!this.isActive) {
+            throw new Error(`Cannot record a journal entry on a transaction in the '${this.#state}' state`);
+        }
+        this.journal.push(entry);
+    }
+    /**
+     * Registers a callback to run once a commit has been attempted, whether it succeeded (`error` is
+     * `undefined`) or failed - the only point at which a deferred write can be reacted to where it was
+     * made. Callbacks run in registration order; the first to throw propagates, the rest do not run, and
+     * on a failed commit its error replaces the commit error. Not run on rollback: nothing was written.
+     */
+    afterCommit(callback) {
+        if (!this.isActive) {
+            throw new Error(`Cannot register a commit callback on a transaction in the '${this.#state}' state`);
+        }
+        this.#afterCommitCallbacks.push(callback);
+    }
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    async commit() {
+        if (this.#state !== 'open') {
+            return;
+        }
+        this.#state = 'committing';
+        try {
+            // The replay re-drives the frontend write path, which checks for cancellation (`tryCancel`)
+            // on every operation - and `@apify/timeout` shares one `AbortController` across nested
+            // frames, so a request-handler timeout that already fired would abort the commit of a
+            // handler that succeeded. Hence a fresh timeout context, which also provides the time bound.
+            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
+        }
+        catch (error) {
+            this.#state = 'failed';
+            // Callbacks are handed an `Error`, so a backend that rejects with anything else gets wrapped.
+            const commitError = error instanceof Error ? error : new Error(String(error), { cause: error });
+            // A callback that throws here propagates instead: it was handed the failure and decided what
+            // the request should fail with.
+            await this.#runAfterCommitCallbacks(commitError);
+            throw commitError;
+        }
+        this.#state = 'committed';
+        try {
+            await this.#runAfterCommitCallbacks();
+        }
+        catch (error) {
+            throw error instanceof NonRetryableError ? error : new AfterCommitError(error);
+        }
+    }
+    /** Fresh timeout context for the same reason as the flush. */
+    async #runAfterCommitCallbacks(error) {
+        await timeoutStorage.exit(async () => {
+            for (const callback of this.#afterCommitCallbacks) {
+                await callback(error);
+            }
+        });
+    }
+    async flush() {
+        // Each participating frontend replays all of its buffered entries in one call. Frontends are
+        // ordered by storage type: key-value stores and request queues first (idempotent under retry),
+        // datasets last (not idempotent), minimizing the blast radius of a partial commit failure.
+        const groups = new Map();
+        for (const entry of this.journal) {
+            if (entry.type === 'requestQueue' && entry.writeThrough)
+                continue;
+            const group = groups.get(entry.participant);
+            if (group)
+                group.push(entry);
+            else
+                groups.set(entry.participant, [entry]);
+        }
+        // A participant only records entries of its own storage type, so the first entry determines
+        // the group's place in the commit order.
+        const orderedGroups = [...groups.values()].sort((a, b) => COMMIT_ORDER.indexOf(a[0].type) - COMMIT_ORDER.indexOf(b[0].type));
+        for (const entries of orderedGroups) {
+            await entries[0].participant.commitJournalEntries(entries);
+        }
+    }
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback() {
+        if (this.#state !== 'open') {
+            return;
+        }
+        this.#state = 'rolledBack';
+    }
+    /**
+     * Releases the journal, the write-time snapshots it holds and any registered commit callbacks. Must
+     * be called for *every* terminal state, `failed` included. Idempotent, never throws, and does not
+     * change `state`. Any {@link StorageTransactionView} of this transaction is only valid until this
+     * is called.
+     */
+    dispose() {
+        if (this.#disposed) {
+            return;
+        }
+        if (this.#state === 'open') {
+            // Disposing an open transaction is an internal invariant violation - roll back first.
+            try {
+                serviceLocator
+                    .getLogger()
+                    .warning('Internal error: a storage transaction was disposed while still open; rolling it back.');
+            }
+            catch {
+                // Never throw from dispose.
+            }
+            this.rollback();
+        }
+        this.#disposed = true;
+        this.journal.length = 0;
+        this.#afterCommitCallbacks.length = 0;
+    }
+    get datasetItems() {
+        return this.journal.flatMap((entry) => entry.type === 'dataset' ? entry.items.map((item) => ({ item, datasetId: entry.storageId })) : []);
+    }
+    get enqueuedUrls() {
+        return this.journal.flatMap((entry) => entry.type === 'requestQueue' ? entry.requests.map(({ url, label }) => ({ url, label })) : []);
+    }
+    get keyValueStoreChanges() {
+        const result = {};
+        for (const entry of this.journal) {
+            if (entry.type !== 'keyValueStore')
+                continue;
+            result[entry.storageId] ??= {};
+            result[entry.storageId][entry.key] = { changedValue: entry.value, options: entry.options };
+        }
+        return result;
+    }
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export function createStorageTransaction(options = {}) {
+    return new StorageTransaction(options);
+}
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export async function withStorageTransaction(callback, options = {}) {
+    const existing = transactionStorage.getStore();
+    if (existing?.isActive) {
+        return callback(existing);
+    }
+    const transaction = createStorageTransaction(options);
+    try {
+        const result = await transaction.run(async () => callback(transaction));
+        await transaction.commit();
+        return result;
+    }
+    catch (error) {
+        transaction.rollback();
+        throw error;
+    }
+    finally {
+        transaction.dispose();
+    }
+}
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export async function withDirectStorageAccess(callback) {
+    return transactionStorage.exit(async () => callback());
+}
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export function activeStorageTransaction() {
+    tryCancel();
+    const transaction = transactionStorage.getStore();
+    return transaction?.isActive ? transaction : undefined;
+}
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export function currentStorageTransaction() {
+    return transactionStorage.getStore();
+}
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export function snapshotValue(value) {
+    try {
+        return structuredClone(value);
+    }
+    catch {
+        return JSON.parse(JSON.stringify(value));
+    }
+}
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export function rejectOperationInTransaction(operation, reason = 'it cannot be rolled back.') {
+    if (activeStorageTransaction() === undefined) {
+        return;
+    }
+    throw operationRejectedInTransaction(operation, reason);
+}
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export function operationRejectedInTransaction(operation, reason = 'it cannot be rolled back.') {
+    return new Error(`${operation} cannot be used inside a storage transaction: ${reason} ` +
+        'If you really need it, wrap the call in withDirectStorageAccess(() => ...) - operations ' +
+        'performed there are applied immediately and are not rolled back.');
+}

package/storages/utils.d.ts

@@ -1,5 +1,7 @@
-import type { Dictionary, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { IProxyConfiguration } from '../proxy_configuration.js';
 /**
  * Options for purging default storage.
  */
@@ -8,18 +10,19 @@ interface PurgeDefaultStorageOptions {
      * If set to `true`, calling multiple times will only have effect at the first time.
      */
     onlyPurgeOnce?: boolean;
-    config?: Configuration;
-    client?: StorageClient;
+    configuration?: Configuration;
+    storageBackend?: StorageBackend;
 }
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
  *
- * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
+ * This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
  * it will call the `purge` method of the underlying storage implementation we are currently using. You can
  * make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in
  * the `options` object
@@ -27,18 +30,19 @@ interface PurgeDefaultStorageOptions {
 export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
  * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
  *
- * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
+ * This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
  * it will call the `purge` method of the underlying storage implementation we are currently using.
  */
-export declare function purgeDefaultStorages(config?: Configuration, client?: StorageClient): Promise<void>;
+export declare function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;
 export interface UseStateOptions {
-    config?: Configuration;
+    configuration?: Configuration;
     /**
      * The name of the key-value store you'd like the state to be stored in.
      * If not provided, the default store will be used.
@@ -52,7 +56,7 @@ export interface UseStateOptions {
  *
  * @param name The name of the store to use.
  * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
- * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
+ * @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
  */
 export declare function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;
 /**
@@ -89,5 +93,49 @@ export declare const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10000;
  * @internal
  */
 export declare const MAX_QUERIES_FOR_CONSISTENCY = 6;
+/** @internal */
+export interface DualIterableOptions<TItem, TRawPage> {
+    /** Factory that returns an async generator yielding pages. */
+    createPages: () => AsyncGenerator<TRawPage>;
+    /** Extracts individual items from a page (for iteration). */
+    extractItems: (page: TRawPage) => TItem[];
+}
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export declare function createDualIterable<TItem, TRawPage>(options: DualIterableOptions<TItem, TRawPage>): AsyncIterable<TItem> & Promise<TItem[]>;
+/**
+ * Options for the static `open()` method on storage classes ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
+ */
+export interface StorageOpenOptions {
+    /**
+     * SDK configuration instance, defaults to the static register.
+     */
+    configuration?: Configuration;
+    /**
+     * Optional storage backend that should be used to open storages.
+     */
+    storageBackend?: StorageBackend;
+    /**
+     * Used to pass the proxy configuration for the `requestsFromUrl` objects.
+     * Takes advantage of the internal address rotation and authentication process.
+     * If undefined, the `requestsFromUrl` requests will be made without proxy.
+     */
+    proxyConfiguration?: IProxyConfiguration;
+    /**
+     * HTTP client to be used to download the list of URLs in `RequestQueue`.
+     */
+    httpClient?: BaseHttpClient;
+}
 export {};
-//# sourceMappingURL=utils.d.ts.map

package/storages/utils.js

@@ -1,21 +1,31 @@
 import crypto from 'node:crypto';
 import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
-export async function purgeDefaultStorages(configOrOptions, client) {
-    const options = configOrOptions instanceof Configuration
+export async function purgeDefaultStorages(configurationOrOptions, storageBackend) {
+    const options = configurationOrOptions instanceof Configuration
         ? {
-            client,
-            config: configOrOptions,
+            storageBackend,
+            configuration: configurationOrOptions,
         }
-        : (configOrOptions ?? {});
-    const { config = Configuration.getGlobalConfig(), onlyPurgeOnce = false } = options;
-    ({ client = config.getStorageClient() } = options);
-    const casted = client;
+        : (configurationOrOptions ?? {});
+    const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
+    ({ storageBackend = serviceLocator.getStorageBackend() } = options);
+    const casted = storageBackend;
+    const runPurge = async () => {
+        try {
+            await casted.purge?.();
+        }
+        catch (e) {
+            casted.__purged = undefined;
+            throw e;
+        }
+    };
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
-    if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
-        casted.__purged = true;
-        await casted.purge?.();
+    if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
+        casted.__purged = runPurge();
     }
+    await casted.__purged;
 }
 /**
  * Easily create and manage state values. All state values are automatically persisted.
@@ -24,11 +34,11 @@ export async function purgeDefaultStorages(configOrOptions, client) {
  *
  * @param name The name of the store to use.
  * @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
- * @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
+ * @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
  */
 export async function useState(name, defaultValue = {}, options) {
-    const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
-        config: options?.config || Configuration.getGlobalConfig(),
+    const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
+        configuration: options?.configuration || serviceLocator.getConfiguration(),
     });
     return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
 }
@@ -69,4 +79,54 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10_000;
  * @internal
  */
 export const MAX_QUERIES_FOR_CONSISTENCY = 6;
-//# sourceMappingURL=utils.js.map
+/**
+ * Creates an object that is both an `AsyncIterable<TItem>` (for `for await...of`)
+ * and a `Promise<TItem[]>` (for `await`) from a single async page generator.
+ *
+ * - `await result` drains all pages from a fresh generator and returns every
+ *   item as a flat array.
+ * - `for await (const item of result)` streams all items across all pages,
+ *   yielding them one by one without buffering everything in memory.
+ *
+ * Each usage path creates its own generator instance, so `await` and
+ * `for await...of` never interfere with each other.
+ *
+ * @internal
+ */
+export function createDualIterable(options) {
+    const { createPages, extractItems } = options;
+    let cached = null;
+    function getOrCreate() {
+        if (!cached) {
+            cached = (async () => {
+                const items = [];
+                for await (const page of createPages()) {
+                    items.push(...extractItems(page));
+                }
+                return items;
+            })();
+        }
+        return cached;
+    }
+    async function* iterateAll() {
+        for await (const page of createPages()) {
+            yield* extractItems(page);
+        }
+    }
+    const result = {
+        [Symbol.asyncIterator]() {
+            return iterateAll();
+        },
+        then(onfulfilled, onrejected) {
+            return getOrCreate().then(onfulfilled, onrejected);
+        },
+        catch(onrejected) {
+            return getOrCreate().catch(onrejected);
+        },
+        finally(onfinally) {
+            return getOrCreate().finally(onfinally);
+        },
+        [Symbol.toStringTag]: 'DualIterable',
+    };
+    return result;
+}

package/system-info/cpu-info.d.ts

@@ -0,0 +1,67 @@
+import type { CrawleeLogger } from '@crawlee/types';
+/**
+ * Gets the "bare metal" cpu load.
+ * Used in
+ *  - AWS Lambda
+ *  - Containers without a cGroup quota
+ *  - Uncontainerized environments
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicks(): number;
+/**
+ * Reads the cgroup cpu quota.
+ * In V1, a quota of -1 means “unlimited.”
+ * In V2, a first field of "max" means unlimited.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu Quota
+ * @internal
+ */
+export declare function getCpuQuota(cgroupsVersion: string): Promise<number | null>;
+/**
+ * Reads the cgroup cpu period.
+ * @param cgroupsVersion the cGroup version
+ * @returns The Cpu quota period
+ * @internal
+ */
+export declare function getCpuPeriod(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the container
+ *
+ * @param cgroupsVersion the cGroup version
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getContainerCpuUsage(cgroupsVersion: string): Promise<number>;
+/**
+ * Reads the cgroup cpu usage of the system from cgroup
+ *
+ * @returns the cpu usage
+ * @internal
+ */
+export declare function getSystemCpuUsage(): Promise<number>;
+/**
+ * a cpu sample with the container usage and system usage
+ */
+export interface CpuSample {
+    containerUsage: number;
+    systemUsage: number;
+}
+/**
+ * Takes a CPU usage sample for both the container and the system.
+ *
+ * @returns An object containing the container and system CPU usage.
+ * @internal
+ */
+export declare function sampleCpuUsage(cGroupsVersion: string): Promise<CpuSample>;
+/**
+ * Gets the cpu usage of the system.
+ * If the crawler is running in a containerized environment, crawlee will check for a cgroup enforced cpu limit.
+ * If a cgroup limit is found, it will be taken as the maximum load against which the current load will be gauged.
+ * @returns a number between 0 and 1 for the cpu load
+ * @internal
+ */
+export declare function getCurrentCpuTicksV2(options?: {
+    containerized?: boolean;
+    logger?: CrawleeLogger;
+}): Promise<number>;