@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/transaction.d.ts

@@ -0,0 +1,252 @@
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { RecordOptions } from './key_value_store.js';
+/**
+ * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
+ * applied immediately (`writeThrough`) or recorded and replayed on commit (`deferred`).
+ */
+export type StorageWriteMode = 'deferred' | 'writeThrough';
+/**
+ * Per-storage-type write policy of a {@link StorageTransaction}. Datasets and key-value stores are
+ * always `deferred` and not configurable — deferring is the only safe mode for non-idempotent writes,
+ * and {@link withDirectStorageAccess} covers one-off immediate writes.
+ */
+export interface StorageWritePolicy {
+    /**
+     * Write mode for request queue additions. Note that this is a *write policy* for the queue, not the
+     * queue instance itself (which is the top-level `requestQueue` crawler option).
+     *
+     * - `writeThrough` (default): requests are added immediately and are **not** rolled back with the
+     *   transaction. This is safe (additions are deduplicated by `uniqueKey`, so a retry is idempotent)
+     *   and keeps new requests visible to the crawler while the handler still runs.
+     * - `deferred`: requests are only added when the transaction commits — strict all-or-nothing
+     *   semantics, at the cost of the crawler not seeing them until the handler finishes.
+     */
+    requestQueue: StorageWriteMode;
+}
+export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';
+/**
+ * A storage frontend that can record operations in a transaction journal.
+ * @internal
+ */
+export interface TransactionParticipant {
+    /**
+     * Replay the given buffered journal entries (all recorded by this participant) into the real storage
+     * backend. Called during commit, with the transaction already in the `committing` state, so the
+     * replayed operations pass through.
+     */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
+}
+/**
+ * A single dataset write (`pushData`) recorded in a transaction journal.
+ */
+export interface DatasetJournalEntry {
+    type: 'dataset';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    /** The pushed items, captured by `structuredClone` at write time. */
+    items: Dictionary[];
+    recordedAt: Date;
+}
+/**
+ * A single key-value store write (`setValue`) recorded in a transaction journal.
+ */
+export interface KeyValueStoreJournalEntry {
+    type: 'keyValueStore';
+    /** @internal **/
+    participant: TransactionParticipant;
+    storageId: string;
+    key: string;
+    /** The original, pre-serialization value captured by `structuredClone`; `null` denotes a deletion. */
+    value: unknown;
+    options?: RecordOptions;
+}
+/**
+ * A request recorded in a transaction journal.
+ */
+export interface JournaledRequest {
+    url: string;
+    uniqueKey: string;
+    label?: string;
+    /**
+     * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
+     * deduplicated and write-through ones are journaled for introspection only.
+     */
+    snapshot?: Dictionary;
+}
+/**
+ * A batch of request queue additions recorded in a transaction journal.
+ */
+export interface RequestQueueJournalEntry {
+    type: 'requestQueue';
+    /** @internal **/
+    participant: TransactionParticipant;
+    requests: JournaledRequest[];
+    forefront: boolean;
+    /** Write-through entries were applied immediately; they are never replayed. */
+    writeThrough: boolean;
+}
+export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;
+/**
+ * A read-only view of a {@link StorageTransaction}: only the journal-backed introspection accessors,
+ * without the lifecycle methods. The accessors are synchronous and expose the original pre-serialization
+ * values. They cover every write recorded while the transaction was open, under either write policy —
+ * with the one exception noted on {@link StorageTransactionView.enqueuedUrls|`enqueuedUrls`}. A view
+ * is valid until the transaction is disposed.
+ */
+export interface StorageTransactionView {
+    readonly state: StorageTransactionState;
+    /** Items pushed to datasets during the transaction, in push order. */
+    readonly datasetItems: {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    /**
+     * URLs enqueued to request queues during the transaction, under either write policy. Recorded as
+     * requested, so duplicate, already-present and backend-rejected URLs are included.
+     *
+     * One gap: unless a caller of `addRequestsBatched()` waits for every chunk
+     * (`waitForAllRequestsToBeAdded` or `maxNewRequests`, both of which {@link enqueueLinks} sets when
+     * a crawl limit applies), the chunks after the first are added by a background writer that outlives
+     * the transaction and is not recorded here.
+     */
+    readonly enqueuedUrls: {
+        url: string;
+        label?: string;
+    }[];
+    /** Key-value store changes made during the transaction, keyed by store id, last write per key. */
+    readonly keyValueStoreChanges: Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+export interface StorageTransactionOptions {
+    /** Overrides of the per-storage-type write policy. See {@link StorageWritePolicy}. */
+    policy?: Partial<StorageWritePolicy>;
+    /**
+     * How long a commit may take before it fails, in milliseconds. There is no automatic retry — the
+     * replay of dataset items is not idempotent.
+     * @default 300000
+     */
+    commitTimeoutMillis?: number;
+}
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export declare class StorageTransaction implements StorageTransactionView {
+    #private;
+    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    readonly journal: JournalEntry[];
+    /** Per-storage-type write policy. */
+    readonly policy: StorageWritePolicy;
+    /** @internal */
+    constructor(options?: StorageTransactionOptions);
+    get state(): StorageTransactionState;
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive(): boolean;
+    /** Runs `callback` with this transaction installed in the async context. */
+    run<T>(callback: () => Awaitable<T>): Promise<T>;
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry: JournalEntry): void;
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    commit(): Promise<void>;
+    private flush;
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback(): void;
+    /**
+     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
+     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
+     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     */
+    dispose(): void;
+    get datasetItems(): {
+        item: Dictionary;
+        datasetId: string;
+    }[];
+    get enqueuedUrls(): {
+        url: string;
+        label?: string;
+    }[];
+    get keyValueStoreChanges(): Record<string, Record<string, {
+        changedValue: unknown;
+        options?: RecordOptions;
+    }>>;
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export declare function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export declare function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export declare function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export declare function activeStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export declare function currentStorageTransaction(): StorageTransaction | undefined;
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export declare function snapshotValue<T>(value: T): T;
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export declare function rejectOperationInTransaction(operation: string, reason?: string): void;
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export declare function operationRejectedInTransaction(operation: string, reason?: string): Error;

package/storages/transaction.js

@@ -0,0 +1,251 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
+import { serviceLocator } from '../service_locator.js';
+const DEFAULT_STORAGE_WRITE_POLICY = { requestQueue: 'writeThrough' };
+const DEFAULT_COMMIT_TIMEOUT_MILLIS = 300_000;
+const transactionStorage = new AsyncLocalStorage();
+const COMMIT_ORDER = ['keyValueStore', 'requestQueue', 'dataset'];
+/**
+ * A storage transaction scoped to a request's lifecycle. Writes made through the storage frontends
+ * ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}) while the transaction is active
+ * are recorded rather than applied; on {@link StorageTransaction.commit|`commit()`} they are replayed
+ * into real storage, on {@link StorageTransaction.rollback|`rollback()`} they are dropped. Reads consult
+ * the recorded writes first, so a handler sees its own writes.
+ *
+ * Create one with {@link createStorageTransaction} (explicit commit/rollback) or
+ * {@link withStorageTransaction} (scoped sugar). Crawlers open one automatically around every request
+ * handler unless `transactionalStorage: false` is set.
+ */
+export class StorageTransaction {
+    /** The ordered, append-only journal — the source of truth for commit, introspection and reads. */
+    journal = [];
+    /** Per-storage-type write policy. */
+    policy;
+    #commitTimeoutMillis;
+    #state = 'open';
+    #disposed = false;
+    /** @internal */
+    constructor(options = {}) {
+        this.policy = { ...DEFAULT_STORAGE_WRITE_POLICY, ...options.policy };
+        this.#commitTimeoutMillis = options.commitTimeoutMillis ?? DEFAULT_COMMIT_TIMEOUT_MILLIS;
+    }
+    get state() {
+        return this.#state;
+    }
+    /**
+     * `true` only while `state === 'open'`. This is the single predicate every storage operation
+     * consults — operations performed after the transaction is closed pass through to the real backend.
+     */
+    get isActive() {
+        return this.#state === 'open';
+    }
+    /** Runs `callback` with this transaction installed in the async context. */
+    async run(callback) {
+        return transactionStorage.run(this, async () => callback());
+    }
+    /**
+     * Records a write operation in the journal.
+     * @internal
+     */
+    recordJournalEntry(entry) {
+        if (!this.isActive) {
+            throw new Error(`Cannot record a journal entry on a transaction in the '${this.#state}' state`);
+        }
+        this.journal.push(entry);
+    }
+    /**
+     * Replays the journaled writes into real storage. A no-op unless the transaction is `open`.
+     *
+     * The transaction transitions to `committing` *before* anything is flushed, so a commit that throws
+     * partway lands in `failed` (never back in `open`) and subsequent storage operations pass through
+     * rather than recording into a dead transaction. Delivery is at-least-once — a commit that fails
+     * partway may have applied some of the writes already.
+     */
+    async commit() {
+        if (this.#state !== 'open') {
+            return;
+        }
+        this.#state = 'committing';
+        try {
+            // The replay re-drives the frontend write path, which checks for cancellation (`tryCancel`)
+            // on every operation - and `@apify/timeout` shares one `AbortController` across nested
+            // frames, so a request-handler timeout that already fired would abort the commit of a
+            // handler that succeeded. Hence a fresh timeout context, which also provides the time bound.
+            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
+            this.#state = 'committed';
+        }
+        catch (error) {
+            this.#state = 'failed';
+            throw error;
+        }
+    }
+    async flush() {
+        // Each participating frontend replays all of its buffered entries in one call. Frontends are
+        // ordered by storage type: key-value stores and request queues first (idempotent under retry),
+        // datasets last (not idempotent), minimizing the blast radius of a partial commit failure.
+        const groups = new Map();
+        for (const entry of this.journal) {
+            if (entry.type === 'requestQueue' && entry.writeThrough)
+                continue;
+            const group = groups.get(entry.participant);
+            if (group)
+                group.push(entry);
+            else
+                groups.set(entry.participant, [entry]);
+        }
+        // A participant only records entries of its own storage type, so the first entry determines
+        // the group's place in the commit order.
+        const orderedGroups = [...groups.values()].sort((a, b) => COMMIT_ORDER.indexOf(a[0].type) - COMMIT_ORDER.indexOf(b[0].type));
+        for (const entries of orderedGroups) {
+            await entries[0].participant.commitJournalEntries(entries);
+        }
+    }
+    /**
+     * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
+     * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing
+     * and never throws.
+     */
+    rollback() {
+        if (this.#state !== 'open') {
+            return;
+        }
+        this.#state = 'rolledBack';
+    }
+    /**
+     * Releases the journal and the write-time snapshots it holds. Must be called for *every* terminal
+     * state, `failed` included. Idempotent, never throws, and does not change `state`. Any
+     * {@link StorageTransactionView} of this transaction is only valid until this is called.
+     */
+    dispose() {
+        if (this.#disposed) {
+            return;
+        }
+        if (this.#state === 'open') {
+            // Disposing an open transaction is an internal invariant violation - roll back first.
+            try {
+                serviceLocator
+                    .getLogger()
+                    .warning('Internal error: a storage transaction was disposed while still open; rolling it back.');
+            }
+            catch {
+                // Never throw from dispose.
+            }
+            this.rollback();
+        }
+        this.#disposed = true;
+        this.journal.length = 0;
+    }
+    get datasetItems() {
+        return this.journal.flatMap((entry) => entry.type === 'dataset' ? entry.items.map((item) => ({ item, datasetId: entry.storageId })) : []);
+    }
+    get enqueuedUrls() {
+        return this.journal.flatMap((entry) => entry.type === 'requestQueue' ? entry.requests.map(({ url, label }) => ({ url, label })) : []);
+    }
+    get keyValueStoreChanges() {
+        const result = {};
+        for (const entry of this.journal) {
+            if (entry.type !== 'keyValueStore')
+                continue;
+            result[entry.storageId] ??= {};
+            result[entry.storageId][entry.key] = { changedValue: entry.value, options: entry.options };
+        }
+        return result;
+    }
+}
+/**
+ * Opens a {@link StorageTransaction} without running anything yet. The caller owns the outcome:
+ * `run()`, then `commit()` or `rollback()`, and always `dispose()` when done. For the common
+ * open-run-commit flow, prefer {@link withStorageTransaction}.
+ */
+export function createStorageTransaction(options = {}) {
+    return new StorageTransaction(options);
+}
+/**
+ * Runs `callback` inside a new {@link StorageTransaction}: storage writes made in the callback are
+ * committed when it returns and rolled back when it throws. If a transaction is already active in the
+ * current async context, it is reused and its outcome is left to its owner (and `options` are ignored)
+ * — there are no nested transaction semantics.
+ */
+export async function withStorageTransaction(callback, options = {}) {
+    const existing = transactionStorage.getStore();
+    if (existing?.isActive) {
+        return callback(existing);
+    }
+    const transaction = createStorageTransaction(options);
+    try {
+        const result = await transaction.run(async () => callback(transaction));
+        await transaction.commit();
+        return result;
+    }
+    catch (error) {
+        transaction.rollback();
+        throw error;
+    }
+    finally {
+        transaction.dispose();
+    }
+}
+/**
+ * Runs `callback` outside of any storage transaction — the per-call-site escape hatch. Storage operations
+ * made inside it hit the real backend directly, are not rolled back, and operations that a transaction
+ * rejects (`drop`, stream-valued `setValue`, request queue internals, ...) are permitted.
+ */
+export async function withDirectStorageAccess(callback) {
+    return transactionStorage.exit(async () => callback());
+}
+/**
+ * The per-operation hook consulted by every storage frontend method: performs the cancellation check
+ * that aborts storage operations when the request handler times out, and returns the active storage
+ * transaction. Returns `undefined` when there is no transaction in the async context *or* when it is no
+ * longer open — operations on a closed transaction deliberately pass through to the real backend.
+ * @internal
+ */
+export function activeStorageTransaction() {
+    tryCancel();
+    const transaction = transactionStorage.getStore();
+    return transaction?.isActive ? transaction : undefined;
+}
+/**
+ * Returns the transaction installed in the current async context, regardless of its state. Used by the
+ * crawler to drive the outcome of the transaction it opened.
+ * @internal
+ */
+export function currentStorageTransaction() {
+    return transactionStorage.getStore();
+}
+/**
+ * Captures a value at write time, so that later mutations of the caller's object affect neither the
+ * read-your-own-writes reads nor the commit replay. `structuredClone` for fidelity (`Date`, `Map`, `Set`,
+ * typed arrays, `undefined`); values it cannot handle fall back to the JSON round-trip the storage
+ * backends perform anyway.
+ * @internal
+ */
+export function snapshotValue(value) {
+    try {
+        return structuredClone(value);
+    }
+    catch {
+        return JSON.parse(JSON.stringify(value));
+    }
+}
+/**
+ * The guard for operations that cannot be performed inside a storage transaction: throws when one is
+ * active, and performs the per-operation cancellation check either way.
+ * @internal
+ */
+export function rejectOperationInTransaction(operation, reason = 'it cannot be rolled back.') {
+    if (activeStorageTransaction() === undefined) {
+        return;
+    }
+    throw operationRejectedInTransaction(operation, reason);
+}
+/**
+ * Builds the "operation not allowed in a transaction" error, for a call site that has already
+ * established a transaction is active and so wants to `throw` unconditionally.
+ * @internal
+ */
+export function operationRejectedInTransaction(operation, reason = 'it cannot be rolled back.') {
+    return new Error(`${operation} cannot be used inside a storage transaction: ${reason} ` +
+        'If you really need it, wrap the call in withDirectStorageAccess(() => ...) - operations ' +
+        'performed there are applied immediately and are not rolled back.');
+}

package/storages/utils.d.ts

@@ -1,4 +1,5 @@
-import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 /**
@@ -14,7 +15,8 @@ interface PurgeDefaultStorageOptions {
 }
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
@@ -28,7 +30,8 @@ interface PurgeDefaultStorageOptions {
 export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
 /**
  * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
- * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ * Purging empties the storages that belong to a single run — the default one and every alias-keyed one —
+ * keeping only INPUT.json in the default KV store. Named storages persist across runs and are not touched.
  *
  * Purging of storages is happening automatically when we run our crawler (or when we open some storage
  * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}

package/storages/utils.js

@@ -12,11 +12,20 @@ export async function purgeDefaultStorages(configurationOrOptions, storageBacken
     const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
     ({ storageBackend = serviceLocator.getStorageBackend() } = options);
     const casted = storageBackend;
+    const runPurge = async () => {
+        try {
+            await casted.purge?.();
+        }
+        catch (e) {
+            casted.__purged = undefined;
+            throw e;
+        }
+    };
     // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
     if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
-        casted.__purged = true;
-        await casted.purge?.();
+        casted.__purged = runPurge();
     }
+    await casted.__purged;
 }
 /**
  * Easily create and manage state values. All state values are automatically persisted.

package/system-info/runtime.js

@@ -50,31 +50,31 @@ export async function isContainerized() {
 export function isLambda() {
     return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
 }
-let _cgroupsVersion;
+let cgroupsVersion;
 /**
  * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
  * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
  */
 export async function getCgroupsVersion(forceReset) {
     // Parameter forceReset is just internal for unit tests.
-    if (_cgroupsVersion !== undefined && !forceReset) {
-        return _cgroupsVersion;
+    if (cgroupsVersion !== undefined && !forceReset) {
+        return cgroupsVersion;
     }
     try {
         // If this directory does not exists, cgroups are not available
         await fs.access('/sys/fs/cgroup/');
     }
     catch (e) {
-        _cgroupsVersion = null;
+        cgroupsVersion = null;
         return null;
     }
-    _cgroupsVersion = 'V1';
+    cgroupsVersion = 'V1';
     try {
         // If this directory does not exists, assume the container is using cgroups V2
         await fs.access('/sys/fs/cgroup/memory/');
     }
     catch (e) {
-        _cgroupsVersion = 'V2';
+        cgroupsVersion = 'V2';
     }
-    return _cgroupsVersion;
+    return cgroupsVersion;
 }

package/url.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The canonical form of a hostname: lower-case, punycode, and without the optional root dot.
+ *
+ * Pass both sides of a hostname comparison through this, so that a domain written as `háčky.cz` still matches the
+ * `xn--hky-ela4t.cz` that `URL` reports.
+ *
+ * @internal
+ */
+export declare function normalizeHostname(hostname: string): string;

package/url.js

@@ -0,0 +1,11 @@
+/**
+ * The canonical form of a hostname: lower-case, punycode, and without the optional root dot.
+ *
+ * Pass both sides of a hostname comparison through this, so that a domain written as `háčky.cz` still matches the
+ * `xn--hky-ela4t.cz` that `URL` reports.
+ *
+ * @internal
+ */
+export function normalizeHostname(hostname) {
+    return hostname.toLowerCase().replace(/\.$/, '');
+}

package/validators.d.ts

@@ -1,28 +1,25 @@
-import type { Dictionary } from '@crawlee/types';
+export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export declare const validators: {
-    browserPage: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    proxyConfiguration: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestList: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestQueue: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    browserPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    sessionPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPage: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    proxyConfiguration: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestList: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestQueue: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    sessionPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestManager: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    storageBackend: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    logger: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    httpClient: import("zod").ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>;
 };