@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/memory-storage/utils.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
+ * Storage names and record keys are used as filesystem path components, so a value containing `..`
+ * or an absolute path could otherwise escape the intended directory.
+ */
+export declare function resolveWithinDirectory(baseDirectory: string, segment: string): string;
+/**
+ * Removes all properties with a null value
+ * from the provided object.
+ */
+export declare function purgeNullsFromObject<T>(object: T): T;
+/**
+ * Creates a standard request ID (same as Platform).
+ */
+export declare function uniqueKeyToRequestId(uniqueKey: string): string;
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/memory-storage/utils.js

@@ -0,0 +1,41 @@
+import { createHash } from 'node:crypto';
+import { resolve, sep } from 'node:path';
+import { REQUEST_ID_LENGTH } from './consts.js';
+/**
+ * Resolves `segment` against `baseDirectory` and ensures the result stays within `baseDirectory`.
+ * Storage names and record keys are used as filesystem path components, so a value containing `..`
+ * or an absolute path could otherwise escape the intended directory.
+ */
+export function resolveWithinDirectory(baseDirectory, segment) {
+    const base = resolve(baseDirectory);
+    const resolved = resolve(base, segment);
+    if (resolved !== base && !resolved.startsWith(`${base}${sep}`)) {
+        throw new Error(`"${segment}" is not allowed because it would resolve outside of the storage directory. ` +
+            `Storage names and record keys must not contain path traversal segments ("..") or absolute paths.`);
+    }
+    return resolved;
+}
+/**
+ * Removes all properties with a null value
+ * from the provided object.
+ */
+export function purgeNullsFromObject(object) {
+    if (object && typeof object === 'object' && !Array.isArray(object)) {
+        for (const [key, value] of Object.entries(object)) {
+            if (value === null)
+                Reflect.deleteProperty(object, key);
+        }
+    }
+    return object;
+}
+/**
+ * Creates a standard request ID (same as Platform).
+ */
+export function uniqueKeyToRequestId(uniqueKey) {
+    const str = createHash('sha256')
+        .update(uniqueKey)
+        .digest('base64')
+        .replace(/(\+|\/|=)/g, '');
+    return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
+}
+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';

package/owned_or_injected.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Captures the "inject-or-default" ownership pattern for a crawler's stateful collaborators (session pool, browser
+ * pool, the crawler-opened request queue, ...). A collaborator is either **injected** by the user (borrowed — the
+ * crawler never drives its lifecycle) or **built by the crawler** as a default (owned — the crawler sequences its
+ * lifecycle). Owned-only lifecycle hooks are gated through a single {@link OwnedOrInjected.ifOwned|`ifOwned()`}.
+ *
+ * This is the deliberate complement to the {@link ServiceLocator}: ambient infrastructure (config/events/storage/
+ * logger) stays in the locator, while crawler-scoped stateful collaborators the crawler sequences go through this.
+ *
+ * @internal
+ */
+export declare class OwnedOrInjected<Injected, Owned extends Injected = Injected> {
+    #private;
+    private constructor();
+    /**
+     * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
+     * {@link OwnedOrInjected.value|`value`}; `Owned` is the concrete type the crawler builds (a subtype with extra
+     * lifecycle methods), which {@link OwnedOrInjected.set|`set()`} and {@link OwnedOrInjected.ifOwned|`ifOwned()`}
+     * deal in — so owned-only lifecycle hooks are statically typed with no casts.
+     *
+     * - `injected` provided → borrowed (present, not owned).
+     * - `injected` omitted → owned; `buildDefault` fills the slot eagerly if given, otherwise it stays empty until a
+     *   later {@link OwnedOrInjected.set|`set()`} (the lazy case, e.g. a request queue opened on first use).
+     */
+    static resolve<Injected, Owned extends Injected = Injected>(injected?: Injected, buildDefault?: () => Owned): OwnedOrInjected<Injected, Owned>;
+    /**
+     * Whether the crawler owns the instance and is therefore responsible for its lifecycle. `true` only for
+     * crawler-built defaults, `false` for user-injected instances.
+     */
+    get isOwned(): boolean;
+    /**
+     * Whether a value is currently available. `false` for an owned slot whose default hasn't been built yet
+     * (e.g. a lazily-opened request queue before its first use).
+     */
+    get isPresent(): boolean;
+    /**
+     * The resolved instance, typed as the public `Injected` type. Throws if the value is not present yet — callers that
+     * expect a lazily-filled owned slot should read {@link OwnedOrInjected.maybeValue|`maybeValue`} instead.
+     */
+    get value(): Injected;
+    /**
+     * The resolved instance, or `undefined` when a lazily-filled owned slot hasn't been built yet. The non-throwing
+     * counterpart to {@link OwnedOrInjected.value|`value`} — pairs naturally with `?? fallback` so callers can read
+     * a possibly-empty slot without the `isPresent ? value : …` dance.
+     */
+    get maybeValue(): Injected | undefined;
+    /**
+     * Fills the (owned) slot with the crawler-built default, returning it for convenience. Only valid on an owned,
+     * not-yet-filled slot: borrowed instances are never replaced and an owned slot is filled exactly once (re-setting
+     * would silently orphan the previous instance's lifecycle).
+     */
+    set(value: Owned): Owned;
+    /**
+     * Runs an owned-only lifecycle hook, invoked (with the value typed as the concrete `Owned`) only when the crawler
+     * owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.
+     */
+    ifOwned<R>(fn: (value: Owned) => R | Promise<R>): Promise<R | undefined>;
+}

package/owned_or_injected.js

@@ -0,0 +1,98 @@
+/**
+ * Captures the "inject-or-default" ownership pattern for a crawler's stateful collaborators (session pool, browser
+ * pool, the crawler-opened request queue, ...). A collaborator is either **injected** by the user (borrowed — the
+ * crawler never drives its lifecycle) or **built by the crawler** as a default (owned — the crawler sequences its
+ * lifecycle). Owned-only lifecycle hooks are gated through a single {@link OwnedOrInjected.ifOwned|`ifOwned()`}.
+ *
+ * This is the deliberate complement to the {@link ServiceLocator}: ambient infrastructure (config/events/storage/
+ * logger) stays in the locator, while crawler-scoped stateful collaborators the crawler sequences go through this.
+ *
+ * @internal
+ */
+export class OwnedOrInjected {
+    #value;
+    #owned;
+    #present;
+    constructor(value, owned, present) {
+        this.#value = value;
+        this.#owned = owned;
+        this.#present = present;
+    }
+    /**
+     * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
+     * {@link OwnedOrInjected.value|`value`}; `Owned` is the concrete type the crawler builds (a subtype with extra
+     * lifecycle methods), which {@link OwnedOrInjected.set|`set()`} and {@link OwnedOrInjected.ifOwned|`ifOwned()`}
+     * deal in — so owned-only lifecycle hooks are statically typed with no casts.
+     *
+     * - `injected` provided → borrowed (present, not owned).
+     * - `injected` omitted → owned; `buildDefault` fills the slot eagerly if given, otherwise it stays empty until a
+     *   later {@link OwnedOrInjected.set|`set()`} (the lazy case, e.g. a request queue opened on first use).
+     */
+    static resolve(injected, buildDefault) {
+        if (injected !== undefined) {
+            return new OwnedOrInjected(injected, false, true);
+        }
+        if (buildDefault !== undefined) {
+            return new OwnedOrInjected(buildDefault(), true, true);
+        }
+        return new OwnedOrInjected(undefined, true, false);
+    }
+    /**
+     * Whether the crawler owns the instance and is therefore responsible for its lifecycle. `true` only for
+     * crawler-built defaults, `false` for user-injected instances.
+     */
+    get isOwned() {
+        return this.#owned;
+    }
+    /**
+     * Whether a value is currently available. `false` for an owned slot whose default hasn't been built yet
+     * (e.g. a lazily-opened request queue before its first use).
+     */
+    get isPresent() {
+        return this.#present;
+    }
+    /**
+     * The resolved instance, typed as the public `Injected` type. Throws if the value is not present yet — callers that
+     * expect a lazily-filled owned slot should read {@link OwnedOrInjected.maybeValue|`maybeValue`} instead.
+     */
+    get value() {
+        if (!this.#present) {
+            throw new Error('OwnedOrInjected value is not initialized yet');
+        }
+        return this.#value;
+    }
+    /**
+     * The resolved instance, or `undefined` when a lazily-filled owned slot hasn't been built yet. The non-throwing
+     * counterpart to {@link OwnedOrInjected.value|`value`} — pairs naturally with `?? fallback` so callers can read
+     * a possibly-empty slot without the `isPresent ? value : …` dance.
+     */
+    get maybeValue() {
+        return this.#present ? this.#value : undefined;
+    }
+    /**
+     * Fills the (owned) slot with the crawler-built default, returning it for convenience. Only valid on an owned,
+     * not-yet-filled slot: borrowed instances are never replaced and an owned slot is filled exactly once (re-setting
+     * would silently orphan the previous instance's lifecycle).
+     */
+    set(value) {
+        if (!this.#owned) {
+            throw new Error('Cannot set() a borrowed OwnedOrInjected value');
+        }
+        if (this.#present) {
+            throw new Error('OwnedOrInjected value is already initialized');
+        }
+        this.#value = value;
+        this.#present = true;
+        return value;
+    }
+    /**
+     * Runs an owned-only lifecycle hook, invoked (with the value typed as the concrete `Owned`) only when the crawler
+     * owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.
+     */
+    async ifOwned(fn) {
+        if (!this.#owned || !this.#present) {
+            return undefined;
+        }
+        return fn(this.#value);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.13",
+    "version": "4.0.0-beta.131",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -50,25 +50,26 @@
         "@apify/consts": "^2.41.0",
         "@apify/datastructures": "^2.0.3",
         "@apify/log": "^2.5.18",
-        "@apify/pseudo_url": "^2.0.59",
-        "@apify/timeout": "^0.3.2",
+        "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.13",
-        "@crawlee/types": "4.0.0-beta.13",
-        "@crawlee/utils": "4.0.0-beta.13",
+        "@crawlee/fs-storage": "4.0.0-beta.131",
+        "@crawlee/http-client": "4.0.0-beta.131",
+        "@crawlee/types": "4.0.0-beta.131",
+        "@crawlee/utils": "4.0.0-beta.131",
         "@sapphire/async-queue": "^1.5.5",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
+        "content-type": "^1.0.5",
         "csv-stringify": "^6.5.2",
-        "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "json5": "^2.2.3",
+        "mime-types": "^3.0.1",
         "minimatch": "^10.0.1",
-        "ow": "^2.0.0",
         "stream-json": "^1.9.1",
         "tldts": "^7.0.6",
-        "tough-cookie": "^5.1.2",
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "type-fest": "^4.41.0",
+        "zod": "^4.4.3"
     },
     "lerna": {
         "command": {
@@ -77,5 +78,5 @@
             }
         }
     },
-    "gitHead": "24df7c8dba6b6c9623c02ccc655d8ee748f003e3"
+    "gitHead": "de741c0b63d0da061e4ffcd6ce6fdc0c92401399"
 }

package/proxy_configuration.d.ts

@@ -1,16 +1,18 @@
+import type { ProxyInfo } from '@crawlee/types';
 import type { Request } from './request.js';
 export interface ProxyConfigurationFunction {
     (options?: {
         request?: Request;
     }): string | null | Promise<string | null>;
 }
+type UrlList = (string | null)[];
 export interface ProxyConfigurationOptions {
     /**
      * An array of custom proxy URLs to be rotated.
      * Custom proxies are not compatible with Apify Proxy and an attempt to use both
      * configuration options will cause an error to be thrown on initialize.
      */
-    proxyUrls?: string[];
+    proxyUrls?: UrlList;
     /**
      * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
      * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
@@ -18,111 +20,26 @@ export interface ProxyConfigurationOptions {
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
      */
     newUrlFunction?: ProxyConfigurationFunction;
-    /**
-     * An array of custom proxy URLs to be rotated stratified in tiers.
-     * This is a more advanced version of `proxyUrls` that allows you to define a hierarchy of proxy URLs
-     * If everything goes well, all the requests will be sent through the first proxy URL in the list.
-     * Whenever the crawler encounters a problem with the current proxy on the given domain, it will switch to the higher tier for this domain.
-     * The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
-     *
-     * This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
-     *
-     * Use `null` as a proxy URL to disable the proxy for the given tier.
-     */
-    tieredProxyUrls?: (string | null)[][];
-}
-export interface TieredProxy {
-    proxyUrl: string | null;
-    proxyTier?: number;
-}
-/**
- * The main purpose of the ProxyInfo object is to provide information
- * about the current proxy connection used by the crawler for the request.
- * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
- *
- * **Example usage:**
- *
- * ```javascript
- * const proxyConfiguration = new ProxyConfiguration({
- *   proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
- * });
- *
- * // Getting proxyInfo object by calling class method directly
- * const proxyInfo = await proxyConfiguration.newProxyInfo();
- *
- * // In crawler
- * const crawler = new CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   requestHandler({ proxyInfo }) {
- *      // Getting used proxy URL
- *       const proxyUrl = proxyInfo.url;
- *   }
- * })
- *
- * ```
- */
-export interface ProxyInfo {
-    /**
-     * The URL of the proxy.
-     */
-    url: string;
-    /**
-     * Username for the proxy.
-     */
-    username?: string;
-    /**
-     * User's password for the proxy.
-     */
-    password: string;
-    /**
-     * Hostname of your proxy.
-     */
-    hostname: string;
-    /**
-     * Proxy port.
-     */
-    port: number | string;
-    /**
-     * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
-     */
-    proxyTier?: number;
-    /**
-     * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
-     *
-     * @default false
-     */
-    ignoreTlsErrors?: boolean;
 }
-interface TieredProxyOptions {
+interface NewUrlOptions {
     request?: Request;
-    proxyTier?: number;
 }
 /**
- * Internal class for tracking the proxy tier history for a specific domain.
+ * Minimal contract that any object passed to a crawler as its `proxyConfiguration`
+ * option must satisfy.
+ *
+ * Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee
+ * crawler — for example a remote proxy service or a thin wrapper around the built-in
+ * `ProxyConfiguration` with different rotation rules. *
  *
- * Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
+ * @category Scaling
  */
-declare class ProxyTierTracker {
-    private histogram;
-    private currentTier;
-    constructor(tieredProxyUrls: (string | null)[][]);
+export interface IProxyConfiguration {
     /**
-     * Processes a single step of the algorithm and updates the current tier prediction based on the error history.
+     * Creates a new {@link ProxyInfo} object describing the proxy to use for the given
+     * request. Returns `undefined` when no proxy should be used.
      */
-    private processStep;
-    /**
-     * Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
-     *
-     * The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
-     * @param tier The proxy tier to mark as problematic.
-     */
-    addError(tier: number): void;
-    /**
-     * Returns the best proxy tier for the next request based on the error history for different proxy tiers.
-     * @returns The proxy tier prediction
-     */
-    predictTier(): number;
+    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
 }
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
@@ -152,16 +69,9 @@ declare class ProxyTierTracker {
  * ```
  * @category Scaling
  */
-export declare class ProxyConfiguration {
-    isManInTheMiddle: boolean;
-    protected nextCustomUrlIndex: number;
-    protected proxyUrls?: string[];
-    protected tieredProxyUrls?: (string | null)[][];
-    protected usedProxyUrls: Map<string, string>;
-    protected newUrlFunction?: ProxyConfigurationFunction;
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected log: import("@apify/log").Log;
-    protected domainTiers: Map<string, ProxyTierTracker>;
+export declare class ProxyConfiguration implements IProxyConfiguration {
+    #private;
+    readonly isManInTheMiddle = false;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -192,38 +102,20 @@ export declare class ProxyConfiguration {
      *
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(options?: TieredProxyOptions): Promise<ProxyInfo | undefined>;
-    /**
-     * Given a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
-     * @param options Options for the tiered proxy rotation
-     * @returns An object with the proxy URL and the proxy tier used.
-     */
-    protected _handleTieredUrl(options?: TieredProxyOptions): TieredProxy;
-    /**
-     * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
-     *
-     * This returns `null` if `tieredProxyUrls` option is not set.
-     */
-    protected predictProxyTier(request: Request): number | null;
+    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
     /**
      * Returns a new proxy URL based on provided configuration options.
      *
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(options?: TieredProxyOptions): Promise<string | undefined>;
-    /**
-     * Handles custom url rotation
-     */
-    protected _handleProxyUrlsList(): string;
+    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
+    private handleProxyUrlsList;
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    protected _callNewUrlFunction(options?: {
-        request?: Request;
-    }): Promise<string | null>;
-    protected _throwCannotCombineCustomMethods(): never;
-    protected _throwNoOptionsProvided(): never;
+    private callNewUrlFunction;
+    private throwCannotCombineCustomMethods;
+    private throwNoOptionsProvided;
 }
 export {};
-//# sourceMappingURL=proxy_configuration.d.ts.map