@crawlee/core 4.0.0-beta.5 → 4.0.0-beta.51

package/log.d.ts

@@ -1,3 +1,83 @@
-import log, { Log, Logger, LoggerJson, LoggerOptions, LoggerText, LogLevel } from '@apify/log';
-export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText };
+import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+import type { LoggerOptions } from '@apify/log';
+import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+export type { CrawleeLogger, CrawleeLoggerOptions };
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
+    private options;
+    private readonly warningsLogged;
+    constructor(options?: Partial<CrawleeLoggerOptions>);
+    /**
+     * Core logging method. Subclasses must implement this to dispatch log messages
+     * to the underlying logger (Winston, Pino, console, etc.).
+     *
+     * Level filtering is the responsibility of the underlying library — this method
+     * is called for every message regardless of the current level.
+     *
+     * @param level Crawlee log level (use {@link LogLevel} constants)
+     * @param message The log message
+     * @param data Optional structured data to attach to the log entry
+     */
+    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    /**
+     * Creates a child logger instance. Subclasses must implement this to define
+     * how child loggers are created for the underlying logger.
+     */
+    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    getOptions(): CrawleeLoggerOptions;
+    setOptions(options: Partial<CrawleeLoggerOptions>): void;
+    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    error(message: string, data?: Record<string, unknown>): void;
+    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>): void;
+    warningOnce(message: string): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>): void;
+    deprecated(message: string): void;
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export declare class ApifyLogAdapter extends BaseCrawleeLogger {
+    private readonly apifyLog;
+    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
+    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+}
+export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
+export type { LoggerOptions };
 //# sourceMappingURL=log.d.ts.map

package/log.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEhF,YAAY,EAAE,aAAa,EAAE,oBAAoB,EAAE,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,8BAAsB,iBAAkB,YAAW,aAAa;IAC5D,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;gBAExC,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM;IAIvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAE3F;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAErF,UAAU,IAAI,oBAAoB;IAIlC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAIxD,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;IAI5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAQlF,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI/D,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI9D,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAOlC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;CAGpC;AAED;;;;;GAKG;AACH,qBAAa,eAAgB,SAAQ,iBAAiB;IAE9C,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,GAAG,EAC9B,OAAO,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC;IAK3C,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAIlF,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,aAAa;CAM/E;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AAC9D,YAAY,EAAE,aAAa,EAAE,CAAC"}

package/log.js

@@ -1,3 +1,105 @@
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export class BaseCrawleeLogger {
+    options;
+    warningsLogged = new Set();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    getOptions() {
+        return this.options;
+    }
+    setOptions(options) {
+        this.options = { ...this.options, ...options };
+    }
+    child(options) {
+        return this.createChild(options);
+    }
+    error(message, data) {
+        this.logWithLevel(LogLevel.ERROR, message, data);
+    }
+    exception(exception, message, data) {
+        this.logWithLevel(LogLevel.ERROR, `${message}: ${exception.message}`, {
+            ...data,
+            stack: exception.stack,
+            exception,
+        });
+    }
+    softFail(message, data) {
+        this.logWithLevel(LogLevel.SOFT_FAIL, message, data);
+    }
+    warning(message, data) {
+        this.logWithLevel(LogLevel.WARNING, message, data);
+    }
+    warningOnce(message) {
+        if (!this.warningsLogged.has(message)) {
+            this.warningsLogged.add(message);
+            this.warning(message);
+        }
+    }
+    info(message, data) {
+        this.logWithLevel(LogLevel.INFO, message, data);
+    }
+    debug(message, data) {
+        this.logWithLevel(LogLevel.DEBUG, message, data);
+    }
+    perf(message, data) {
+        this.logWithLevel(LogLevel.PERF, `[PERF] ${message}`, data);
+    }
+    deprecated(message) {
+        this.warningOnce(`[DEPRECATED] ${message}`);
+    }
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export class ApifyLogAdapter extends BaseCrawleeLogger {
+    apifyLog;
+    constructor(apifyLog, options) {
+        super(options ?? {});
+        this.apifyLog = apifyLog;
+    }
+    logWithLevel(level, message, data) {
+        this.apifyLog.internal(level, message, data);
+    }
+    createChild(options) {
+        return new ApifyLogAdapter(this.apifyLog.child({ prefix: options.prefix ?? null }), {
+            ...this.getOptions(),
+            ...options,
+        });
+    }
+}
 export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
 //# sourceMappingURL=log.js.map

package/log.js.map

@@ -1 +1 @@
-{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAiB,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EAAE,GAAG,EAAE,GAAG,EAAiB,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAGA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAIhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAgB,iBAAiB;IAC3B,OAAO,CAAuB;IACrB,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpD,YAAY,UAAyC,EAAE;QACnD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAqBD,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,UAAU,CAAC,OAAsC;QAC7C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAsC;QACxC,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,SAAS,CAAC,SAAgB,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,OAAO,KAAK,SAAS,CAAC,OAAO,EAAE,EAAE;YAClE,GAAG,IAAI;YACP,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,SAAS;SACZ,CAAC,CAAC;IACP,CAAC;IAED,QAAQ,CAAC,OAAe,EAAE,IAA8B;QACpD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,OAAe,EAAE,IAA8B;QACnD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,WAAW,CAAC,OAAe;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,IAA8B;QACjD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,IAA8B;QAChD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,UAAU,CAAC,OAAe;QACtB,IAAI,CAAC,WAAW,CAAC,gBAAgB,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;CACJ;AAED;;;;;GAKG;AACH,MAAM,OAAO,eAAgB,SAAQ,iBAAiB;IAE7B;IADrB,YACqB,QAAa,EAC9B,OAAuC;QAEvC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAHJ,aAAQ,GAAR,QAAQ,CAAK;IAIlC,CAAC;IAED,YAAY,CAAC,KAAa,EAAE,OAAe,EAAE,IAA8B;QACvE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7D,CAAC;IAES,WAAW,CAAC,OAAsC;QACxD,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,EAAE;YAChF,GAAG,IAAI,CAAC,UAAU,EAAE;YACpB,GAAG,OAAO;SACb,CAAC,CAAC;IACP,CAAC;CACJ;AAED,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.5",
+    "version": "4.0.0-beta.51",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -53,22 +53,21 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.5",
-        "@crawlee/types": "4.0.0-beta.5",
-        "@crawlee/utils": "4.0.0-beta.5",
+        "@crawlee/memory-storage": "4.0.0-beta.51",
+        "@crawlee/types": "4.0.0-beta.51",
+        "@crawlee/utils": "4.0.0-beta.51",
         "@sapphire/async-queue": "^1.5.5",
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "csv-stringify": "^6.5.2",
-        "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "json5": "^2.2.3",
         "minimatch": "^10.0.1",
         "ow": "^2.0.0",
         "stream-json": "^1.9.1",
         "tldts": "^7.0.6",
-        "tough-cookie": "^5.1.2",
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "type-fest": "^4.41.0",
+        "zod": "^3.24.0 || ^4.0.0"
     },
     "lerna": {
         "command": {
@@ -77,5 +76,5 @@
             }
         }
     },
-    "gitHead": "9db9799f5eb88ab29f6156ee1574140b438aa199"
+    "gitHead": "321493aea4dfb8073bb5e5d112c331645cac3d11"
 }

package/proxy_configuration.d.ts

@@ -1,129 +1,28 @@
+import type { ProxyInfo } from '@crawlee/types';
 import type { Request } from './request.js';
 export interface ProxyConfigurationFunction {
-    (sessionId: string | number, options?: {
+    (options?: {
         request?: Request;
     }): string | null | Promise<string | null>;
 }
+type UrlList = (string | null)[];
 export interface ProxyConfigurationOptions {
     /**
      * An array of custom proxy URLs to be rotated.
      * Custom proxies are not compatible with Apify Proxy and an attempt to use both
      * configuration options will cause an error to be thrown on initialize.
      */
-    proxyUrls?: string[];
+    proxyUrls?: UrlList;
     /**
-     * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter and an optional parameter with the `Request` object when applicable.
+     * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
      * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
      *
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
      */
     newUrlFunction?: ProxyConfigurationFunction;
-    /**
-     * An array of custom proxy URLs to be rotated stratified in tiers.
-     * This is a more advanced version of `proxyUrls` that allows you to define a hierarchy of proxy URLs
-     * If everything goes well, all the requests will be sent through the first proxy URL in the list.
-     * Whenever the crawler encounters a problem with the current proxy on the given domain, it will switch to the higher tier for this domain.
-     * The crawler probes lower-level proxies at intervals to check if it can make the tier downshift.
-     *
-     * This feature is useful when you have a set of proxies with different performance characteristics (speed, price, antibot performance etc.) and you want to use the best one for each domain.
-     *
-     * Use `null` as a proxy URL to disable the proxy for the given tier.
-     */
-    tieredProxyUrls?: (string | null)[][];
 }
-export interface TieredProxy {
-    proxyUrl: string | null;
-    proxyTier?: number;
-}
-/**
- * The main purpose of the ProxyInfo object is to provide information
- * about the current proxy connection used by the crawler for the request.
- * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
- *
- * **Example usage:**
- *
- * ```javascript
- * const proxyConfiguration = new ProxyConfiguration({
- *   proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
- * });
- *
- * // Getting proxyInfo object by calling class method directly
- * const proxyInfo = await proxyConfiguration.newProxyInfo();
- *
- * // In crawler
- * const crawler = new CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   requestHandler({ proxyInfo }) {
- *      // Getting used proxy URL
- *       const proxyUrl = proxyInfo.url;
- *
- *      // Getting ID of used Session
- *       const sessionIdentifier = proxyInfo.sessionId;
- *   }
- * })
- *
- * ```
- */
-export interface ProxyInfo {
-    /**
-     * The identifier of used {@link Session}, if used.
-     */
-    sessionId?: string;
-    /**
-     * The URL of the proxy.
-     */
-    url: string;
-    /**
-     * Username for the proxy.
-     */
-    username?: string;
-    /**
-     * User's password for the proxy.
-     */
-    password: string;
-    /**
-     * Hostname of your proxy.
-     */
-    hostname: string;
-    /**
-     * Proxy port.
-     */
-    port: number | string;
-    /**
-     * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
-     */
-    proxyTier?: number;
-}
-interface TieredProxyOptions {
+interface NewUrlOptions {
     request?: Request;
-    proxyTier?: number;
-}
-/**
- * Internal class for tracking the proxy tier history for a specific domain.
- *
- * Predicts the best proxy tier for the next request based on the error history for different proxy tiers.
- */
-declare class ProxyTierTracker {
-    private histogram;
-    private currentTier;
-    constructor(tieredProxyUrls: (string | null)[][]);
-    /**
-     * Processes a single step of the algorithm and updates the current tier prediction based on the error history.
-     */
-    private processStep;
-    /**
-     * Increases the error score for the given proxy tier. This raises the chance of picking a different proxy tier for the subsequent requests.
-     *
-     * The error score is increased by 10 for the given tier. This means that this tier will be disadvantaged for the next 10 requests (every new request prediction decreases the error score by 1).
-     * @param tier The proxy tier to mark as problematic.
-     */
-    addError(tier: number): void;
-    /**
-     * Returns the best proxy tier for the next request based on the error history for different proxy tiers.
-     * @returns The proxy tier prediction
-     */
-    predictTier(): number;
 }
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
@@ -156,13 +55,11 @@ declare class ProxyTierTracker {
 export declare class ProxyConfiguration {
     isManInTheMiddle: boolean;
     protected nextCustomUrlIndex: number;
-    protected proxyUrls?: string[];
-    protected tieredProxyUrls?: (string | null)[][];
-    protected usedProxyUrls: Map<string, string>;
+    protected proxyUrls?: UrlList;
+    protected usedProxyUrls: Map<string, string | null>;
     protected newUrlFunction?: ProxyConfigurationFunction;
 // @ts-ignore optional peer dependency or compatibility with es2022
-    protected log: import("@apify/log").Log;
-    protected domainTiers: Map<string, ProxyTierTracker>;
+    protected log: import("@crawlee/types").CrawleeLogger;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -190,53 +87,22 @@ export declare class ProxyConfiguration {
      * the currently used proxy via the requestHandler parameter `proxyInfo`.
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
      *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(sessionId?: string | number, options?: TieredProxyOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
     /**
-     * Given a session identifier and a request / proxy tier, this function returns a new proxy URL based on the provided configuration options.
-     * @param _sessionId Session identifier
-     * @param options Options for the tiered proxy rotation
-     * @returns An object with the proxy URL and the proxy tier used.
-     */
-    protected _handleTieredUrl(_sessionId: string, options?: TieredProxyOptions): TieredProxy;
-    /**
-     * Given a `Request` object, this function returns the tier of the proxy that should be used for the request.
+     * Returns a new proxy URL based on provided configuration options.
      *
-     * This returns `null` if `tieredProxyUrls` option is not set.
-     */
-    protected predictProxyTier(request: Request): number | null;
-    /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(sessionId?: string | number, options?: TieredProxyOptions): Promise<string | undefined>;
-    /**
-     * Handles custom url rotation with session
-     */
-    protected _handleCustomUrl(sessionId?: string): string;
+    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
+    protected _handleProxyUrlsList(): string | null;
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    protected _callNewUrlFunction(sessionId?: string, options?: {
+    protected _callNewUrlFunction(options?: {
         request?: Request;
     }): Promise<string | null>;
     protected _throwCannotCombineCustomMethods(): never;

package/proxy_configuration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;IAE5C;;;;;;;;;;OAUG;IACH,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;CACzC;AAED,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,kBAAkB;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,cAAM,gBAAgB;IAClB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,WAAW,CAAS;gBAEhB,eAAe,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE;IAKhD;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB;;;OAGG;IACH,WAAW;CAId;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IAC3B,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAChD,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAC5D,SAAS,CAAC,WAAW,gCAAuC;IAE5D;;;;;;;;;;;;;;;;;;;OAmBG;gBACS,OAAO,GAAE,yBAA8B;IAwBnD;;;;;;;;;;;;;;;;OAgBG;IACG,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IA4B7G;;;;;OAKG;IACH,SAAS,CAAC,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,WAAW;IAwBzF;;;;OAIG;IACH,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAiC3D;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAcpG;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;cACa,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE;IAcvF,SAAS,CAAC,gCAAgC,IAAI,KAAK;IAMnD,SAAS,CAAC,uBAAuB,IAAI,KAAK;CAG7C"}
+{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,WAAW,0BAA0B;IACvC,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC7E;AAED,KAAK,OAAO,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;AAEjC,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED,UAAU,aAAa;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IAC3B,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,aAAa,6BAAoC;IAC3D,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,yCAAsE;IAEnF;;;;;;;;;;;;;;;;;;;OAmBG;gBACS,OAAO,GAAE,yBAA8B;IA2BnD;;;;;;;;OAQG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAe3E;;;;;OAKG;IACG,MAAM,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAQlE,SAAS,CAAC,oBAAoB,IAAI,MAAM,GAAG,IAAI;IAI/C;;OAEG;cACa,mBAAmB,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE;IAcnE,SAAS,CAAC,gCAAgC,IAAI,KAAK;IAMnD,SAAS,CAAC,uBAAuB,IAAI,KAAK;CAG7C"}