@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/events/local_event_manager.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalEventManager = void 0;
+const tslib_1 = require("tslib");
+// eslint-disable-next-line max-classes-per-file
+const os_1 = tslib_1.__importDefault(require("os"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const utilities_1 = require("@apify/utilities");
+const utils_1 = require("@crawlee/utils");
+const event_manager_1 = require("./event_manager");
+class LocalEventManager extends event_manager_1.EventManager {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "intervals", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "previousTicks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: { idle: 0, total: 0 }
+        });
+    }
+    /**
+     * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
+     * This is an internal function that is automatically called by `Actor.main()`.
+     */
+    async init() {
+        if (this.initialized) {
+            return;
+        }
+        await super.init();
+        const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
+        this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
+        this.intervals.systemInfo = (0, utilities_1.betterSetInterval)(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
+    }
+    /**
+     * Closes websocket providing events from Actor infrastructure and also stops sending internal events
+     * of Apify package such as `persistState`.
+     * This is automatically called at the end of `Actor.main()`.
+     */
+    async close() {
+        if (!this.initialized) {
+            return;
+        }
+        await super.close();
+        (0, utilities_1.betterClearInterval)(this.intervals.systemInfo);
+    }
+    /**
+     * @internal
+     */
+    async emitSystemInfoEvent(intervalCallback) {
+        const info = await this.createSystemInfo({
+            maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
+        });
+        this.events.emit("systemInfo" /* SYSTEM_INFO */, info);
+        intervalCallback();
+    }
+    getCurrentCpuTicks() {
+        const cpus = os_1.default.cpus();
+        return cpus.reduce((acc, cpu) => {
+            const cpuTimes = Object.values(cpu.times);
+            return {
+                idle: acc.idle + cpu.times.idle,
+                total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
+            };
+        }, { idle: 0, total: 0 });
+    }
+    /**
+     * Creates a SystemInfo object based on local metrics.
+     */
+    async createSystemInfo(options) {
+        return {
+            createdAt: new Date(),
+            ...this.createCpuInfo(options),
+            ...await this.createMemoryInfo(),
+        };
+    }
+    createCpuInfo(options) {
+        const ticks = this.getCurrentCpuTicks();
+        const idleTicksDelta = ticks.idle - this.previousTicks.idle;
+        const totalTicksDelta = ticks.total - this.previousTicks.total;
+        const usedCpuRatio = totalTicksDelta ? 1 - (idleTicksDelta / totalTicksDelta) : 0;
+        Object.assign(this.previousTicks, ticks);
+        return {
+            cpuCurrentUsage: usedCpuRatio * 100,
+            isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
+        };
+    }
+    async createMemoryInfo() {
+        try {
+            const memInfo = await this._getMemoryInfo();
+            const { mainProcessBytes, childProcessesBytes } = memInfo;
+            return {
+                memCurrentBytes: mainProcessBytes + childProcessesBytes,
+            };
+        }
+        catch (err) {
+            log_1.default.exception(err, 'Memory snapshot failed.');
+            return {};
+        }
+    }
+    /**
+     * Helper method for easier mocking.
+     */
+    async _getMemoryInfo() {
+        return (0, utils_1.getMemoryInfo)();
+    }
+}
+exports.LocalEventManager = LocalEventManager;
+//# sourceMappingURL=local_event_manager.js.map

package/dist/events/local_event_manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,oDAAoB;AACpB,6DAA6B;AAC7B,gDAA4F;AAC5F,0CAA+C;AAE/C,mDAA0D;AAO1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACI;;;;mBAA+B,EAAE;WAAC;QAClC;;;;mBAAwB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IAkGlD,CAAC;IAhGG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,iCAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,YAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9B,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;SACrB,CAAC;IACpB,CAAC;IAEO,aAAa,CAAC,OAAoC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI;YACA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC5C,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;YAE1D,OAAO;gBACH,eAAe,EAAE,gBAAgB,GAAG,mBAAmB;aAC1D,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;SACb;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AApGD,8CAoGC"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+export * from './autoscaling';
+export * from './configuration';
+export * from './constants';
+export * from './crawlers';
+export * from './enqueue_links';
+export * from './events';
+export * from './log';
+export * from './proxy_configuration';
+export * from './request';
+export * from './serialization';
+export * from './session_pool';
+export * from './storages';
+export * from './validators';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./autoscaling"), exports);
+tslib_1.__exportStar(require("./configuration"), exports);
+tslib_1.__exportStar(require("./constants"), exports);
+tslib_1.__exportStar(require("./crawlers"), exports);
+tslib_1.__exportStar(require("./enqueue_links"), exports);
+tslib_1.__exportStar(require("./events"), exports);
+tslib_1.__exportStar(require("./log"), exports);
+tslib_1.__exportStar(require("./proxy_configuration"), exports);
+tslib_1.__exportStar(require("./request"), exports);
+tslib_1.__exportStar(require("./serialization"), exports);
+tslib_1.__exportStar(require("./session_pool"), exports);
+tslib_1.__exportStar(require("./storages"), exports);
+tslib_1.__exportStar(require("./validators"), exports);
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B"}

package/dist/index.mjs

@@ -0,0 +1,63 @@
+import mod from "./index.js";
+
+export default mod;
+export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
+export const AutoscaledPool = mod.AutoscaledPool;
+export const Configuration = mod.Configuration;
+export const CookieParseError = mod.CookieParseError;
+export const CrawlerExtension = mod.CrawlerExtension;
+export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
+export const Dataset = mod.Dataset;
+export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
+export const EnqueueStrategy = mod.EnqueueStrategy;
+export const EventManager = mod.EventManager;
+export const EventType = mod.EventType;
+export const KeyValueStore = mod.KeyValueStore;
+export const LocalEventManager = mod.LocalEventManager;
+export const Log = mod.Log;
+export const LogLevel = mod.LogLevel;
+export const Logger = mod.Logger;
+export const LoggerJson = mod.LoggerJson;
+export const LoggerText = mod.LoggerText;
+export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
+export const ProxyConfiguration = mod.ProxyConfiguration;
+export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
+export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
+export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
+export const Request = mod.Request;
+export const RequestList = mod.RequestList;
+export const RequestQueue = mod.RequestQueue;
+export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
+export const STATUS_CODES_BLOCKED = mod.STATUS_CODES_BLOCKED;
+export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
+export const Session = mod.Session;
+export const SessionPool = mod.SessionPool;
+export const Snapshotter = mod.Snapshotter;
+export const Statistics = mod.Statistics;
+export const StorageManager = mod.StorageManager;
+export const SystemStatus = mod.SystemStatus;
+export const checkAndSerialize = mod.checkAndSerialize;
+export const chunkBySize = mod.chunkBySize;
+export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
+export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
+export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
+export const createDeserialize = mod.createDeserialize;
+export const createRequestOptions = mod.createRequestOptions;
+export const createRequests = mod.createRequests;
+export const deserializeArray = mod.deserializeArray;
+export const diffCookies = mod.diffCookies;
+export const enqueueLinks = mod.enqueueLinks;
+export const getCookiesFromResponse = mod.getCookiesFromResponse;
+export const getRequestId = mod.getRequestId;
+export const handleRequestTimeout = mod.handleRequestTimeout;
+export const log = mod.log;
+export const maybeStringify = mod.maybeStringify;
+export const mergeCookies = mod.mergeCookies;
+export const openDataset = mod.openDataset;
+export const pushData = mod.pushData;
+export const serializeArray = mod.serializeArray;
+export const storage = mod.storage;
+export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
+export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
+export const validateGlobPattern = mod.validateGlobPattern;
+export const validators = mod.validators;

package/dist/log.d.ts

@@ -0,0 +1,3 @@
+import log, { Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText } from '@apify/log';
+export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, };
+//# sourceMappingURL=log.d.ts.map

package/dist/log.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.d.ts","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE/F,OAAO,EACH,GAAG,EACH,GAAG,EACH,aAAa,EACb,QAAQ,EACR,MAAM,EACN,UAAU,EACV,UAAU,GACb,CAAC"}

package/dist/log.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoggerText = exports.LoggerJson = exports.Logger = exports.LogLevel = exports.Log = exports.log = void 0;
+const tslib_1 = require("tslib");
+const log_1 = tslib_1.__importStar(require("@apify/log"));
+exports.log = log_1.default;
+Object.defineProperty(exports, "Log", { enumerable: true, get: function () { return log_1.Log; } });
+Object.defineProperty(exports, "LogLevel", { enumerable: true, get: function () { return log_1.LogLevel; } });
+Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return log_1.Logger; } });
+Object.defineProperty(exports, "LoggerJson", { enumerable: true, get: function () { return log_1.LoggerJson; } });
+Object.defineProperty(exports, "LoggerText", { enumerable: true, get: function () { return log_1.LoggerText; } });
+//# sourceMappingURL=log.js.map

package/dist/log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"log.js","sourceRoot":"","sources":["../src/log.ts"],"names":[],"mappings":";;;;AAAA,0DAA+F;AAG3F,cAHG,aAAG,CAGH;AACH,oFAJU,SAAG,OAIV;AAEH,yFAN8B,cAAQ,OAM9B;AACR,uFAPwC,YAAM,OAOxC;AACN,2FARgD,gBAAU,OAQhD;AACV,2FAT4D,gBAAU,OAS5D"}

package/dist/package.json

@@ -0,0 +1,82 @@
+{
+    "name": "@crawlee/core",
+    "version": "3.0.0-alpha.3",
+    "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
+    "engines": {
+        "node": ">=16.0.0"
+    },
+    "types": "index.d.ts",
+    "exports": {
+        ".": {
+            "import": "./index.mjs",
+            "require": "./index.js"
+        }
+    },
+    "keywords": [
+        "apify",
+        "headless",
+        "chrome",
+        "puppeteer",
+        "crawler",
+        "scraper"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Jan Curn <jan@apify.com>",
+        "Marek Trunkat <marek@apify.com>",
+        "Ondra Urban <ondra@apify.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/apify-js"
+    },
+    "bugs": {
+        "url": "https://github.com/apify/apify-js/issues"
+    },
+    "homepage": "https://sdk.apify.com/",
+    "files": [
+        "dist"
+    ],
+    "scripts": {
+        "build": "npm run clean && npm run compile && npm run copy",
+        "clean": "rimraf ./dist",
+        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
+        "copy": "ts-node -T ../../scripts/copy.ts"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {
+        "@apify/consts": "^1.4.2",
+        "@apify/datastructures": "^1.0.1",
+        "@apify/log": "^1.1.6",
+        "@apify/ps-tree": "^1.1.4",
+        "@apify/pseudo_url": "^1.1.0",
+        "@apify/timeout": "^0.2.1",
+        "@apify/utilities": "^1.2.5",
+        "@apify/storage-local": "^2.0.3-beta.3",
+        "@crawlee/browser-pool": "^3.0.0-alpha.3",
+        "@crawlee/storage": "^3.0.0-alpha.3",
+        "@crawlee/utils": "^3.0.0-alpha.3",
+        "@types/mime-types": "^2.1.1",
+        "@types/stream-json": "^1.7.1",
+        "content-type": "^1.0.4",
+        "got-scraping": "^3.2.7",
+        "htmlparser2": "^7.0.0",
+        "iconv-lite": "^0.6.3",
+        "jquery": "^3.6.0",
+        "mime-types": "^2.1.34",
+        "minimatch": "^5.0.1",
+        "ow": "^0.28.0",
+        "proxy-chain": "^2.0.1-beta.0",
+        "stream-json": "^1.7.3",
+        "tldts": "^5.7.69",
+        "tough-cookie": "^4.0.0",
+        "ws": "^7.5.3"
+    }
+}

package/dist/proxy_configuration.d.ts

@@ -0,0 +1,181 @@
+import { Configuration } from './configuration';
+export interface ProxyConfigurationFunction {
+    (sessionId: string | number): string;
+}
+export interface ProxyConfigurationOptions {
+    /**
+     * An array of custom proxy URLs to be rotated.
+     * Custom proxies are not compatible with Apify Proxy and an attempt to use both
+     * configuration options will cause an error to be thrown on initialize.
+     */
+    proxyUrls?: string[];
+    /**
+     * Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
+     * and should always return stringified proxy URL.
+     * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
+     */
+    newUrlFunction?: ProxyConfigurationFunction;
+}
+/**
+ * The main purpose of the ProxyInfo object is to provide information
+ * about the current proxy connection used by the crawler for the request.
+ * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // TODO fix examples and jsdoc
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   groups: ['GROUP1', 'GROUP2'] // List of Apify Proxy groups
+ *   countryCode: 'US',
+ * });
+ *
+ * // Getting proxyInfo object by calling class method directly
+ * const proxyInfo = proxyConfiguration.newProxyInfo();
+ *
+ * // In crawler
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   handlePageFunction: ({ proxyInfo }) => {
+ *      // Getting used proxy URL
+ *       const proxyUrl = proxyInfo.url;
+ *
+ *      // Getting ID of used Session
+ *       const sessionIdentifier = proxyInfo.sessionId;
+ *   }
+ * })
+ *
+ * ```
+ */
+export interface ProxyInfo {
+    /**
+     * The identifier of used {@link Session}, if used.
+     */
+    sessionId?: string;
+    /**
+     * The URL of the proxy.
+     */
+    url: string;
+    /**
+     * Username for the proxy.
+     */
+    username?: string;
+    /**
+     * User's password for the proxy.
+     */
+    password: string;
+    /**
+     * Hostname of your proxy.
+     */
+    hostname: string;
+    /**
+     * Proxy port.
+     */
+    port: number | string;
+}
+/**
+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
+ *
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
+ * be rotated by the configuration if this option is provided.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ *
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'],
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   handlePageFunction: ({ proxyInfo }) => {
+ *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+ *   }
+ * })
+ *
+ * ```
+ * @category Scaling
+ */
+export declare class ProxyConfiguration {
+    readonly config: Configuration;
+    isManInTheMiddle: boolean;
+    protected nextCustomUrlIndex: number;
+    protected proxyUrls?: string[];
+    protected usedProxyUrls: Map<string, string>;
+    protected newUrlFunction?: ProxyConfigurationFunction;
+    protected log: import("@apify/log").Log;
+    /**
+     * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
+     * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+     * them to use the selected proxies for all connections.
+     *
+     * ```javascript
+     * const proxyConfiguration = new ProxyConfiguration({
+     *     proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
+     * });
+     *
+     * const crawler = new CheerioCrawler({
+     *   // ...
+     *   proxyConfiguration,
+     *   handlePageFunction: ({ proxyInfo }) => {
+     *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+     *   }
+     * })
+     *
+     * ```
+     * @param options
+     * @param config
+     * @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
+     */
+    constructor(options?: ProxyConfigurationOptions, config?: Configuration, validateRequired?: boolean);
+    /**
+     * This function creates a new {@link ProxyInfo} info object.
+     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
+     * the currently used proxy via the handlePageFunction parameter: proxyInfo.
+     * Use it if you want to work with a rich representation of a proxy URL.
+     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
+     * @param [sessionId]
+     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
+     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
+     *  {@link ProxyInfo} is always returned as a type string.
+     *
+     *  All the HTTP requests going through the proxy with the same session identifier
+     *  will use the same target proxy server (i.e. the same IP address).
+     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
+     * @return Represents information about used proxy and its configuration.
+     */
+    newProxyInfo(sessionId?: string | number): ProxyInfo;
+    /**
+     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
+     * @param [sessionId]
+     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
+     *  When the provided sessionId is a number, it's converted to a string.
+     *
+     *  All the HTTP requests going through the proxy with the same session identifier
+     *  will use the same target proxy server (i.e. the same IP address).
+     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
+     * @return A string with a proxy URL, including authentication credentials and port number.
+     *  For example, `http://bob:password123@proxy.example.com:8000`
+     */
+    newUrl(sessionId?: string | number): string;
+    /**
+     * Handles custom url rotation with session
+     */
+    protected _handleCustomUrl(sessionId?: string): string;
+    /**
+     * Calls the custom newUrlFunction and checks format of its return value
+     */
+    protected _callNewUrlFunction(sessionId?: string): string | undefined;
+    protected _throwNewUrlFunctionInvalid(err: Error): void;
+    protected _throwCannotCombineCustomMethods(): void;
+    protected _throwNoOptionsProvided(): void;
+}
+//# sourceMappingURL=proxy_configuration.d.ts.map

package/dist/proxy_configuration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy_configuration.d.ts","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,0BAA0B;IACvC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,yBAAyB;IACtC;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB;;;;OAIG;IACH,cAAc,CAAC,EAAE,0BAA0B,CAAC;CAC/C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,kBAAkB;IA+B0B,QAAQ,CAAC,MAAM;IA9BpE,gBAAgB,UAAS;IACzB,SAAS,CAAC,kBAAkB,SAAK;IACjC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,aAAa,sBAA6B;IACpD,SAAS,CAAC,cAAc,CAAC,EAAE,0BAA0B,CAAC;IACtD,SAAS,CAAC,GAAG,2BAA+C;IAE5D;;;;;;;;;;;;;;;;;;;;;;OAsBG;gBACS,OAAO,GAAE,yBAA8B,EAAW,MAAM,gBAAkC,EAAE,gBAAgB,UAAO;IAe/H;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAgBpD;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM;IAU3C;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM;IAiBtD;;OAEG;IACH,SAAS,CAAC,mBAAmB,CAAC,SAAS,CAAC,EAAE,MAAM;IAahD,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,KAAK;IAIhD,SAAS,CAAC,gCAAgC;IAI1C,SAAS,CAAC,uBAAuB;CAGpC"}