@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/proxy_configuration.js

@@ -0,0 +1,209 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProxyConfiguration = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const configuration_1 = require("./configuration");
+/**
+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
+ *
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
+ * be rotated by the configuration if this option is provided.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ *
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'],
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   handlePageFunction: ({ proxyInfo }) => {
+ *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+ *   }
+ * })
+ *
+ * ```
+ * @category Scaling
+ */
+class ProxyConfiguration {
+    /**
+     * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
+     * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+     * them to use the selected proxies for all connections.
+     *
+     * ```javascript
+     * const proxyConfiguration = new ProxyConfiguration({
+     *     proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
+     * });
+     *
+     * const crawler = new CheerioCrawler({
+     *   // ...
+     *   proxyConfiguration,
+     *   handlePageFunction: ({ proxyInfo }) => {
+     *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+     *   }
+     * })
+     *
+     * ```
+     * @param options
+     * @param config
+     * @param validateRequired internal flag to disable validation that requires one of `proxyUrls` or `newUrlFunction`, defaults to true
+     */
+    constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig(), validateRequired = true) {
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
+        });
+        Object.defineProperty(this, "isManInTheMiddle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "nextCustomUrlIndex", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "proxyUrls", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "usedProxyUrls", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "newUrlFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: log_1.default.child({ prefix: 'ProxyConfiguration' })
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            proxyUrls: ow_1.default.optional.array.nonEmpty.ofType(ow_1.default.string.url),
+            newUrlFunction: ow_1.default.optional.function,
+        }));
+        const { proxyUrls, newUrlFunction } = options;
+        if (proxyUrls && newUrlFunction)
+            this._throwCannotCombineCustomMethods();
+        if (!proxyUrls && !newUrlFunction && validateRequired)
+            this._throwNoOptionsProvided();
+        this.proxyUrls = proxyUrls;
+        this.newUrlFunction = newUrlFunction;
+    }
+    /**
+     * This function creates a new {@link ProxyInfo} info object.
+     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
+     * the currently used proxy via the handlePageFunction parameter: proxyInfo.
+     * Use it if you want to work with a rich representation of a proxy URL.
+     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
+     * @param [sessionId]
+     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
+     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
+     *  {@link ProxyInfo} is always returned as a type string.
+     *
+     *  All the HTTP requests going through the proxy with the same session identifier
+     *  will use the same target proxy server (i.e. the same IP address).
+     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
+     * @return Represents information about used proxy and its configuration.
+     */
+    newProxyInfo(sessionId) {
+        if (typeof sessionId === 'number')
+            sessionId = `${sessionId}`;
+        const url = this.newUrl(sessionId);
+        const { username, password, port, hostname } = new URL(url);
+        return {
+            sessionId,
+            url,
+            username,
+            password,
+            hostname,
+            port: port,
+        };
+    }
+    /**
+     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
+     * @param [sessionId]
+     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
+     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
+     *  When the provided sessionId is a number, it's converted to a string.
+     *
+     *  All the HTTP requests going through the proxy with the same session identifier
+     *  will use the same target proxy server (i.e. the same IP address).
+     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
+     * @return A string with a proxy URL, including authentication credentials and port number.
+     *  For example, `http://bob:password123@proxy.example.com:8000`
+     */
+    newUrl(sessionId) {
+        if (typeof sessionId === 'number')
+            sessionId = `${sessionId}`;
+        if (this.newUrlFunction) {
+            return this._callNewUrlFunction(sessionId);
+        }
+        return this._handleCustomUrl(sessionId);
+    }
+    /**
+     * Handles custom url rotation with session
+     */
+    _handleCustomUrl(sessionId) {
+        let customUrlToUse;
+        if (!sessionId) {
+            return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+        }
+        if (this.usedProxyUrls.has(sessionId)) {
+            customUrlToUse = this.usedProxyUrls.get(sessionId);
+        }
+        else {
+            customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+            this.usedProxyUrls.set(sessionId, customUrlToUse);
+        }
+        return customUrlToUse;
+    }
+    /**
+     * Calls the custom newUrlFunction and checks format of its return value
+     */
+    _callNewUrlFunction(sessionId) {
+        let proxyUrl;
+        try {
+            proxyUrl = this.newUrlFunction(sessionId);
+            new URL(proxyUrl); // eslint-disable-line no-new
+        }
+        catch (err) {
+            this._throwNewUrlFunctionInvalid(err);
+        }
+        return proxyUrl;
+    }
+    _throwNewUrlFunctionInvalid(err) {
+        throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
+    }
+    _throwCannotCombineCustomMethods() {
+        throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
+    }
+    _throwNoOptionsProvided() {
+        throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
+    }
+}
+exports.ProxyConfiguration = ProxyConfiguration;
+//# sourceMappingURL=proxy_configuration.js.map

package/dist/proxy_configuration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,6DAA6B;AAC7B,mDAAgD;AAsFhD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,kBAAkB;IAQ3B;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,YAAY,UAAqC,EAAE,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE,EAAE,gBAAgB,GAAG,IAAI;;;;;mBAAjE;;QA9B9D;;;;mBAAmB,KAAK;WAAC;QACzB;;;;mBAA+B,CAAC;WAAC;QACjC;;;;;WAA+B;QAC/B;;;;mBAA0B,IAAI,GAAG,EAAkB;WAAC;QACpD;;;;;WAAsD;QACtD;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;WAAC;QA0BxD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAE9C,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACzE,IAAI,CAAC,SAAS,IAAI,CAAC,cAAc,IAAI,gBAAgB;YAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAEtF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,SAA2B;QACpC,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEnC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO;YACH,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,IAAI,EAAE,IAAK;SACd,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,SAA2B;QAC9B,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAE9D,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAE,CAAC;SAC/C;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,gBAAgB,CAAC,SAAkB;QACzC,IAAI,cAAsB,CAAC;QAE3B,IAAI,CAAC,SAAS,EAAE;YACZ,OAAO,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;SAC9E;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;SACvD;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;SACrD;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,mBAAmB,CAAC,SAAkB;QAC5C,IAAI,QAA4B,CAAC;QAEjC,IAAI;YACA,QAAQ,GAAG,IAAI,CAAC,cAAe,CAAC,SAAU,CAAC,CAAC;YAC5C,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;SACnD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAY,CAAC,CAAC;SAClD;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAES,2BAA2B,CAAC,GAAU;QAC5C,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAES,gCAAgC;QACtC,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;IAES,uBAAuB;QAC7B,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IACpG,CAAC;CACJ;AArJD,gDAqJC"}

package/dist/request.d.ts

@@ -0,0 +1,174 @@
+/// <reference types="node" />
+import { BinaryLike } from 'crypto';
+import { AllowedHttpMethods } from './typedefs';
+/**
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
+ * The `Request` object also stores information about errors that occurred during processing of the request.
+ *
+ * Each `Request` instance has the `uniqueKey` property, which can be either specified
+ * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
+ * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
+ * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ *
+ * Example use:
+ *
+ * ```javascript
+ * const request = new Request({
+ *     url: 'http://www.example.com',
+ *     headers: { Accept: 'application/json' },
+ * });
+ *
+ * ...
+ *
+ * request.userData.foo = 'bar';
+ * request.pushErrorMessage(new Error('Request failed!'));
+ *
+ * ...
+ *
+ * const foo = request.userData.foo;
+ * ```
+ * @category Sources
+ */
+export declare class Request {
+    /** Request ID */
+    id?: string;
+    /** URL of the web page to crawl. */
+    url: string;
+    /**
+     * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
+     * to be included.
+     *
+     * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
+     * or may not be included, depending on their nature. This generally means that redirects,
+     * which happen immediately will most likely be included, but delayed redirects will not.
+     */
+    loadedUrl?: string;
+    /**
+     * A unique key identifying the request.
+     * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
+     */
+    uniqueKey: string;
+    /** HTTP method, e.g. `GET` or `POST`. */
+    method: AllowedHttpMethods;
+    /** HTTP request payload, e.g. for POST requests. */
+    payload?: string;
+    /** The `true` value indicates that the request will not be automatically retried on error. */
+    noRetry: boolean;
+    /** Indicates the number of times the crawling of the request has been retried on error. */
+    retryCount: number;
+    /** An array of error messages from request processing. */
+    errorMessages: string[];
+    /** Object with HTTP headers. Key is header name, value is the value. */
+    headers?: Record<string, string>;
+    /** Custom user data assigned to the request. */
+    userData?: Record<string, unknown>;
+    /**
+     * ISO datetime string that indicates the time when the request has been processed.
+     * Is `null` if the request has not been crawled yet.
+     */
+    handledAt?: string;
+    /**
+     * `Request` parameters including the URL, HTTP method and headers, and others.
+     */
+    constructor(options: RequestOptions);
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
+    protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions): string;
+    protected _hashPayload(payload: BinaryLike): string;
+}
+/**
+ * Specifies required and optional fields for constructing a {@link Request}.
+ */
+export interface RequestOptions {
+    /** URL of the web page to crawl. It must be a non-empty string. */
+    url: string;
+    /**
+     * A unique key identifying the request.
+     * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
+     *
+     * If `uniqueKey` is not provided, then it is automatically generated by normalizing the URL.
+     * For example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `uniqueKey`
+     * of `http://www.example.com/something`.
+     *
+     * The `keepUrlFragment` option determines whether URL hash fragment is included in the `uniqueKey` or not.
+     *
+     * The `useExtendedUniqueKey` options determines whether method and payload are included in the `uniqueKey`,
+     * producing a `uniqueKey` in the following format: `METHOD(payloadHash):normalizedUrl`. This is useful
+     * when requests point to the same URL, but with different methods and payloads. For example: form submits.
+     *
+     * Pass an arbitrary non-empty text value to the `uniqueKey` property
+     * to override the default behavior and specify which URLs shall be considered equal.
+     */
+    uniqueKey?: string;
+    /** @default 'GET' */
+    method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
+    /** HTTP request payload, e.g. for POST requests. */
+    payload?: string;
+    /**
+     * HTTP headers in the following format:
+     * ```
+     * {
+     *     Accept: 'text/html',
+     *     'Content-Type': 'application/json'
+     * }
+     * ```
+     */
+    headers?: Record<string, string>;
+    /**
+     * Custom user data assigned to the request. Use this to save any request related data to the
+     * request's scope, keeping them accessible on retries, failures etc.
+     */
+    userData?: Record<string, unknown>;
+    /**
+     * If `false` then the hash part of a URL is removed when computing the `uniqueKey` property.
+     * For example, this causes the `http://www.example.com#foo` and `http://www.example.com#bar` URLs
+     * to have the same `uniqueKey` of `http://www.example.com` and thus the URLs are considered equal.
+     * Note that this option only has an effect if `uniqueKey` is not set.
+     * @default false
+     */
+    keepUrlFragment?: boolean;
+    /**
+     * If `true` then the `uniqueKey` is computed not only from the URL, but also from the method and payload
+     * properties. This is useful when making requests to the same URL that are differentiated by method
+     * or payload, such as form submit navigations in browsers.
+     * @default false
+     */
+    useExtendedUniqueKey?: boolean;
+    /**
+     * The `true` value indicates that the request will not be automatically retried on error.
+     * @default false
+     */
+    noRetry?: boolean;
+    /** @internal */
+    id?: string;
+    /** @internal */
+    handledAt?: string;
+}
+export interface PushErrorMessageOptions {
+    /**
+     * Only push the error message without stack trace when true.
+     * @default false
+     */
+    omitStack?: boolean;
+}
+interface ComputeUniqueKeyOptions {
+    url: string;
+    method: AllowedHttpMethods;
+    payload?: string | Buffer;
+    keepUrlFragment?: boolean;
+    useExtendedUniqueKey?: boolean;
+}
+export {};
+//# sourceMappingURL=request.d.ts.map

package/dist/request.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";AACA,OAAe,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAI5C,OAAO,EAAE,kBAAkB,EAAQ,MAAM,YAAY,CAAC;AAqBtD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,OAAO;IAChB,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc;IA0DnC;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,SAAS,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,EAAE,uBAAuB;IAgBpH,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAQtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAE3B,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEnC;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC"}

package/dist/request.js

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Request = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const crypto_1 = tslib_1.__importDefault(require("crypto"));
+const ow_1 = tslib_1.__importStar(require("ow"));
+const util_1 = tslib_1.__importDefault(require("util"));
+const log_1 = require("./log");
+const typedefs_1 = require("./typedefs");
+// new properties on the Request object breaks serialization
+const log = log_1.log.child({ prefix: 'Request' });
+const requestOptionalPredicates = {
+    id: ow_1.default.optional.string,
+    loadedUrl: ow_1.default.optional.string.url,
+    uniqueKey: ow_1.default.optional.string,
+    method: ow_1.default.optional.string,
+    payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
+    noRetry: ow_1.default.optional.boolean,
+    retryCount: ow_1.default.optional.number,
+    errorMessages: ow_1.default.optional.array.ofType(ow_1.default.string),
+    headers: ow_1.default.optional.object,
+    userData: ow_1.default.optional.object,
+    handledAt: ow_1.default.optional.any(ow_1.default.string.date, ow_1.default.date),
+    keepUrlFragment: ow_1.default.optional.boolean,
+    useExtendedUniqueKey: ow_1.default.optional.boolean,
+};
+/**
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
+ * The `Request` object also stores information about errors that occurred during processing of the request.
+ *
+ * Each `Request` instance has the `uniqueKey` property, which can be either specified
+ * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
+ * are considered as pointing to the same web resource. This behavior applies to all Apify SDK classes,
+ * such as {@link RequestList}, {@link RequestQueue}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ *
+ * Example use:
+ *
+ * ```javascript
+ * const request = new Request({
+ *     url: 'http://www.example.com',
+ *     headers: { Accept: 'application/json' },
+ * });
+ *
+ * ...
+ *
+ * request.userData.foo = 'bar';
+ * request.pushErrorMessage(new Error('Request failed!'));
+ *
+ * ...
+ *
+ * const foo = request.userData.foo;
+ * ```
+ * @category Sources
+ */
+class Request {
+    /**
+     * `Request` parameters including the URL, HTTP method and headers, and others.
+     */
+    constructor(options) {
+        /** Request ID */
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** URL of the web page to crawl. */
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * An actually loaded URL after redirects, if present. HTTP redirects are guaranteed
+         * to be included.
+         *
+         * When using {@link PuppeteerCrawler} or {@link PlaywrightCrawler}, meta tag and JavaScript redirects may,
+         * or may not be included, depending on their nature. This generally means that redirects,
+         * which happen immediately will most likely be included, but delayed redirects will not.
+         */
+        Object.defineProperty(this, "loadedUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A unique key identifying the request.
+         * Two requests with the same `uniqueKey` are considered as pointing to the same URL.
+         */
+        Object.defineProperty(this, "uniqueKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** HTTP method, e.g. `GET` or `POST`. */
+        Object.defineProperty(this, "method", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** HTTP request payload, e.g. for POST requests. */
+        Object.defineProperty(this, "payload", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** The `true` value indicates that the request will not be automatically retried on error. */
+        Object.defineProperty(this, "noRetry", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Indicates the number of times the crawling of the request has been retried on error. */
+        Object.defineProperty(this, "retryCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** An array of error messages from request processing. */
+        Object.defineProperty(this, "errorMessages", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Object with HTTP headers. Key is header name, value is the value. */
+        Object.defineProperty(this, "headers", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Custom user data assigned to the request. */
+        Object.defineProperty(this, "userData", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * ISO datetime string that indicates the time when the request has been processed.
+         * Is `null` if the request has not been crawled yet.
+         */
+        Object.defineProperty(this, "handledAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, 'RequestOptions', ow_1.default.object);
+        (0, ow_1.default)(options.url, 'RequestOptions.url', ow_1.default.string);
+        // 'ow' validation is slow, because it checks all predicates
+        // even if the validated object has only 1 property.
+        // This custom validation loop iterates only over existing
+        // properties and speeds up the validation cca 3-fold.
+        // See https://github.com/sindresorhus/ow/issues/193
+        (0, typedefs_1.keys)(options).forEach((prop) => {
+            const predicate = requestOptionalPredicates[prop];
+            const value = options[prop];
+            if (predicate) {
+                (0, ow_1.default)(value, `RequestOptions.${prop}`, predicate);
+                // 'url' is checked above because it's not optional
+            }
+            else if (prop !== 'url') {
+                const msg = `Did not expect property \`${prop}\` to exist, got \`${value}\` in object \`RequestOptions\``;
+                throw new ow_1.ArgumentError(msg, this.constructor);
+            }
+        });
+        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, errorMessages = [], headers = {}, userData = {}, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, } = options;
+        let { method = 'GET', } = options;
+        method = method.toUpperCase();
+        if (method === 'GET' && payload)
+            throw new Error('Request with GET method cannot have a payload.');
+        this.id = id;
+        this.url = url;
+        this.loadedUrl = loadedUrl;
+        this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+        this.method = method;
+        this.payload = payload;
+        this.noRetry = noRetry;
+        this.retryCount = retryCount;
+        this.errorMessages = [...errorMessages];
+        this.headers = { ...headers };
+        this.userData = { ...userData };
+        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+    }
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage, options = {}) {
+        const { omitStack } = options;
+        let message;
+        const type = typeof errorOrMessage;
+        if (type === 'object') {
+            if (!errorOrMessage) {
+                message = 'null';
+            }
+            else if (errorOrMessage instanceof Error) {
+                message = omitStack
+                    ? errorOrMessage.message
+                    // .stack includes the message
+                    : errorOrMessage.stack;
+            }
+            else if (Reflect.has(Object(errorOrMessage), 'message')) {
+                message = Reflect.get(Object(errorOrMessage), 'message');
+            }
+            else if (errorOrMessage.toString() !== '[object Object]') {
+                message = errorOrMessage.toString();
+            }
+            else {
+                try {
+                    message = util_1.default.inspect(errorOrMessage);
+                }
+                catch (err) {
+                    message = 'Unable to extract any message from the received object.';
+                }
+            }
+        }
+        else if (type === 'undefined') {
+            message = 'undefined';
+        }
+        else {
+            message = errorOrMessage.toString();
+        }
+        this.errorMessages.push(message);
+    }
+    _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
+        const normalizedMethod = method.toUpperCase();
+        const normalizedUrl = (0, utilities_1.normalizeUrl)(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+        if (!useExtendedUniqueKey) {
+            if (normalizedMethod !== 'GET' && payload) {
+                // Using log.deprecated to log only once. We should add log.once or some such.
+                log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
+                    + 'This is fine. Just letting you know that if your requests point to the same URL '
+                    + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
+            }
+            return normalizedUrl;
+        }
+        const payloadHash = payload ? this._hashPayload(payload) : '';
+        return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+    }
+    _hashPayload(payload) {
+        return crypto_1.default
+            .createHash('sha256')
+            .update(payload)
+            .digest('base64')
+            .replace(/[+/=]/g, '')
+            .substring(0, 8);
+    }
+}
+exports.Request = Request;
+//# sourceMappingURL=request.js.map

package/dist/request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":";;;;AAAA,gDAAgD;AAChD,4DAA4C;AAC5C,iDAAsD;AACtD,wDAAwB;AACxB,+BAA0C;AAC1C,yCAAsD;AAEtD,4DAA4D;AAC5D,MAAM,GAAG,GAAG,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;IAC9C,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,CAAC,IAAI,EAAE,YAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;CAC5C,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAa,OAAO;IAkDhB;;OAEG;IACH,YAAY,OAAuB;QApDnC,iBAAiB;QACjB;;;;;WAAY;QAEZ,oCAAoC;QACpC;;;;;WAAY;QAEZ;;;;;;;WAOG;QACH;;;;;WAAmB;QAEnB;;;WAGG;QACH;;;;;WAAkB;QAElB,yCAAyC;QACzC;;;;;WAA2B;QAE3B,oDAAoD;QACpD;;;;;WAAiB;QAEjB,8FAA8F;QAC9F;;;;;WAAiB;QAEjB,2FAA2F;QAC3F;;;;;WAAmB;QAEnB,0DAA0D;QAC1D;;;;;WAAwB;QAExB,wEAAwE;QACxE;;;;;WAAiC;QAEjC,gDAAgD;QAChD;;;;;WAAmC;QAEnC;;;WAGG;QACH;;;;;WAAmB;QAMf,IAAA,YAAE,EAAC,OAAO,EAAE,gBAAgB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE;gBACX,IAAA,YAAE,EAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;gBAChE,mDAAmD;aACtD;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE;gBACvB,MAAM,GAAG,GAAG,6BAA6B,IAAI,sBAAsB,KAAK,iCAAiC,CAAC;gBAC1G,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;aAClD;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GAC/B,GAAG,OAA4H,CAAC;QAEjI,IAAI,EACA,MAAM,GAAG,KAAK,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,SAAoB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;IAC3G,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,IAAI,CAAC,cAAc,EAAE;gBACjB,OAAO,GAAG,MAAM,CAAC;aACpB;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE;gBACxC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,8BAA8B;oBAC9B,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC;aAC9B;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE;gBACvD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;aAC5D;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE;gBACpE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;aACnD;iBAAM;gBACH,IAAI;oBACA,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;iBAC1C;gBAAC,OAAO,GAAG,EAAE;oBACV,OAAO,GAAG,yDAAyD,CAAC;iBACvE;aACJ;SACJ;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE;YAC7B,OAAO,GAAG,WAAW,CAAC;SACzB;aAAM;YACH,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;SACnD;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAES,iBAAiB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAA2B;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE;YACvB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE;gBACvC,8EAA8E;gBAC9E,GAAG,CAAC,UAAU,CAAC,uBAAuB,gBAAgB,2BAA2B;sBAC3E,kFAAkF;sBAClF,iHAAiH,CAAC,CAAC;aAC5H;YACD,OAAO,aAAa,CAAC;SACxB;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAES,YAAY,CAAC,OAAmB;QACtC,OAAO,gBAAM;aACR,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,OAAO,CAAC;aACf,MAAM,CAAC,QAAQ,CAAC;aAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;aACrB,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;CACJ;AApLD,0BAoLC"}