apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/enqueue_links/enqueue_links.js

@@ -1,163 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractUrlsFromCheerio = exports.extractUrlsFromPage = exports.enqueueLinks = void 0;
-const tslib_1 = require("tslib");
-const ow_1 = (0, tslib_1.__importStar)(require("ow"));
-const url_1 = require("url");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
-/* eslint-disable import/no-duplicates */
-const shared_1 = require("./shared");
-const validators_1 = require("../validators");
-/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
-/**
- * The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)
- * either in a Puppeteer page, or in a Cheerio object (parsed HTML),
- * and enqueues the URLs in their `href` attributes to the provided {@link RequestQueue}.
- * If you're looking to find URLs in JavaScript heavy pages where links are not available
- * in `href` elements, but rather navigations are triggered in click handlers
- * see {@link puppeteer#enqueueLinksByClickingElements}.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await Apify.utils.enqueueLinks({
- *   page,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]',
- *       'https://www.example.com/purses/[.*]'
- *   ],
- * });
- * ```
- *
- * @param {object} options
- *   All `enqueueLinks()` parameters are passed
- *   via an options object with the following keys:
- * @param {PuppeteerPage|PlaywrightPage} [options.page]
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- *   Either `page` or `$` option must be provided.
- * @param {Number} [options.limit]
- *   Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.
- * @param {CheerioAPI} [options.$]
- *   [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.
- *   Either `page` or `$` option must be provided.
- * @param {RequestQueue} options.requestQueue
- *   A request queue to which the URLs will be enqueued.
- * @param {string} [options.selector='a']
- *   A CSS selector matching links to be enqueued.
- * @param {string} [options.baseUrl]
- *   A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
- *   since the relative URL resolution is done inside the browser automatically.
- * @param {Array<Object<string, *>>|Array<string>} [options.pseudoUrls]
- *   An array of {@link PseudoUrl}s matching the URLs to be enqueued,
- *   or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
- *
- *   The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.
- *   All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,
- *   which lets you specify special properties for the enqueued {@link Request} objects.
- *
- *   If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
- *   enqueues all links found on the page.
- * @param {RequestTransform} [options.transformRequestFunction]
- *   Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
- *   to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- *   when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- *   or to dynamically update or create `userData`.
- *
- *   For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
- *   when `uniqueKey` is computed.
- *
- *   **Example:**
- *   ```javascript
- *   {
- *       transformRequestFunction: (request) => {
- *           request.userData.foo = 'bar';
- *           request.keepUrlFragment = true;
- *           return request;
- *       }
- *   }
- *   ```
- * @return {Promise<Array<QueueOperationInfo>>}
- *   Promise that resolves to an array of {@link QueueOperationInfo} objects.
- * @memberOf utils
- * @name enqueueLinks
- * @function
- */
-async function enqueueLinks(options) {
-    const { page, $, requestQueue, limit, selector = 'a', baseUrl, pseudoUrls, transformRequestFunction, } = options;
-    if (!page && !$) {
-        throw new ow_1.ArgumentError('One of the parameters "options.page" or "options.$" must be provided!', enqueueLinks);
-    }
-    if (page && $) {
-        throw new ow_1.ArgumentError('Only one of the parameters "options.page" or "options.$" must be provided!', enqueueLinks);
-    }
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        page: ow_1.default.optional.object.hasKeys('goto', 'evaluate'),
-        $: ow_1.default.optional.function,
-        requestQueue: ow_1.default.object.hasKeys('fetchNextRequest', 'addRequest'),
-        limit: ow_1.default.optional.number,
-        selector: ow_1.default.optional.string,
-        baseUrl: ow_1.default.optional.string,
-        pseudoUrls: ow_1.default.any(ow_1.default.null, ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.regExp, ow_1.default.object.hasKeys('purl'), ow_1.default.object.validate(validators_1.validators.pseudoUrl)))),
-        transformRequestFunction: ow_1.default.optional.function,
-    }));
-    if (baseUrl && page)
-        utils_log_1.default.warning('The parameter options.baseUrl can only be used when parsing a Cheerio object. It will be ignored.');
-    // Construct pseudoUrls from input where necessary.
-    const pseudoUrlInstances = (0, shared_1.constructPseudoUrlInstances)(pseudoUrls || []);
-    const urls = page ? await extractUrlsFromPage(page, selector) : extractUrlsFromCheerio($, selector, baseUrl);
-    let requestOptions = (0, shared_1.createRequestOptions)(urls);
-    if (transformRequestFunction) {
-        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
-    }
-    let requests = (0, shared_1.createRequests)(requestOptions, pseudoUrlInstances);
-    if (limit)
-        requests = requests.slice(0, limit);
-    return (0, shared_1.addRequestsToQueueInBatches)(requests, requestQueue);
-}
-exports.enqueueLinks = enqueueLinks;
-/**
- * Extracts URLs from a given Puppeteer Page.
- *
- * @param {PuppeteerPage|PlaywrightPage} page
- * @param {string} selector
- * @return {Promise<Array<string>>}
- * @ignore
- */
-async function extractUrlsFromPage(page, selector) {
-    /* istanbul ignore next */
-    return page.$$eval(selector, (linkEls) => linkEls.map((link) => link.href).filter((href) => !!href));
-}
-exports.extractUrlsFromPage = extractUrlsFromPage;
-/**
- * Extracts URLs from a given Cheerio object.
- *
- * @param {CheerioAPI} $
- * @param {string} selector
- * @param {string} baseUrl
- * @return {string[]}
- * @ignore
- */
-function extractUrlsFromCheerio($, selector, baseUrl) {
-    return $(selector)
-        .map((i, el) => $(el).attr('href'))
-        .get()
-        .filter((href) => !!href)
-        .map((href) => {
-        // Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
-        const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
-        if (!isHrefAbsolute && !baseUrl) {
-            throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
-                + 'Use options.baseUrl in utils.enqueueLinks() to automatically resolve relative URLs.');
-        }
-        return baseUrl
-            ? (new url_1.URL(href, baseUrl)).href
-            : href;
-    });
-}
-exports.extractUrlsFromCheerio = extractUrlsFromCheerio;
-//# sourceMappingURL=enqueue_links.js.map

package/build/enqueue_links/enqueue_links.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.js"],"names":[],"mappings":";;;;AAAA,sDAAuC;AACvC,6BAA0B;AAC1B,0EAA+B;AAC/B,yCAAyC;AACzC,qCAA0H;AAU1H,8CAA2C;AAE3C,iFAAiF;AAEjF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACI,KAAK,UAAU,YAAY,CAAC,OAAO;IACtC,MAAM,EACF,IAAI,EACJ,CAAC,EACD,YAAY,EACZ,KAAK,EACL,QAAQ,GAAG,GAAG,EACd,OAAO,EACP,UAAU,EACV,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE;QACb,MAAM,IAAI,kBAAa,CAAC,uEAAuE,EAAE,YAAY,CAAC,CAAC;KAClH;IACD,IAAI,IAAI,IAAI,CAAC,EAAE;QACX,MAAM,IAAI,kBAAa,CAAC,4EAA4E,EAAE,YAAY,CAAC,CAAC;KACvH;IACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QACpD,CAAC,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACvB,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,IAAI,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvD,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EACzB,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,SAAS,CAAC,CAC3C,CAAC,CAAC;QACH,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;KACjD,CAAC,CAAC,CAAC;IAEJ,IAAI,OAAO,IAAI,IAAI;QAAE,mBAAG,CAAC,OAAO,CAAC,mGAAmG,CAAC,CAAC;IAEtI,mDAAmD;IACnD,MAAM,kBAAkB,GAAG,IAAA,oCAA2B,EAAC,UAAU,IAAI,EAAE,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7G,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,CAAC,CAAC;IAChD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpF;IACD,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAClE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,IAAA,oCAA2B,EAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;AAC/D,CAAC;AAhDD,oCAgDC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,mBAAmB,CAAC,IAAI,EAAE,QAAQ;IACpD,0BAA0B;IAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACzG,CAAC;AAHD,kDAGC;AAED;;;;;;;;GAQG;AACH,SAAgB,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO;IACvD,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAClC,GAAG,EAAE;SACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACxB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,+CAA+C;kBAClF,qFAAqF,CAAC,CAAC;SAChG;QACD,OAAO,OAAO;YACV,CAAC,CAAC,CAAC,IAAI,SAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI;YAC/B,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC,CAAC;AACX,CAAC;AAhBD,wDAgBC"}

package/build/enqueue_links/shared.d.ts

@@ -1,42 +0,0 @@
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function.
- * @param {Array<(string|RegExp|{ purl: string|RegExp })>} pseudoUrls
- * @return {Array<PseudoUrl>}
- * @ignore
- */
-export function constructPseudoUrlInstances(pseudoUrls: Array<(string | RegExp | {
-    purl: string | RegExp;
-})>): Array<PseudoUrl>;
-/**
- * @param {Array<(string|Object)>} requestOptions
- * @param {Array<PseudoUrl>} pseudoUrls
- * @return {Array<Request>}
- * @ignore
- */
-export function createRequests(requestOptions: Array<(string | Object)>, pseudoUrls: Array<PseudoUrl>): Array<Request>;
-/**
- * @param {Array<(string|Object)>} sources
- * @ignore
- */
-export function createRequestOptions(sources: Array<(string | Object)>): (Object | {
-    url: string;
-})[];
-/**
- * @param {Array<Request>} requests
- * @param {RequestQueue} requestQueue
- * @param {number} batchSize
- * @return {Promise<Array<QueueOperationInfo>>}
- * @ignore
- */
-export function addRequestsToQueueInBatches(requests: Array<Request>, requestQueue: RequestQueue, batchSize?: number): Promise<Array<QueueOperationInfo>>;
-/**
- * Takes an Apify {RequestOptions} object and changes it's attributes in a desired way. This user-function is used
- * {@link utilsenqueueLinks } to modify requests before enqueuing them.
- */
-export type RequestTransform = (original: RequestOptions) => RequestOptions;
-import PseudoUrl from "../pseudo_url";
-import Request from "../request";
-import { RequestQueue } from "../storages/request_queue";
-import { QueueOperationInfo } from "../storages/request_queue";
-import { RequestOptions } from "../request";
-//# sourceMappingURL=shared.d.ts.map

package/build/enqueue_links/shared.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.js"],"names":[],"mappings":"AAqBA;;;;;GAKG;AACH,wDAJW,MAAM,CAAC,MAAM,GAAC,MAAM,GAAC;IAAE,IAAI,EAAE,MAAM,GAAC,MAAM,CAAA;CAAE,CAAC,CAAC,GAC7C,MAAM,SAAS,CAAC,CAyB3B;AACD;;;;;GAKG;AACH,+CALW,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC,cACtB,MAAM,SAAS,CAAC,GACf,MAAM,OAAO,CAAC,CAkBzB;AAED;;;GAGG;AACH,8CAHW,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC;;KAqBhC;AAED;;;;;;GAMG;AACH,sDANW,MAAM,OAAO,CAAC,gBACd,YAAY,cACZ,MAAM,GACL,QAAQ,MAAM,kBAAkB,CAAC,CAAC,CAU7C;;;;;0CAMU,cAAc,KACb,cAAc"}

package/build/enqueue_links/shared.js

@@ -1,121 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.addRequestsToQueueInBatches = exports.createRequestOptions = exports.createRequests = exports.constructPseudoUrlInstances = void 0;
-const tslib_1 = require("tslib");
-const url_1 = require("url");
-const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
-const pseudo_url_1 = (0, tslib_1.__importDefault)(require("../pseudo_url"));
-const request_1 = (0, tslib_1.__importDefault)(require("../request")); // eslint-disable-line import/no-duplicates
-/* eslint-enable */
-const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
-/**
- * To enable direct use of the Actor UI `pseudoUrls` output while keeping high performance,
- * all the pseudoUrls from the output are only constructed once and kept in a cache
- * by the `enqueueLinks()` function.
- * @ignore
- */
-const enqueueLinksPseudoUrlCache = new Map();
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function.
- * @param {Array<(string|RegExp|{ purl: string|RegExp })>} pseudoUrls
- * @return {Array<PseudoUrl>}
- * @ignore
- */
-function constructPseudoUrlInstances(pseudoUrls) {
-    return pseudoUrls.map((item) => {
-        // Get pseudoUrl instance from cache.
-        let pUrl = enqueueLinksPseudoUrlCache.get(item);
-        if (pUrl)
-            return pUrl;
-        // Nothing in cache, make a new instance.
-        // If it's already a PseudoURL, just save it.
-        if (item instanceof pseudo_url_1.default)
-            pUrl = item;
-        // If it's a string or RegExp, construct a PURL from it directly.
-        else if (typeof item === 'string' || item instanceof RegExp)
-            pUrl = new pseudo_url_1.default(item);
-        // If it's an object, look for a purl property and use it and the rest to construct a PURL with a Request template.
-        else
-            pUrl = new pseudo_url_1.default(item.purl, underscore_1.default.omit(item, 'purl'));
-        // Manage cache
-        enqueueLinksPseudoUrlCache.set(item, pUrl);
-        if (enqueueLinksPseudoUrlCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
-            const key = enqueueLinksPseudoUrlCache.keys().next().value;
-            enqueueLinksPseudoUrlCache.delete(key);
-        }
-        return pUrl;
-    });
-}
-exports.constructPseudoUrlInstances = constructPseudoUrlInstances;
-/**
- * @param {Array<(string|Object)>} requestOptions
- * @param {Array<PseudoUrl>} pseudoUrls
- * @return {Array<Request>}
- * @ignore
- */
-function createRequests(requestOptions, pseudoUrls) {
-    if (!(pseudoUrls && pseudoUrls.length)) {
-        return requestOptions.map((opts) => new request_1.default(opts));
-    }
-    const requests = [];
-    requestOptions.forEach((opts) => {
-        pseudoUrls
-            .filter((purl) => purl.matches(opts.url))
-            .forEach((purl) => {
-            const request = purl.createRequest(opts);
-            requests.push(request);
-        });
-    });
-    return requests;
-}
-exports.createRequests = createRequests;
-/**
- * @param {Array<(string|Object)>} sources
- * @ignore
- */
-function createRequestOptions(sources) {
-    return sources
-        .map((src) => {
-        const reqOpts = typeof src === 'string'
-            ? { url: src }
-            : src;
-        // TODO Remove with v1, there are examples
-        // which depend on userData existing here.
-        reqOpts.userData = { ...reqOpts.userData };
-        return reqOpts;
-    })
-        .filter(({ url }) => {
-        try {
-            return new url_1.URL(url).href;
-        }
-        catch (err) {
-            return false;
-        }
-    });
-}
-exports.createRequestOptions = createRequestOptions;
-/**
- * @param {Array<Request>} requests
- * @param {RequestQueue} requestQueue
- * @param {number} batchSize
- * @return {Promise<Array<QueueOperationInfo>>}
- * @ignore
- */
-async function addRequestsToQueueInBatches(requests, requestQueue, batchSize = 5) {
-    const queueOperationInfos = [];
-    for (const request of requests) {
-        queueOperationInfos.push(requestQueue.addRequest(request));
-        if (queueOperationInfos.length % batchSize === 0)
-            await Promise.all(queueOperationInfos);
-    }
-    return Promise.all(queueOperationInfos);
-}
-exports.addRequestsToQueueInBatches = addRequestsToQueueInBatches;
-/**
- * Takes an Apify {RequestOptions} object and changes it's attributes in a desired way. This user-function is used
- * {@link utils#enqueueLinks} to modify requests before enqueuing them.
- * @callback RequestTransform
- * @param {RequestOptions} original Request options to be modified.
- * @return {RequestOptions} The modified request options to enqueue.
- */
-//# sourceMappingURL=shared.js.map

package/build/enqueue_links/shared.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.js"],"names":[],"mappings":";;;;AAAA,6BAA0B;AAC1B,yEAA2B;AAC3B,4EAAsC;AACtC,sEAAiC,CAAC,2CAA2C;AAM7E,mBAAmB;AAEnB,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,0BAA0B,GAAG,IAAI,GAAG,EAAE,CAAC;AAE7C;;;;;GAKG;AACH,SAAgB,2BAA2B,CAAC,UAAU;IAClD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,qCAAqC;QACrC,IAAI,IAAI,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAEtB,yCAAyC;QACzC,6CAA6C;QAC7C,IAAI,IAAI,YAAY,oBAAS;YAAE,IAAI,GAAG,IAAI,CAAC;QAC3C,iEAAiE;aAC5D,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,YAAY,MAAM;YAAE,IAAI,GAAG,IAAI,oBAAS,CAAC,IAAI,CAAC,CAAC;QACxF,mHAAmH;;YAC9G,IAAI,GAAG,IAAI,oBAAS,CAAC,IAAI,CAAC,IAAI,EAAE,oBAAC,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAE3D,eAAe;QACf,0BAA0B,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,IAAI,0BAA0B,CAAC,IAAI,GAAG,4BAA4B,EAAE;YAChE,MAAM,GAAG,GAAG,0BAA0B,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAC3D,0BAA0B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;SAC1C;QACD,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC,CAAC;AACP,CAAC;AAtBD,kEAsBC;AACD;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,cAAc,EAAE,UAAU;IACrD,IAAI,CAAC,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE;QACpC,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC;KAC1D;IAED,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,cAAc,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC5B,UAAU;aACL,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACxC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACd,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACpB,CAAC;AAfD,wCAeC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,OAAO;IACxC,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACT,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,QAAQ;YACnC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE;YACd,CAAC,CAAC,GAAG,CAAC;QACV,0CAA0C;QAC1C,0CAA0C;QAC1C,OAAO,CAAC,QAAQ,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3C,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI;YACA,OAAO,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;SAC5B;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,KAAK,CAAC;SAChB;IACL,CAAC,CAAC,CAAC;AACX,CAAC;AAlBD,oDAkBC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,2BAA2B,CAAC,QAAQ,EAAE,YAAY,EAAE,SAAS,GAAG,CAAC;IACnF,MAAM,mBAAmB,GAAG,EAAE,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;QAC5B,mBAAmB,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3D,IAAI,mBAAmB,CAAC,MAAM,GAAG,SAAS,KAAK,CAAC;YAAE,MAAM,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;KAC5F;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;AAC5C,CAAC;AAPD,kEAOC;AAED;;;;;;GAMG"}

package/build/errors.d.ts

@@ -1,29 +0,0 @@
-export const APIFY_CALL_ERROR_NAME: "ApifyCallError";
-/**
- * The class represents exceptions thrown
- * by the {@link Apify#call} function.
- *
- * @property {string} message
- *   Error message
- * @property {ActorRun} run
- *   Object representing the failed actor run.
- * @property {string} name
- *   Contains `"ApifyCallError"`
- */
-export class ApifyCallError extends Error {
-    /**
-     * @param {ActorRun} run
-     * @param {string} [message]
-     */
-    constructor(run: ActorRun, message?: string | undefined);
-    run: ActorRun;
-}
-/**
- * TimeoutError class.
- * This error should be thrown after request timeout from `requestAsBrowser`.
- * @ignore
- */
-export class TimeoutError extends Error {
-}
-import { ActorRun } from "./typedefs";
-//# sourceMappingURL=errors.d.ts.map

package/build/errors.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.js"],"names":[],"mappings":"AACA,qDAAsD;AAKtD;;;;;;;;;;GAUG;AACH;IACI;;;OAGG;IACH,iBAHW,QAAQ,gCASlB;IAHG,cAAc;CAIrB;AAED;;;;GAIG;AACH;CAA0C"}

package/build/errors.js

@@ -1,38 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TimeoutError = exports.ApifyCallError = exports.APIFY_CALL_ERROR_NAME = void 0;
-/* eslint-disable max-classes-per-file */
-exports.APIFY_CALL_ERROR_NAME = 'ApifyCallError';
-/**
- * The class represents exceptions thrown
- * by the {@link Apify#call} function.
- *
- * @property {string} message
- *   Error message
- * @property {ActorRun} run
- *   Object representing the failed actor run.
- * @property {string} name
- *   Contains `"ApifyCallError"`
- */
-class ApifyCallError extends Error {
-    /**
-     * @param {ActorRun} run
-     * @param {string} [message]
-     */
-    constructor(run, message = 'The actor invoked by Apify.call() did not succeed') {
-        super(`${message} (run ID: ${run.id})`);
-        this.name = exports.APIFY_CALL_ERROR_NAME;
-        this.run = run;
-        Error.captureStackTrace(this, ApifyCallError);
-    }
-}
-exports.ApifyCallError = ApifyCallError;
-/**
- * TimeoutError class.
- * This error should be thrown after request timeout from `requestAsBrowser`.
- * @ignore
- */
-class TimeoutError extends Error {
-}
-exports.TimeoutError = TimeoutError;
-//# sourceMappingURL=errors.js.map

package/build/errors.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.js"],"names":[],"mappings":";;;AAAA,yCAAyC;AAC5B,QAAA,qBAAqB,GAAG,gBAAgB,CAAC;AAKtD;;;;;;;;;;GAUG;AACH,MAAa,cAAe,SAAQ,KAAK;IACrC;;;OAGG;IACH,YAAY,GAAG,EAAE,OAAO,GAAG,mDAAmD;QAC1E,KAAK,CAAC,GAAG,OAAO,aAAa,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,GAAG,6BAAqB,CAAC;QAClC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QAEf,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAClD,CAAC;CACJ;AAZD,wCAYC;AAED;;;;GAIG;AACH,MAAa,YAAa,SAAQ,KAAK;CAAG;AAA1C,oCAA0C"}

package/build/events.d.ts

@@ -1,11 +0,0 @@
-export default events;
-export function initializeEvents(config?: Configuration | undefined): void;
-export function stopEvents(): void;
-/**
- * Event emitter providing events from underlying Actor infrastructure and Apify package.
- * @ignore
- */
-declare const events: EventEmitter;
-import { Configuration } from "./configuration";
-import { EventEmitter } from "events";
-//# sourceMappingURL=events.d.ts.map

package/build/events.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.js"],"names":[],"mappings":";AAyFO,2EA6CN;AAYM,mCAIN;AA/ID;;;GAGG;AACH,mCAAkC"}

package/build/events.js

@@ -1,147 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.stopEvents = exports.initializeEvents = void 0;
-const tslib_1 = require("tslib");
-const events_1 = require("events");
-const ws_1 = (0, tslib_1.__importDefault)(require("ws"));
-const consts_1 = require("@apify/consts");
-const constants_1 = require("./constants");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-const configuration_1 = require("./configuration");
-/**
- * Event emitter providing events from underlying Actor infrastructure and Apify package.
- * @ignore
- */
-const events = new events_1.EventEmitter();
-/**
- * Websocket connection to actor events.
- * @type {*}
- * @ignore
- */
-let eventsWs = null;
-/**
- * Interval that emits persist state events.
- * @type {*}
- * @ignore
- */
-let persistStateInterval = null;
-/**
- * Gets an instance of a Node.js'
- * [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)
- * class that emits various events from the SDK or the Apify platform.
- * The event emitter is initialized by calling the {@link Apify#main} function.
- *
- * **Example usage:**
- *
- * ```javascript
- * Apify.events.on('cpuInfo', (data) => {
- *   if (data.isCpuOverloaded) console.log('Oh no, the CPU is overloaded!');
- * });
- * ```
- *
- * The following events are emitted:
- *
- * - `cpuInfo`: `{ "isCpuOverloaded": Boolean }`
- *   The event is emitted approximately every second
- *   and it indicates whether the actor is using the maximum of available CPU resources.
- *   If that's the case, the actor should not add more workload.
- *   For example, this event is used by the {@link AutoscaledPool} class.
- * - `migrating`: `void`
- *   Emitted when the actor running on the Apify platform is going to be migrated to another worker server soon.
- *   You can use it to persist the state of the actor and abort the run, to speed up migration.
- *   For example, this is used by the {@link RequestList} class.
- * - `aborting`: `void`
- *   When a user aborts an actor run on the Apify platform, they can choose to abort gracefully to allow
- *   the actor some time before getting killed. This graceful abort emits the `aborting` event which the SDK
- *   uses to gracefully stop running crawls and you can use it to do your own cleanup as well.
- * - `persistState`: `{ "isMigrating": Boolean }`
- *   Emitted in regular intervals (by default 60 seconds) to notify all components of Apify SDK that it is time to persist
- *   their state, in order to avoid repeating all work when the actor restarts.
- *   This event is automatically emitted together with the `migrating` event,
- *   in which case the `isMigrating` flag is set to `true`. Otherwise the flag is `false`.
- *   Note that the `persistState` event is provided merely for user convenience,
- *   you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
- *
- * @memberof module:Apify
- * @name events
- */
-exports.default = events;
-/**
- * Emits event telling all components that they should persist their state at regular intervals and also when an actor is being
- * migrated to another worker.
- *
- * @ignore
- */
-const emitPersistStateEvent = (isMigrating = false) => {
-    events.emit(constants_1.ACTOR_EVENT_NAMES_EX.PERSIST_STATE, { isMigrating });
-};
-/**
- * Initializes `Apify.events` event emitter by creating a connection to a websocket that provides them.
- * This is an internal function that is automatically called by `Apify.main()`.
- *
- * @memberof module:Apify
- * @name initializeEvents
- * @param {Configuration} [config]
- * @function
- * @ignore
- */
-const initializeEvents = (config = configuration_1.Configuration.getGlobalConfig()) => {
-    if (eventsWs)
-        return;
-    const log = utils_log_1.default.child({ prefix: 'Events' });
-    if (!persistStateInterval) {
-        const intervalMillis = config.get('persistStateIntervalMillis');
-        persistStateInterval = setInterval(() => emitPersistStateEvent(), intervalMillis);
-    }
-    const eventsWsUrl = config.get('actorEventsWsUrl');
-    // Locally there is no web socket to connect, so just print a log message.
-    if (!eventsWsUrl) {
-        log.debug(`Environment variable ${consts_1.ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
-        return;
-    }
-    eventsWs = new ws_1.default(eventsWsUrl);
-    eventsWs.on('message', (message) => {
-        if (!message)
-            return;
-        try {
-            const { name, data } = JSON.parse(message);
-            events.emit(name, data);
-            if (name === consts_1.ACTOR_EVENT_NAMES.MIGRATING) {
-                clearInterval(persistStateInterval); // Don't send any other persist state event.
-                emitPersistStateEvent(true);
-            }
-        }
-        catch (err) {
-            log.exception(err, 'Cannot parse actor event');
-        }
-    });
-    eventsWs.on('error', (err) => {
-        // Don't print this error as this happens in the case of very short Apify.main().
-        if (err.message === 'WebSocket was closed before the connection was established')
-            return;
-        log.exception(err, 'web socket connection failed');
-    });
-    eventsWs.on('close', () => {
-        log.warning('web socket has been closed');
-        eventsWs = null;
-    });
-};
-exports.initializeEvents = initializeEvents;
-/**
- * Closes websocket providing events from Actor infrastructure and also stops sending internal events
- * of Apify package such as `persistState`.
- * This is automatically called at the end of `Apify.main()`.
- *
- * @memberof module:Apify
- * @name stopEvents
- * @function
- * @ignore
- */
-const stopEvents = () => {
-    if (eventsWs)
-        eventsWs.close();
-    clearInterval(persistStateInterval);
-    persistStateInterval = null;
-};
-exports.stopEvents = stopEvents;
-//# sourceMappingURL=events.js.map

package/build/events.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"events.js","sourceRoot":"","sources":["../src/events.js"],"names":[],"mappings":";;;;AAAA,mCAAsC;AACtC,yDAA2B;AAC3B,0CAA4D;AAC5D,2CAAmD;AACnD,yEAAqC;AACrC,mDAAgD;AAEhD;;;GAGG;AACH,MAAM,MAAM,GAAG,IAAI,qBAAY,EAAE,CAAC;AAElC;;;;GAIG;AACH,IAAI,QAAQ,GAAG,IAAI,CAAC;AAEpB;;;;GAIG;AACH,IAAI,oBAAoB,GAAG,IAAI,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,kBAAe,MAAM,CAAC;AAEtB;;;;;GAKG;AACH,MAAM,qBAAqB,GAAG,CAAC,WAAW,GAAG,KAAK,EAAE,EAAE;IAClD,MAAM,CAAC,IAAI,CAAC,gCAAoB,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;AACrE,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACI,MAAM,gBAAgB,GAAG,CAAC,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EAAE,EAAE;IACzE,IAAI,QAAQ;QAAE,OAAO;IAErB,MAAM,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEnD,IAAI,CAAC,oBAAoB,EAAE;QACvB,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAChE,oBAAoB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,qBAAqB,EAAE,EAAE,cAAc,CAAC,CAAC;KACrF;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAEnD,0EAA0E;IAC1E,IAAI,CAAC,WAAW,EAAE;QACd,GAAG,CAAC,KAAK,CAAC,wBAAwB,iBAAQ,CAAC,mBAAmB,6DAA6D,CAAC,CAAC;QAC7H,OAAO;KACV;IAED,QAAQ,GAAG,IAAI,YAAS,CAAC,WAAW,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;QAC/B,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,IAAI;YACA,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAE3C,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAExB,IAAI,IAAI,KAAK,0BAAiB,CAAC,SAAS,EAAE;gBACtC,aAAa,CAAC,oBAAoB,CAAC,CAAC,CAAC,4CAA4C;gBACjF,qBAAqB,CAAC,IAAI,CAAC,CAAC;aAC/B;SACJ;QAAC,OAAO,GAAG,EAAE;YACV,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC;SAClD;IACL,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACzB,iFAAiF;QACjF,IAAI,GAAG,CAAC,OAAO,KAAK,4DAA4D;YAAE,OAAO;QAEzF,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,8BAA8B,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACtB,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;QAC1C,QAAQ,GAAG,IAAI,CAAC;IACpB,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AA7CW,QAAA,gBAAgB,oBA6C3B;AAEF;;;;;;;;;GASG;AACI,MAAM,UAAU,GAAG,GAAG,EAAE;IAC3B,IAAI,QAAQ;QAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC/B,aAAa,CAAC,oBAAoB,CAAC,CAAC;IACpC,oBAAoB,GAAG,IAAI,CAAC;AAChC,CAAC,CAAC;AAJW,QAAA,UAAU,cAIrB"}

package/build/index.d.ts

@@ -1,4 +0,0 @@
-export default Apify;
-export * from "./main";
-import * as Apify from "./main";
-//# sourceMappingURL=index.d.ts.map

package/build/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.js"],"names":[],"mappings":""}

package/build/index.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-const Apify = (0, tslib_1.__importStar)(require("./main"));
-exports.default = Apify;
-(0, tslib_1.__exportStar)(require("./main"), exports);
-//# sourceMappingURL=index.js.map

package/build/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.js"],"names":[],"mappings":";;;AAAA,2DAAgC;AAEhC,kBAAe,KAAK,CAAC;AACrB,sDAAuB"}