apify 2.0.8-beta.6 → 2.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/{types → build}/actor.d.ts +1 -0
- package/build/actor.d.ts.map +1 -0
- package/build/actor.js +13 -12
- package/build/actor.js.map +1 -0
- package/{types → build}/apify.d.ts +3 -0
- package/build/apify.d.ts.map +1 -0
- package/build/apify.js +4 -2
- package/build/apify.js.map +1 -0
- package/{types → build}/autoscaling/autoscaled_pool.d.ts +1 -0
- package/build/autoscaling/autoscaled_pool.d.ts.map +1 -0
- package/build/autoscaling/autoscaled_pool.js +6 -4
- package/build/autoscaling/autoscaled_pool.js.map +1 -0
- package/{types → build}/autoscaling/snapshotter.d.ts +1 -0
- package/build/autoscaling/snapshotter.d.ts.map +1 -0
- package/build/autoscaling/snapshotter.js +7 -5
- package/build/autoscaling/snapshotter.js.map +1 -0
- package/{types → build}/autoscaling/system_status.d.ts +1 -0
- package/build/autoscaling/system_status.d.ts.map +1 -0
- package/build/autoscaling/system_status.js +4 -2
- package/build/autoscaling/system_status.js.map +1 -0
- package/{types → build}/browser_launchers/browser_launcher.d.ts +5 -0
- package/build/browser_launchers/browser_launcher.d.ts.map +1 -0
- package/build/browser_launchers/browser_launcher.js +15 -8
- package/build/browser_launchers/browser_launcher.js.map +1 -0
- package/{types → build}/browser_launchers/browser_plugin.d.ts +1 -0
- package/build/browser_launchers/browser_plugin.d.ts.map +1 -0
- package/build/browser_launchers/browser_plugin.js +1 -0
- package/build/browser_launchers/browser_plugin.js.map +1 -0
- package/{types → build}/browser_launchers/playwright_launcher.d.ts +6 -0
- package/build/browser_launchers/playwright_launcher.d.ts.map +1 -0
- package/build/browser_launchers/playwright_launcher.js +13 -6
- package/build/browser_launchers/playwright_launcher.js.map +1 -0
- package/{types → build}/browser_launchers/puppeteer_launcher.d.ts +1 -0
- package/build/browser_launchers/puppeteer_launcher.d.ts.map +1 -0
- package/build/browser_launchers/puppeteer_launcher.js +17 -10
- package/build/browser_launchers/puppeteer_launcher.js.map +1 -0
- package/{types → build}/cache_container.d.ts +1 -0
- package/build/cache_container.d.ts.map +1 -0
- package/build/cache_container.js +1 -0
- package/build/cache_container.js.map +1 -0
- package/{types → build}/configuration.d.ts +1 -0
- package/build/configuration.d.ts.map +1 -0
- package/build/configuration.js +88 -65
- package/build/configuration.js.map +1 -0
- package/{types → build}/constants.d.ts +1 -0
- package/build/constants.d.ts.map +1 -0
- package/build/constants.js +1 -0
- package/build/constants.js.map +1 -0
- package/{types → build}/crawlers/basic_crawler.d.ts +1 -0
- package/build/crawlers/basic_crawler.d.ts.map +1 -0
- package/build/crawlers/basic_crawler.js +35 -28
- package/build/crawlers/basic_crawler.js.map +1 -0
- package/{types → build}/crawlers/browser_crawler.d.ts +1 -0
- package/build/crawlers/browser_crawler.d.ts.map +1 -0
- package/build/crawlers/browser_crawler.js +26 -19
- package/build/crawlers/browser_crawler.js.map +1 -0
- package/{types → build}/crawlers/cheerio_crawler.d.ts +1 -0
- package/build/crawlers/cheerio_crawler.d.ts.map +1 -0
- package/build/crawlers/cheerio_crawler.js +41 -34
- package/build/crawlers/cheerio_crawler.js.map +1 -0
- package/{types → build}/crawlers/crawler_extension.d.ts +1 -0
- package/build/crawlers/crawler_extension.d.ts.map +1 -0
- package/build/crawlers/crawler_extension.js +3 -1
- package/build/crawlers/crawler_extension.js.map +1 -0
- package/{types → build}/crawlers/crawler_utils.d.ts +1 -0
- package/build/crawlers/crawler_utils.d.ts.map +1 -0
- package/build/crawlers/crawler_utils.js +1 -0
- package/build/crawlers/crawler_utils.js.map +1 -0
- package/{types → build}/crawlers/playwright_crawler.d.ts +2 -0
- package/build/crawlers/playwright_crawler.d.ts.map +1 -0
- package/build/crawlers/playwright_crawler.js +15 -8
- package/build/crawlers/playwright_crawler.js.map +1 -0
- package/{types → build}/crawlers/puppeteer_crawler.d.ts +1 -0
- package/build/crawlers/puppeteer_crawler.d.ts.map +1 -0
- package/build/crawlers/puppeteer_crawler.js +15 -8
- package/build/crawlers/puppeteer_crawler.js.map +1 -0
- package/{types → build}/crawlers/statistics.d.ts +1 -0
- package/build/crawlers/statistics.d.ts.map +1 -0
- package/build/crawlers/statistics.js +5 -3
- package/build/crawlers/statistics.js.map +1 -0
- package/{types → build}/enqueue_links/click_elements.d.ts +1 -0
- package/build/enqueue_links/click_elements.d.ts.map +1 -0
- package/build/enqueue_links/click_elements.js +4 -2
- package/build/enqueue_links/click_elements.js.map +1 -0
- package/{types → build}/enqueue_links/enqueue_links.d.ts +2 -0
- package/build/enqueue_links/enqueue_links.d.ts.map +1 -0
- package/build/enqueue_links/enqueue_links.js +4 -2
- package/build/enqueue_links/enqueue_links.js.map +1 -0
- package/{types → build}/enqueue_links/shared.d.ts +1 -0
- package/build/enqueue_links/shared.d.ts.map +1 -0
- package/build/enqueue_links/shared.js +6 -4
- package/build/enqueue_links/shared.js.map +1 -0
- package/{types → build}/errors.d.ts +1 -0
- package/build/errors.d.ts.map +1 -0
- package/build/errors.js +1 -0
- package/build/errors.js.map +1 -0
- package/{types → build}/events.d.ts +1 -0
- package/build/events.d.ts.map +1 -0
- package/build/events.js +5 -3
- package/build/events.js.map +1 -0
- package/{types → build}/index.d.ts +1 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +4 -12
- package/build/index.js.map +1 -0
- package/{types → build}/main.d.ts +3 -1
- package/build/main.d.ts.map +1 -0
- package/build/main.js +11 -9
- package/build/main.js.map +1 -0
- package/{types → build}/playwright_utils.d.ts +3 -3
- package/build/playwright_utils.d.ts.map +1 -0
- package/build/playwright_utils.js +25 -18
- package/build/playwright_utils.js.map +1 -0
- package/{types → build}/proxy_configuration.d.ts +1 -0
- package/build/proxy_configuration.d.ts.map +1 -0
- package/build/proxy_configuration.js +8 -2
- package/build/proxy_configuration.js.map +1 -0
- package/{types → build}/pseudo_url.d.ts +1 -0
- package/build/pseudo_url.d.ts.map +1 -0
- package/build/pseudo_url.js +7 -5
- package/build/pseudo_url.js.map +1 -0
- package/{types → build}/puppeteer_request_interception.d.ts +1 -0
- package/build/puppeteer_request_interception.d.ts.map +1 -0
- package/build/puppeteer_request_interception.js +15 -7
- package/build/puppeteer_request_interception.js.map +1 -0
- package/{types → build}/puppeteer_utils.d.ts +1 -0
- package/build/puppeteer_utils.d.ts.map +1 -0
- package/build/puppeteer_utils.js +23 -19
- package/build/puppeteer_utils.js.map +1 -0
- package/{types → build}/request.d.ts +1 -0
- package/build/request.d.ts.map +1 -0
- package/build/request.js +8 -6
- package/build/request.js.map +1 -0
- package/{types → build}/request_list.d.ts +17 -3
- package/build/request_list.d.ts.map +1 -0
- package/build/request_list.js +36 -16
- package/build/request_list.js.map +1 -0
- package/{types → build}/serialization.d.ts +1 -0
- package/build/serialization.d.ts.map +1 -0
- package/build/serialization.js +14 -12
- package/build/serialization.js.map +1 -0
- package/{types → build}/session_pool/errors.d.ts +1 -0
- package/build/session_pool/errors.d.ts.map +1 -0
- package/build/session_pool/errors.js +1 -0
- package/build/session_pool/errors.js.map +1 -0
- package/{types → build}/session_pool/events.d.ts +1 -0
- package/build/session_pool/events.d.ts.map +1 -0
- package/build/session_pool/events.js +1 -0
- package/build/session_pool/events.js.map +1 -0
- package/{types → build}/session_pool/session.d.ts +1 -0
- package/build/session_pool/session.d.ts.map +1 -0
- package/build/session_pool/session.js +5 -3
- package/build/session_pool/session.js.map +1 -0
- package/{types → build}/session_pool/session_pool.d.ts +1 -0
- package/build/session_pool/session_pool.d.ts.map +1 -0
- package/build/session_pool/session_pool.js +5 -3
- package/build/session_pool/session_pool.js.map +1 -0
- package/{types → build}/session_pool/session_utils.d.ts +1 -0
- package/build/session_pool/session_utils.d.ts.map +1 -0
- package/build/session_pool/session_utils.js +1 -0
- package/build/session_pool/session_utils.js.map +1 -0
- package/{types → build}/stealth/hiding_tricks.d.ts +1 -0
- package/build/stealth/hiding_tricks.d.ts.map +1 -0
- package/build/stealth/hiding_tricks.js +1 -0
- package/build/stealth/hiding_tricks.js.map +1 -0
- package/{types → build}/stealth/stealth.d.ts +1 -0
- package/build/stealth/stealth.d.ts.map +1 -0
- package/build/stealth/stealth.js +6 -4
- package/build/stealth/stealth.js.map +1 -0
- package/{types → build}/storages/dataset.d.ts +1 -0
- package/build/storages/dataset.d.ts.map +1 -0
- package/build/storages/dataset.js +7 -5
- package/build/storages/dataset.js.map +1 -0
- package/{types → build}/storages/key_value_store.d.ts +1 -0
- package/build/storages/key_value_store.d.ts.map +1 -0
- package/build/storages/key_value_store.js +4 -2
- package/build/storages/key_value_store.js.map +1 -0
- package/{types → build}/storages/request_queue.d.ts +1 -0
- package/build/storages/request_queue.d.ts.map +1 -0
- package/build/storages/request_queue.js +7 -5
- package/build/storages/request_queue.js.map +1 -0
- package/{types → build}/storages/storage_manager.d.ts +1 -0
- package/build/storages/storage_manager.d.ts.map +1 -0
- package/build/storages/storage_manager.js +3 -1
- package/build/storages/storage_manager.js.map +1 -0
- package/build/tsconfig.tsbuildinfo +1 -0
- package/{types → build}/typedefs.d.ts +1 -0
- package/build/typedefs.d.ts.map +1 -0
- package/build/typedefs.js +1 -0
- package/build/typedefs.js.map +1 -0
- package/{types → build}/utils.d.ts +3 -0
- package/build/utils.d.ts.map +1 -0
- package/build/utils.js +104 -46
- package/build/utils.js.map +1 -0
- package/{types → build}/utils_log.d.ts +1 -0
- package/build/utils_log.d.ts.map +1 -0
- package/build/utils_log.js +3 -1
- package/build/utils_log.js.map +1 -0
- package/{types → build}/utils_request.d.ts +1 -0
- package/build/utils_request.d.ts.map +1 -0
- package/build/utils_request.js +5 -3
- package/build/utils_request.js.map +1 -0
- package/{types → build}/utils_social.d.ts +1 -0
- package/build/utils_social.d.ts.map +1 -0
- package/build/utils_social.js +23 -11
- package/build/utils_social.js.map +1 -0
- package/{types → build}/validators.d.ts +1 -0
- package/build/validators.d.ts.map +1 -0
- package/build/validators.js +3 -1
- package/build/validators.js.map +1 -0
- package/package.json +26 -29
- package/types/playwright/index.d.ts +0 -22
- package/types/playwright/types/protocol.d.ts +0 -17846
- package/types/playwright/types/structs.d.ts +0 -50
- package/types/playwright/types/types.d.ts +0 -13078
- package/types-apify/apify-http-request.d.ts +0 -8
- package/types-apify/ps-tree.d.ts +0 -4
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"puppeteer_launcher.js","sourceRoot":"","sources":["../../src/browser_launchers/puppeteer_launcher.js"],"names":[],"mappings":";;;;AAAA,yDAAoB;AAEpB,+CAA+C;AAC/C,uFAAiD;AACjD,oCAAoC;AACpC,4CAAkD;AAElD,8EAA2E,CAAC,kDAAkD;AAE9H,MAAM,iCAAiC,GAAG;IACtC,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AAEH;;;GAGG;AACH,MAAa,iBAAkB,SAAQ,0BAAe;IASlD;;;MAGE;IACF,YAAY,aAAa,GAAG,EAAE;QAC1B,IAAA,YAAE,EAAC,aAAa,EAAE,mBAAmB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE7F,MAAM,EACF,QAAQ,GAAG,0BAAe,CAAC,sBAAsB,CAAC,WAAW,EAAE,mCAAmC,CAAC,EACnG,SAAS,EACT,OAAO,GAAG,KAAK,EACf,cAAc,GAAG,EAAE,EACnB,GAAG,sBAAsB,EAC5B,GAAG,aAAa,CAAC;QAElB,KAAK,CAAC;YACF,GAAG,sBAAsB;YACzB,QAAQ;SACX,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,6BAA6B;QAC7B,IAAI,CAAC,cAAc,GAAG;YAClB,aAAa,EAAE,IAAI;YACnB,GAAG,cAAc;SACpB,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,8BAAe,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,MAAM;QACR,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;QAErC,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC;YAEpE,IAAA,iBAAqB,EAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;SACrD;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,mBAAmB;QACf,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;QAClD,aAAa,CAAC,IAAI,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC;QAE9C,IAAI,IAAA,gBAAQ,GAAE;YAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAExD,IAAI,aAAa,CAAC,eAAe,KAAK,SAAS,EAAE;YAC7C,aAAa,CAAC,eAAe,GAAG,iCAAiC,CAAC;SACrE;QAED,wEAAwE;QACxE,uEAAuE;QACvE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,aAAa,CAAC,cAAc,IAAI,aAAa,CAAC,QAAQ,CAAC,EAAE;YACzE,SAAS,GAAG,8BAAkB,CAAC;SAClC;QAED,IAAI,SAAS,EAAE;YACX,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE;YAC1D,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAC/F,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE;gBACZ,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,uBAAuB,CAAC;aAC3D;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;aAC5E;SACJ;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;;AAnFL,8CAoFC;AAnFU;;;;WAAe;QAClB,GAAG,0BAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACrC;GAAA;AA+EL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACI,MAAM,eAAe,GAAG,KAAK,EAAE,aAAa,EAAE,EAAE;IACnD,MAAM,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,aAAa,CAAC,CAAC;IAE/D,OAAO,iBAAiB,CAAC,MAAM,EAAE,CAAC;AACtC,CAAC,CAAC;AAJW,QAAA,eAAe,mBAI1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache_container.d.ts","sourceRoot":"","sources":["../src/cache_container.js"],"names":[],"mappings":";;AAEA;;;;;;;GAOG;AACH;IAEQ,sBAAuB;IAG3B;;;;OAIG;IACH,gBAJW,MAAM,WACN,MAAM,GACL,QAAQ,CASnB;IAED;;;OAGG;IACH,eAHW,MAAM,GACJ,QAAQ,QAIpB;IAED;;OAEG;IACH,iBAFW,MAAM,QAKhB;IAED,uBAEC;CACJ"}
|
package/build/cache_container.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache_container.js","sourceRoot":"","sources":["../src/cache_container.js"],"names":[],"mappings":";;AAAA,0DAAiD;AAEjD;;;;;;;GAOG;AACH,MAAM,cAAc;IAChB;QACI,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO;QACnB,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE;YACR,KAAK,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAChC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,IAAI;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,IAAI;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,KAAK,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,cAAc;QACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAClD,CAAC;CACJ;AAED,kBAAe,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.js"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH;IAEI;;;;;;;;;;;;;;;;;;;;;;;;MA8BE;IAGF,4BAGO;IAEP,8BAAoD;IAEpD,8BAAqE;IAErE;;;;;;;;;;;;;;MAcE;IAmLF;;;;;OAKG;IACH,0BAFY,aAAa,CAQxB;IA7LD;;;;OAIG;IACH,sBAFW,OAAO,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,EAYnD;IATG,gDAA+C;IAC/C,wBAAyB;IAU7B;;;;;;;;OAQG;IACH,SAJW,MAAM,yDAEL,MAAM,GAAG,MAAM,GAAG,OAAO,CAkBpC;IAED;;;;;OAKG;IACH,sBAWC;IAED;;;;;;OAMG;IACH,SAHW,MAAM,uDAKhB;IAED;;;;;;;;;;;;;;OAcG;IACH;;;;wBAMC;IAED;;;;;;;;;;;;;OAaG;IACH;;;oBAHY,iBAAiB,CAM5B;IAED;;;;;;;;;OASG;IACH,oBAQC;IAED;;;;;;;;;OASG;IACH;;;;wBAMC;IAED;;;;;;;;OAQG;IACH;;;oBAHY,iBAAiB,CAe5B;CAeJ"}
|
package/build/configuration.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Configuration = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
4
5
|
const consts_1 = require("@apify/consts");
|
|
5
6
|
const path_1 = require("path");
|
|
6
7
|
const storage_local_1 = require("@apify/storage-local");
|
|
7
|
-
const
|
|
8
|
-
const utils_log_1 = require("./utils_log");
|
|
8
|
+
const apify_client_1 = (0, tslib_1.__importDefault)(require("apify-client"));
|
|
9
|
+
const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
|
|
9
10
|
/**
|
|
10
11
|
* `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
|
|
11
12
|
*
|
|
@@ -86,9 +87,8 @@ class Configuration {
|
|
|
86
87
|
* @return {string | number | boolean}
|
|
87
88
|
*/
|
|
88
89
|
get(key, defaultValue) {
|
|
89
|
-
var _a, _b;
|
|
90
90
|
// prefer env vars
|
|
91
|
-
const envKey =
|
|
91
|
+
const envKey = Configuration.ENV_MAP_REVERSED[key] ?? key;
|
|
92
92
|
const envValue = process.env[envKey];
|
|
93
93
|
if (envValue != null) {
|
|
94
94
|
return this._castEnvValue(key, envValue);
|
|
@@ -98,7 +98,7 @@ class Configuration {
|
|
|
98
98
|
return this.options.get(key);
|
|
99
99
|
}
|
|
100
100
|
// fallback to defaults
|
|
101
|
-
return
|
|
101
|
+
return defaultValue ?? Configuration.DEFAULTS[key] ?? envValue;
|
|
102
102
|
}
|
|
103
103
|
/**
|
|
104
104
|
* @param {string} key
|
|
@@ -142,9 +142,8 @@ class Configuration {
|
|
|
142
142
|
* @internal
|
|
143
143
|
*/
|
|
144
144
|
getClient(options = {}) {
|
|
145
|
-
|
|
146
|
-
const
|
|
147
|
-
const token = (_b = options.token) !== null && _b !== void 0 ? _b : this.get('token');
|
|
145
|
+
const baseUrl = options.baseUrl ?? this.get('apiBaseUrl');
|
|
146
|
+
const token = options.token ?? this.get('token');
|
|
148
147
|
const cacheKey = `${baseUrl}~${token}`;
|
|
149
148
|
return this._getService('ApifyClient', () => this.createClient(options), cacheKey);
|
|
150
149
|
}
|
|
@@ -163,8 +162,7 @@ class Configuration {
|
|
|
163
162
|
* @internal
|
|
164
163
|
*/
|
|
165
164
|
getStorageLocal(options = {}) {
|
|
166
|
-
|
|
167
|
-
const cacheKey = (_a = options.storageDir) !== null && _a !== void 0 ? _a : this.get('localStorageDir');
|
|
165
|
+
const cacheKey = options.storageDir ?? this.get('localStorageDir');
|
|
168
166
|
return this._getService('ApifyStorageLocal', () => this.createStorageLocal(options), cacheKey);
|
|
169
167
|
}
|
|
170
168
|
/**
|
|
@@ -195,7 +193,7 @@ class Configuration {
|
|
|
195
193
|
* @internal
|
|
196
194
|
*/
|
|
197
195
|
createClient(options = {}) {
|
|
198
|
-
return new
|
|
196
|
+
return new apify_client_1.default({
|
|
199
197
|
baseUrl: this.get('apiBaseUrl'),
|
|
200
198
|
token: this.get('token'),
|
|
201
199
|
...options, // allow overriding the instance configuration
|
|
@@ -211,9 +209,8 @@ class Configuration {
|
|
|
211
209
|
* @internal
|
|
212
210
|
*/
|
|
213
211
|
createStorageLocal(options = {}) {
|
|
214
|
-
|
|
215
|
-
const
|
|
216
|
-
const enableWalMode = (_b = options.enableWalMode) !== null && _b !== void 0 ? _b : this.get('localStorageEnableWalMode');
|
|
212
|
+
const storageDir = options.storageDir ?? this.get('localStorageDir');
|
|
213
|
+
const enableWalMode = options.enableWalMode ?? this.get('localStorageEnableWalMode');
|
|
217
214
|
const storage = new storage_local_1.ApifyStorageLocal({ ...options, storageDir, enableWalMode });
|
|
218
215
|
process.on('exit', () => {
|
|
219
216
|
// TODO this is not public API, need to update
|
|
@@ -237,55 +234,81 @@ class Configuration {
|
|
|
237
234
|
}
|
|
238
235
|
exports.Configuration = Configuration;
|
|
239
236
|
// maps environment variables to config keys (e.g. `APIFY_PROXY_PORT` to `proxyPort`)
|
|
240
|
-
Configuration
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
237
|
+
Object.defineProperty(Configuration, "ENV_MAP", {
|
|
238
|
+
enumerable: true,
|
|
239
|
+
configurable: true,
|
|
240
|
+
writable: true,
|
|
241
|
+
value: {
|
|
242
|
+
APIFY_TOKEN: 'token',
|
|
243
|
+
APIFY_LOCAL_STORAGE_DIR: 'localStorageDir',
|
|
244
|
+
APIFY_LOCAL_STORAGE_ENABLE_WAL_MODE: 'localStorageEnableWalMode',
|
|
245
|
+
APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId',
|
|
246
|
+
APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
|
|
247
|
+
APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
|
|
248
|
+
APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis',
|
|
249
|
+
APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
|
|
250
|
+
APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis',
|
|
251
|
+
APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl',
|
|
252
|
+
APIFY_INPUT_KEY: 'inputKey',
|
|
253
|
+
APIFY_ACTOR_ID: 'actorId',
|
|
254
|
+
APIFY_API_BASE_URL: 'apiBaseUrl',
|
|
255
|
+
APIFY_IS_AT_HOME: 'isAtHome',
|
|
256
|
+
APIFY_ACTOR_RUN_ID: 'actorRunId',
|
|
257
|
+
APIFY_ACTOR_TASK_ID: 'actorTaskId',
|
|
258
|
+
APIFY_CONTAINER_PORT: 'containerPort',
|
|
259
|
+
APIFY_CONTAINER_URL: 'containerUrl',
|
|
260
|
+
APIFY_USER_ID: 'userId',
|
|
261
|
+
APIFY_PROXY_HOSTNAME: 'proxyHostname',
|
|
262
|
+
APIFY_PROXY_PASSWORD: 'proxyPassword',
|
|
263
|
+
APIFY_PROXY_STATUS_URL: 'proxyStatusUrl',
|
|
264
|
+
APIFY_PROXY_PORT: 'proxyPort',
|
|
265
|
+
// not supported, use env vars directly:
|
|
266
|
+
// APIFY_MEMORY_MBYTES: 'memoryMbytes',
|
|
267
|
+
// APIFY_HEADLESS: 'headless',
|
|
268
|
+
// APIFY_XVFB: 'xvfb',
|
|
269
|
+
// APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
270
|
+
}
|
|
271
|
+
});
|
|
270
272
|
// maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`)
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
Configuration.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
};
|
|
273
|
+
Object.defineProperty(Configuration, "ENV_MAP_REVERSED", {
|
|
274
|
+
enumerable: true,
|
|
275
|
+
configurable: true,
|
|
276
|
+
writable: true,
|
|
277
|
+
value: Object.entries(Configuration.ENV_MAP).reduce((obj, [key, value]) => {
|
|
278
|
+
obj[value] = key;
|
|
279
|
+
return obj;
|
|
280
|
+
}, {})
|
|
281
|
+
});
|
|
282
|
+
Object.defineProperty(Configuration, "BOOLEAN_VARS", {
|
|
283
|
+
enumerable: true,
|
|
284
|
+
configurable: true,
|
|
285
|
+
writable: true,
|
|
286
|
+
value: ['localStorageEnableWalMode']
|
|
287
|
+
});
|
|
288
|
+
Object.defineProperty(Configuration, "INTEGER_VARS", {
|
|
289
|
+
enumerable: true,
|
|
290
|
+
configurable: true,
|
|
291
|
+
writable: true,
|
|
292
|
+
value: ['proxyPort', 'memoryMbytes', 'containerPort']
|
|
293
|
+
});
|
|
294
|
+
Object.defineProperty(Configuration, "DEFAULTS", {
|
|
295
|
+
enumerable: true,
|
|
296
|
+
configurable: true,
|
|
297
|
+
writable: true,
|
|
298
|
+
value: {
|
|
299
|
+
defaultKeyValueStoreId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
|
|
300
|
+
defaultDatasetId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_DATASET_ID],
|
|
301
|
+
defaultRequestQueueId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
|
|
302
|
+
inputKey: 'INPUT',
|
|
303
|
+
apiBaseUrl: 'https://api.apify.com',
|
|
304
|
+
proxyStatusUrl: 'http://proxy.apify.com',
|
|
305
|
+
proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
|
|
306
|
+
proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
|
|
307
|
+
containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
|
|
308
|
+
containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
|
|
309
|
+
metamorphAfterSleepMillis: 300e3,
|
|
310
|
+
persistStateIntervalMillis: 60e3,
|
|
311
|
+
localStorageEnableWalMode: true,
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
//# sourceMappingURL=configuration.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.js"],"names":[],"mappings":";;;;AAAA,0CAAyD;AACzD,+BAA4B;AAC5B,wDAAyD;AACzD,6EAAuC;AACvC,yEAA8B;AAE9B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,MAAa,aAAa;IA4DtB;;;;OAIG;IACH,YAAY,OAAO,GAAG,EAAE;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAE1B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;YACpD,MAAM,GAAG,GAAG,IAAA,WAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iBAAiB,CAAC,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;YACjC,mCAAmC;YACnC,mBAAG,CAAC,OAAO,CAAC,WAAW,iBAAQ,CAAC,iBAAiB,QAAQ,iBAAQ,CAAC,KAAK,+CAA+C,iBAAQ,CAAC,iBAAiB,KAAK,GAAG,GAAG,CAAC,CAAC;SAChK;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,GAAG,CAAC,GAAG,EAAE,YAAY;QACjB,kBAAkB;QAClB,MAAM,MAAM,GAAG,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErC,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;SAC5C;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAChC;QAED,uBAAuB;QACvB,OAAO,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;IACnE,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,GAAG,EAAE,KAAK;QACpB,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;SAC5D;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,KAAK;QACV,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,SAAS,CAAC,OAAO,GAAG,EAAE;QAClB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC;QAEvC,OAAO,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,eAAe,CAAC,OAAO,GAAG,EAAE;QACxB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;;;OASG;IACH,WAAW,CAAC,IAAI,EAAE,cAAc,EAAE,QAAQ,GAAG,IAAI;QAC7C,QAAQ,GAAG,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;QAEjC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;SACjD;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;;;OASG;IACH,YAAY,CAAC,OAAO,GAAG,EAAE;QACrB,OAAO,IAAI,sBAAW,CAAC;YACnB,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;YAC/B,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YACxB,GAAG,OAAO,EAAE,8CAA8C;SAC7D,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;OAQG;IACH,kBAAkB,CAAC,OAAO,GAAG,EAAE;QAC3B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACrE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACrF,MAAM,OAAO,GAAG,IAAI,iCAAiB,CAAC,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC;QAEjF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACpB,8CAA8C;YAC9C,mCAAmC;YACnC,OAAO,CAAC,aAAa,CAAC,mBAAmB,EAAE,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE;YAC7B,aAAa,CAAC,YAAY,GAAG,IAAI,aAAa,EAAE,CAAC;SACpD;QAED,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;;AAzPL,sCA0PC;AAzPG,qFAAqF;AAC9E;;;;WAAU;QACb,WAAW,EAAE,OAAO;QACpB,uBAAuB,EAAE,iBAAiB;QAC1C,mCAAmC,EAAE,2BAA2B;QAChE,wBAAwB,EAAE,kBAAkB;QAC5C,gCAAgC,EAAE,wBAAwB;QAC1D,8BAA8B,EAAE,uBAAuB;QACvD,kCAAkC,EAAE,2BAA2B;QAC/D,mCAAmC,EAAE,4BAA4B;QACjE,kCAAkC,EAAE,4BAA4B;QAChE,yBAAyB,EAAE,kBAAkB;QAC7C,eAAe,EAAE,UAAU;QAC3B,cAAc,EAAE,SAAS;QACzB,kBAAkB,EAAE,YAAY;QAChC,gBAAgB,EAAE,UAAU;QAC5B,kBAAkB,EAAE,YAAY;QAChC,mBAAmB,EAAE,aAAa;QAClC,oBAAoB,EAAE,eAAe;QACrC,mBAAmB,EAAE,cAAc;QACnC,aAAa,EAAE,QAAQ;QACvB,oBAAoB,EAAE,eAAe;QACrC,oBAAoB,EAAE,eAAe;QACrC,sBAAsB,EAAE,gBAAgB;QACxC,gBAAgB,EAAE,WAAW;QAE7B,wCAAwC;QACxC,uCAAuC;QACvC,8BAA8B;QAC9B,sBAAsB;QACtB,wDAAwD;KAC3D;GAAC;AAEF,qFAAqF;AAC9E;;;;WAAmB,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACzF,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACjB,OAAO,GAAG,CAAC;IACf,CAAC,EAAE,EAAE,CAAC;GAAC;AAEA;;;;WAAe,CAAC,2BAA2B,CAAC;GAAC;AAE7C;;;;WAAe,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,CAAC;GAAC;AAE9D;;;;WAAW;QACd,sBAAsB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,0BAA0B,CAAC;QAC3E,gBAAgB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,kBAAkB,CAAC;QAC7D,qBAAqB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QACxE,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,uBAAuB;QACnC,cAAc,EAAE,wBAAwB;QACxC,aAAa,EAAE,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACtD,SAAS,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,UAAU,CAAC;QAC/C,aAAa,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACvD,YAAY,EAAE,uBAAc,CAAC,iBAAQ,CAAC,aAAa,CAAC;QACpD,yBAAyB,EAAE,KAAK;QAChC,0BAA0B,EAAE,IAAI;QAChC,yBAAyB,EAAE,IAAI;KAClC;GAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.js"],"names":[],"mappings":"AAEA;;;GAGG;AAEH,4JAA6J;;;;;;AAa7J;;;;GAIG;AACH,mCAFU;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAEjB;AAE5F;;;GAGG;AACH,iCAFU,MAAM,CAE6C;AAE7D;;;;;GAKG;AACH,gDAFU,MAAM,CAEoC;AAEpD,wCAA+C;AAE/C,4CAAoD"}
|
package/build/constants.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.js"],"names":[],"mappings":";;;AAAA,0CAAkD;AAElD;;;GAGG;AACH,mCAAmC;AACtB,QAAA,kBAAkB,GAAG,0HAA0H,CAAC;AAE7J;;;;GAIG;AACU,QAAA,UAAU,GAAG;IACtB,OAAO,EAAE,CAAC;IACV,yBAAyB,EAAE,EAAE;IAC7B,aAAa,EAAE,EAAE;CACpB,CAAC;AAEF;;;;GAIG;AACU,QAAA,oBAAoB,GAAG,EAAE,GAAG,0BAAiB,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC;AAE5F;;;GAGG;AACU,QAAA,kBAAkB,GAAG,0BAA0B,CAAC;AAE7D;;;;;GAKG;AACU,QAAA,iCAAiC,GAAG,EAAE,CAAC;AAEvC,QAAA,kBAAkB,GAAG,YAAY,CAAC;AAElC,QAAA,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC"}
|
|
@@ -433,3 +433,4 @@ import Request from "../request";
|
|
|
433
433
|
import { Session } from "../session_pool/session";
|
|
434
434
|
import { ProxyInfo } from "../proxy_configuration";
|
|
435
435
|
import { AutoscaledPoolOptions } from "../autoscaling/autoscaled_pool";
|
|
436
|
+
//# sourceMappingURL=basic_crawler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":"AA8CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH;IACI;;;OAGG;IACH,qBAFS,GAAG,CAuBV;IAEF;;;OAGG;IACH,qBAHW,mBAAmB,EA+G7B;IA/EG,6BAA6B;IAC7B,SAAc;IACd,qCAA8B;IAC9B,uCAAgC;IAChC,mCAAgD;IAChD,sDAAuD;IACvD,mCAAiE;IACjE,6DAA8D;IAC9D,0BAA0C;IAC1C,6BAA6B;IAC7B,kBAA6F;IAC7F,iCAAiC;IACjC,oBADW,kBAAkB,CAI5B;IACD,wBAAoC;IACpC,gCAAiC;IAuDjC,2BAA2G;IAE3G,uCAA4B;IAOhC;;;;OAIG;IACH,OAFY,QAAQ,IAAI,CAAC,CAuBxB;IAED;;;;;OAKG;IACH,mBALY,QAAQ,IAAI,CAAC,CAkBxB;IATG,2CAAoE;IAGhE,4EAAiE;IAQzE;;;;;;OAMG;IACH,kDANW,eAAe,GACd,QAAQ,IAAI,CAAC,CAOxB;IAED;;;;OAIG;IACH,6CAmCC;IAED;;;;;;;OAOG;IACH,uDAoBC;IAED;;;;;;;OAOG;IACH,4CAuDC;IAED;;;;;;OAMG;IACH,mDAOC;IAED;;;;;;OAMG;IACH,yDAUC;IAED;;;;;;;;;;OAUG;IACH,8CATW,KAAK;QAEoB,OAAO,EAAhC,OAAO;eACP,CAAC,WAAW,GAAC,YAAY,CAAC,GACzB,QAAQ,IAAI,CAAC,CA4BxB;IAED;;;;;;;;OAQG;IACH;QAPkC,KAAK,EAA5B,KAAK;QACoB,OAAO,EAAhC,OAAO;QACN,QAAQ,IAAI,CAAC,CAgBxB;IAED;;;;;;;;;;;;OAYG;IACH,sCALY,QAAQ,IAAI,CAAC,CAWxB;IAED;;;;;;OAMG;IACH,+BANW,UAAW,+BAYrB;IAED;;;OAGG;IACH,0BAIC;CACJ;;QAtoBa,MAAM;aACN,OAAO;aACP,OAAO;eACP,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BAmBT,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qCAonBhB,mBAAmB,KACjB,QAAQ,IAAI,CAAC;;;;;;;;;;aAIZ,OAAO;;;;2CAYV,wBAAwB,KACtB,QAAQ,IAAI,CAAC;;;;;WAKZ,KAAK;;;;aACL,OAAO;aACP,OAAO;eACP,SAAS"}
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BasicCrawler = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
4
5
|
const consts_1 = require("@apify/consts");
|
|
5
6
|
const utilities_1 = require("@apify/utilities");
|
|
6
|
-
const ow_1 = require("ow");
|
|
7
|
-
const
|
|
8
|
-
const autoscaled_pool_1 = require("../autoscaling/autoscaled_pool"); // eslint-disable-line import/no-duplicates
|
|
9
|
-
const events_1 = require("../events");
|
|
7
|
+
const ow_1 = (0, tslib_1.__importStar)(require("ow"));
|
|
8
|
+
const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
|
|
9
|
+
const autoscaled_pool_1 = (0, tslib_1.__importDefault)(require("../autoscaling/autoscaled_pool")); // eslint-disable-line import/no-duplicates
|
|
10
|
+
const events_1 = (0, tslib_1.__importDefault)(require("../events"));
|
|
10
11
|
const session_pool_1 = require("../session_pool/session_pool"); // eslint-disable-line import/no-duplicates
|
|
11
|
-
const statistics_1 = require("./statistics");
|
|
12
|
+
const statistics_1 = (0, tslib_1.__importDefault)(require("./statistics"));
|
|
12
13
|
const utils_1 = require("../utils");
|
|
13
|
-
const utils_log_1 = require("../utils_log");
|
|
14
|
+
const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
|
|
14
15
|
const validators_1 = require("../validators");
|
|
15
16
|
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
|
|
16
17
|
/**
|
|
@@ -260,7 +261,7 @@ class BasicCrawler {
|
|
|
260
261
|
},
|
|
261
262
|
log,
|
|
262
263
|
};
|
|
263
|
-
this.autoscaledPoolOptions =
|
|
264
|
+
this.autoscaledPoolOptions = underscore_1.default.defaults({}, basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);
|
|
264
265
|
this.isRunningPromise = null;
|
|
265
266
|
// Attach a listener to handle migration and aborting events gracefully.
|
|
266
267
|
events_1.default.on(consts_1.ACTOR_EVENT_NAMES.MIGRATING, this._pauseOnMigration.bind(this));
|
|
@@ -501,7 +502,7 @@ class BasicCrawler {
|
|
|
501
502
|
const shouldRetryRequest = !request.noRetry && request.retryCount < this.maxRequestRetries;
|
|
502
503
|
if (shouldRetryRequest) {
|
|
503
504
|
request.retryCount++;
|
|
504
|
-
this.log.exception(error, 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
|
|
505
|
+
this.log.exception(error, 'handleRequestFunction failed, reclaiming failed request back to the list or queue', underscore_1.default.pick(request, 'url', 'retryCount', 'id'));
|
|
505
506
|
await source.reclaimRequest(request);
|
|
506
507
|
}
|
|
507
508
|
else {
|
|
@@ -583,26 +584,31 @@ exports.BasicCrawler = BasicCrawler;
|
|
|
583
584
|
* @internal
|
|
584
585
|
* @type any
|
|
585
586
|
*/
|
|
586
|
-
BasicCrawler
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
587
|
+
Object.defineProperty(BasicCrawler, "optionsShape", {
|
|
588
|
+
enumerable: true,
|
|
589
|
+
configurable: true,
|
|
590
|
+
writable: true,
|
|
591
|
+
value: {
|
|
592
|
+
requestList: ow_1.default.optional.object.validate(validators_1.validators.requestList),
|
|
593
|
+
requestQueue: ow_1.default.optional.object.validate(validators_1.validators.requestQueue),
|
|
594
|
+
// Subclasses override this function instead of passing it
|
|
595
|
+
// in constructor, so this validation needs to apply only
|
|
596
|
+
// if the user creates an instance of BasicCrawler directly.
|
|
597
|
+
handleRequestFunction: ow_1.default.function,
|
|
598
|
+
handleRequestTimeoutSecs: ow_1.default.optional.number,
|
|
599
|
+
handleFailedRequestFunction: ow_1.default.optional.function,
|
|
600
|
+
maxRequestRetries: ow_1.default.optional.number,
|
|
601
|
+
maxRequestsPerCrawl: ow_1.default.optional.number,
|
|
602
|
+
autoscaledPoolOptions: ow_1.default.optional.object,
|
|
603
|
+
sessionPoolOptions: ow_1.default.optional.object,
|
|
604
|
+
useSessionPool: ow_1.default.optional.boolean,
|
|
605
|
+
// AutoscaledPool shorthands
|
|
606
|
+
minConcurrency: ow_1.default.optional.number,
|
|
607
|
+
maxConcurrency: ow_1.default.optional.number,
|
|
608
|
+
// internal
|
|
609
|
+
log: ow_1.default.optional.object,
|
|
610
|
+
}
|
|
611
|
+
});
|
|
606
612
|
/**
|
|
607
613
|
* @callback HandleRequest
|
|
608
614
|
* @param {HandleRequestInputs} inputs Arguments passed to this callback.
|
|
@@ -631,3 +637,4 @@ BasicCrawler.optionsShape = {
|
|
|
631
637
|
* @property {Session} session
|
|
632
638
|
* @property {ProxyInfo} proxyInfo
|
|
633
639
|
*/
|
|
640
|
+
//# sourceMappingURL=basic_crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic_crawler.js","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":";;;;AAAA,0CAAkD;AAClD,gDAAwD;AACxD,sDAAuC;AACvC,yEAA2B;AAC3B,kGAA4D,CAAC,2CAA2C;AACxG,oEAA+B;AAC/B,+DAA+D,CAAC,2CAA2C;AAC3G,2EAAsC;AACtC,oCAA+C;AAC/C,0EAAsC;AACtC,8CAA2C;AAY3C,iFAAiF;AAEjF;;;;;;;GAOG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH,MAAa,YAAY;IA4BrB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,qBAAqB,EACrB,wBAAwB,GAAG,EAAE,EAC7B,2BAA2B,EAC3B,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc;QAEd,WAAW;QACX,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,GAC5D,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE;YAC/B,MAAM,GAAG,GAAG,mGAAmG,CAAC;YAChH,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SAClD;QAED,6DAA6D;QAC7D,6BAA6B;QAC7B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,2BAA2B,CAAC;QACxD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,GAAG,IAAI,CAAC;QAClE,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,CAAC,CAAC;QAC9F,iCAAiC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,EAAE;YACpD,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,0BAA0B,EAAE;kBACrE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,CAAC;SACtD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAG;YAC5C,cAAc;YACd,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,oBAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,uCAAuC,EAAE,qBAAqB,CAAC,CAAC;QAE5G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,wEAAwE;QACxE,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1E,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC;QAExD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,gBAAgB,CAAC;SAC/B;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,gBAAgB;gBAChB,cAAc;gBACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBAChD,GAAG,UAAU;aAChB,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,IAAA,8BAAe,EAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,gHAAgH;8BAChH,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,+CAA+C;8BACjE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE;YACpC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC;SAC/C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAErD,IAAI,OAAO,CAAC;QACZ,IAAI,OAAO,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SACrG;aAAM;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;SAC5C;QAED,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,MAAM,eAAe,GAAG;YACpB,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,OAAO;YACP,OAAO;SACV,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,2BAAmB,EACrB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,EAC5C,IAAI,CAAC,0BAA0B,EAC/B,yCAAyC,IAAI,CAAC,0BAA0B,GAAG,IAAI,WAAW,CAC7F,CAAC;YACF,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;aACzE;YAAC,OAAO,cAAc,EAAE;gBACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,oDAAoD;sBACjF,6GAA6G;sBAC7G,qGAAqG;sBACrG,4GAA4G;sBAC5G,sGAAsG,CAAC,CAAC;gBAC9G,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB;QACtB,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,0BAA0B;QAC5B,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,4BAA4B,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC3F,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,KAAK,EACL,mFAAmF,EACnF,oBAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9C,eAAe,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,MAAM,IAAI,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC,CAAC,yCAAyC;SACtG;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,4BAA4B,CAAC,eAAe;QAC9C,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,eAAe,CAAC,KAAK,EACrB,4CAA4C,EAC5C,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;SACL;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,wBAAwB;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,IAAI;QAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;SACrC;IACL,CAAC;;AA5dL,oCA6dC;AA5dG;;;GAGG;AACI;;;;WAAe;QAClB,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,qBAAqB,EAAE,YAAE,CAAC,QAAQ;QAClC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAElC,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC;AAqcN;;;;GAIG;AACH;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;GAMG"}
|
|
@@ -505,3 +505,4 @@ import { RequestList } from "../request_list";
|
|
|
505
505
|
import { RequestQueue } from "../storages/request_queue";
|
|
506
506
|
import { AutoscaledPoolOptions } from "../autoscaling/autoscaled_pool";
|
|
507
507
|
import { SessionPoolOptions } from "../session_pool/session_pool";
|
|
508
|
+
//# sourceMappingURL=browser_crawler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G;;;;;;;;uCAUE;IAiDN;;;;;;OAMG;IACH,+DANW,sBAAsB,GAAG,eAAe,mBAkBlD;IAED;;;;;OAKG;IACH,6CALW,sBAAsB,GAAG,eAAe,iBAkBlD;IAED;;;;;;;OAOG;IACH,oDANW,sBAAsB,GAAG,eAAe,SACxC,KAAK,QAWf;IAED;;;;;;OAMG;IACH,8CANW,sBAAsB,GAAG,eAAe;YACjC,MAAM;qBAYvB;IAED;;;;;;;;OAQG;IACH,4CANW,sBAAsB,GAAG,eAAe,GACvC,QAAQ,IAAI,CAAC,CAiBxB;IAED;;;;;;OAMG;IACH,uCANW,MAAM,qCA6BhB;IAED;;;;;;;OAOG;IACH,kDANW,MAAM,gCAkBhB;CAUJ;;;;qCAhhBU,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,QAAQ,IAAI,CAAC;kDAIf,sBAAsB,GAAG,eAAe,KACtC,QAAQ,IAAI,CAAC;qCAIf,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,YAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAKT,yBAAyB"}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
const
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
|
|
4
5
|
const browser_pool_1 = require("browser-pool"); // eslint-disable-line import/no-duplicates,no-unused-vars
|
|
5
6
|
const constants_1 = require("../constants");
|
|
6
|
-
const events_1 = require("../session_pool/events"); // eslint-disable-line import/no-duplicates
|
|
7
|
+
const events_1 = (0, tslib_1.__importDefault)(require("../session_pool/events")); // eslint-disable-line import/no-duplicates
|
|
7
8
|
const utils_1 = require("../utils");
|
|
8
9
|
const validators_1 = require("../validators");
|
|
9
10
|
const crawler_utils_1 = require("./crawler_utils");
|
|
@@ -477,20 +478,26 @@ exports.default = BrowserCrawler;
|
|
|
477
478
|
* @internal
|
|
478
479
|
* @type any
|
|
479
480
|
*/
|
|
480
|
-
BrowserCrawler
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
481
|
+
Object.defineProperty(BrowserCrawler, "optionsShape", {
|
|
482
|
+
enumerable: true,
|
|
483
|
+
configurable: true,
|
|
484
|
+
writable: true,
|
|
485
|
+
value: {
|
|
486
|
+
...basic_crawler_1.BasicCrawler.optionsShape,
|
|
487
|
+
// TODO temporary until the API is unified in V2
|
|
488
|
+
handleRequestFunction: ow_1.default.undefined,
|
|
489
|
+
handlePageFunction: ow_1.default.function,
|
|
490
|
+
gotoFunction: ow_1.default.optional.function,
|
|
491
|
+
gotoTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
|
|
492
|
+
navigationTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
|
|
493
|
+
handlePageTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
|
|
494
|
+
preNavigationHooks: ow_1.default.optional.array,
|
|
495
|
+
postNavigationHooks: ow_1.default.optional.array,
|
|
496
|
+
browserPoolOptions: ow_1.default.object,
|
|
497
|
+
sessionPoolOptions: ow_1.default.optional.object,
|
|
498
|
+
persistCookiesPerSession: ow_1.default.optional.boolean,
|
|
499
|
+
useSessionPool: ow_1.default.optional.boolean,
|
|
500
|
+
proxyConfiguration: ow_1.default.optional.object.validate(validators_1.validators.proxyConfiguration),
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
//# sourceMappingURL=browser_crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,oCAA+C;AAC/C,8CAA2C;AAC3C,mDAGyB;AAEzB,kFAAkF;AAClF,mDAAgE;AAUhE,iFAAiF;AAEjF;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAqB,cAAe,SAAQ,4BAAY;IA0BpD;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,kBAAkB,EAClB,qBAAqB,GAAG,EAAE,EAC1B,qBAAqB,GAAG,EAAE,EAC1B,YAAY,EAAE,aAAa;QAC3B,eAAe,EAAE,aAAa;QAC9B,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,IAAI,CAAC;YACxE,wBAAwB,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,6CAAiC;SAC9G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,8EAA8E,CAAC,CAAC;SACvG;QAED,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QACjE,IAAI,CAAC,uBAAuB,GAAG,CAAC,eAAe,IAAI,qBAAqB,CAAC,GAAG,IAAI,CAAC;QAEjF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACxC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,0BAA0B;QAC1B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,0BAA0B;QAC1B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAAC;YAC/B,GAAG,IAAI;YACP,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAEhE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,cAAc,CAAC,MAAM,EAAE;gBACvB,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;aAC5E;SACJ;QAED,IAAI;YACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAE9C,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAE7C,eAAe;YACf,qEAAqE;YACrE,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aAC3D;YAED,MAAM,IAAA,2BAAmB,EACrB,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EACxC,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;YAEF,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SACxF;IACL,CAAC;IAED;;;;;;OAMG;IACH,mCAAmC,CAAC,eAAe,EAAE,IAAI;QACrD,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC;QACpF,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAO,CAAC;QAC1E,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAS,CAAC;IAClF,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,gBAAgB;QAChB,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1F;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,MAAM,KAAK,CAAC;SACf;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;OAOG;IACH,wBAAwB,CAAC,eAAe,EAAE,KAAK;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,oCAAoB,EAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;SAChD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,oDAAoD;YACpD,oFAAoF;YACpF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;SACvE;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,eAAe;QAClC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE;YAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAA,qCAAqB,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACrD;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAM,EAAE,aAAa;QAC5C,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxH,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,aAAa,CAAC,aAAa,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxD;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;OAOG;IACH,+BAA+B,CAAC,MAAM,EAAE,iBAAiB;QACrD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAO,EAAE,EAAE;gBACzB,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAK,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;oBACzC,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;iBAC/D;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YACtD,iBAAiB,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AA9SL,iCA+SC;AA9SG;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAElC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cheerio_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAgDc,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAk0BjB,OAAO;;;;;;;;;;;;sCAYV,oBAAoB,KAClB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;cAKnB,CAAC,eAAe,GAAC,QAAQ,CAAC;;;;aAC1B,OAAO;;;;;;;;;;aAON,cAAc;;oCAKlB,kBAAkB,KAChB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;;;;;UAOnB,CAAC,MAAM,GAAC,MAAM,CAAC;;;;;;;;aAIf,OAAO;;;;iBAEP;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;;;;cAElC,eAAe;aAEf,OAAO;;;;;eACP,SAAS;aAGT,cAAc;;yCAKjB,uBAAuB,KACrB,QAAQ,IAAI,CAAC;AA/3B1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAgF/B;IAvCG,gCAA2F;IAO3F,gCAA2D;IAC3D,6BAAqD;IACrD,yBAAsC;IACtC,4CAAsD;IACtD,0CAAkD;IAClD,mDAAoD;IACpD,+CAAgD;IAChD,mDAA4C;IAC5C;;;SAGK;IACL,oBAHU,UAAW,CAGuB;IAC5C;;;SAGK;IACL,qBAHU,UAAW,CAMpB;IACD;;;SAGK;IACL,oCAAwC;IAGpC,kCAAwG;IAMhH;;;;OAIG;IACH,eAFW,gBAAgB,QA+B1B;IAuED;;;;;OAKG;IACH,6CALW,eAAe,iBA4BzB;IAED;;;;;;;;;;;;;;OAcG;IACH;QAT4B,OAAO,EAAxB,OAAO;QACU,OAAO,EAAxB,OAAO;QACS,QAAQ,EAAxB,MAAM;QAC2B,uBAAuB;QACtD,QAAQ,eAAe,GAAC,QAAQ,CAAC,CAyB7C;IAED;;;;;;;;OAQG;IACH,kCAPW,OAAO,kBACP,eAAe,GAAC,QAAQ,GACtB,QAAQ,MAAM,CAAC,CAgC3B;IAED;;;;;;;;;OASG;IACH,sCARW,OAAO,oGAsCjB;IAED;;;;;;;OAOG;IACH;;;MA+BC;IAED;;;;;OAKG;IACH,uDAYC;IAED;;;;;;OAMG;IACH,yDALW,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC,QAchC;IAED;;;;;;;OAOG;IACH,0CANW,OAAO,cACP,MAAM,QAWhB;IAED;;;;;;OAMG;IACH,yCALW,OAAO,QAQjB;IAED;;;;OAIG;IACH,6BAcC;CACJ"}
|