apify 2.1.1-beta.3 → 2.1.1-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/actor.d.ts.map +1 -1
- package/build/actor.js +3 -2
- package/build/actor.js.map +1 -1
- package/build/apify.d.ts +1 -2
- package/build/apify.d.ts.map +1 -1
- package/build/apify.js +3 -2
- package/build/apify.js.map +1 -1
- package/build/autoscaling/autoscaled_pool.js +2 -2
- package/build/autoscaling/autoscaled_pool.js.map +1 -1
- package/build/browser_launchers/browser_launcher.d.ts.map +1 -1
- package/build/browser_launchers/browser_launcher.js.map +1 -1
- package/build/browser_launchers/playwright_launcher.d.ts +1 -2
- package/build/browser_launchers/playwright_launcher.d.ts.map +1 -1
- package/build/browser_launchers/playwright_launcher.js.map +1 -1
- package/build/browser_launchers/puppeteer_launcher.d.ts.map +1 -1
- package/build/browser_launchers/puppeteer_launcher.js.map +1 -1
- package/build/crawlers/basic_crawler.d.ts +6 -0
- package/build/crawlers/basic_crawler.d.ts.map +1 -1
- package/build/crawlers/basic_crawler.js +32 -15
- package/build/crawlers/basic_crawler.js.map +1 -1
- package/build/crawlers/browser_crawler.d.ts +11 -7
- package/build/crawlers/browser_crawler.d.ts.map +1 -1
- package/build/crawlers/browser_crawler.js +41 -9
- package/build/crawlers/browser_crawler.js.map +1 -1
- package/build/crawlers/cheerio_crawler.d.ts +2 -2
- package/build/crawlers/cheerio_crawler.d.ts.map +1 -1
- package/build/crawlers/cheerio_crawler.js +11 -3
- package/build/crawlers/cheerio_crawler.js.map +1 -1
- package/build/crawlers/playwright_crawler.d.ts +5 -6
- package/build/crawlers/playwright_crawler.js +1 -1
- package/build/crawlers/playwright_crawler.js.map +1 -1
- package/build/crawlers/puppeteer_crawler.d.ts +3 -3
- package/build/crawlers/puppeteer_crawler.js +1 -1
- package/build/crawlers/puppeteer_crawler.js.map +1 -1
- package/build/crawlers/statistics.d.ts +1 -1
- package/build/crawlers/statistics.d.ts.map +1 -1
- package/build/enqueue_links/enqueue_links.d.ts +1 -2
- package/build/main.d.ts +1 -2
- package/build/playwright_utils.d.ts +3 -5
- package/build/request_list.d.ts +0 -9
- package/build/request_list.d.ts.map +1 -1
- package/build/request_list.js +3 -22
- package/build/request_list.js.map +1 -1
- package/build/storages/dataset.d.ts +1 -1
- package/build/storages/dataset.d.ts.map +1 -1
- package/build/storages/request_queue.d.ts.map +1 -1
- package/build/storages/request_queue.js +2 -0
- package/build/storages/request_queue.js.map +1 -1
- package/build/tsconfig.tsbuildinfo +1 -1
- package/build/utils.d.ts +0 -1
- package/build/utils.d.ts.map +1 -1
- package/build/utils.js +9 -28
- package/build/utils.js.map +1 -1
- package/package.json +12 -10
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G
|
|
1
|
+
{"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G;;;;;;;;;;uCAUE;IA8EN;;;;;;;OAOG;IACH,+DAPW,sBAAsB,GAAG,eAAe,gCAExC,OAAO,QAsBjB;IAED;;;;;OAKG;IACH,6CALW,sBAAsB,GAAG,eAAe,iBAqBlD;IAED;;;;;;;OAOG;IACH,oDANW,sBAAsB,GAAG,eAAe,SACxC,KAAK,QAWf;IAED;;;;;;OAMG;IACH,8CANW,sBAAsB,GAAG,eAAe;YACjC,MAAM;qBAYvB;IAED;;;;;;;;OAQG;IACH,4CANW,sBAAsB,GAAG,eAAe,GACvC,QAAQ,IAAI,CAAC,CAiBxB;IAED;;;;;;OAMG;IACH,uCANW,MAAM,qCA6BhB;IAED;;;;;;;OAOG;IACH,kDANW,MAAM,qBACN,iBAAiB,QAiB3B;CAUJ;;uBA1jBa,iBAAiB;;qCAIpB,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,QAAQ,IAAI,CAAC;kDAIf,sBAAsB,GAAG,eAAe,KACtC,QAAQ,IAAI,CAAC;qCAIf,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,YAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAKT,yBAAyB"}
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
|
|
5
|
+
const timeout_1 = require("@apify/timeout");
|
|
5
6
|
const browser_pool_1 = require("browser-pool"); // eslint-disable-line import/no-duplicates,no-unused-vars
|
|
6
7
|
const constants_1 = require("../constants");
|
|
7
8
|
const events_1 = (0, tslib_1.__importDefault)(require("../session_pool/events")); // eslint-disable-line import/no-duplicates
|
|
8
|
-
const utils_1 = require("../utils");
|
|
9
9
|
const validators_1 = require("../validators");
|
|
10
10
|
const crawler_utils_1 = require("./crawler_utils");
|
|
11
11
|
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
|
|
@@ -101,7 +101,7 @@ const basic_crawler_1 = require("./basic_crawler");
|
|
|
101
101
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
102
102
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
103
103
|
* If set, `PuppeteerCrawler` will be configured for all connections to use
|
|
104
|
-
* [Apify Proxy](https://
|
|
104
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
105
105
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
106
106
|
* @property {Array<Hook>} [preNavigationHooks]
|
|
107
107
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
@@ -303,26 +303,51 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
303
303
|
* @internal
|
|
304
304
|
*/
|
|
305
305
|
async _handleRequestFunction(crawlingContext) {
|
|
306
|
-
const { id } = crawlingContext;
|
|
307
|
-
const page = await this.browserPool.newPage({ id });
|
|
308
|
-
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page);
|
|
309
306
|
const { request, session } = crawlingContext;
|
|
307
|
+
const newPageOptions = {
|
|
308
|
+
id: crawlingContext.id,
|
|
309
|
+
};
|
|
310
|
+
const useIncognitoPages = this.launchContext && this.launchContext.useIncognitoPages;
|
|
311
|
+
if (this.proxyConfiguration && useIncognitoPages) {
|
|
312
|
+
const proxyInfo = this.proxyConfiguration.newProxyInfo(session && session.id);
|
|
313
|
+
crawlingContext.session = session;
|
|
314
|
+
crawlingContext.proxyInfo = proxyInfo;
|
|
315
|
+
newPageOptions.proxyUrl = proxyInfo.url;
|
|
316
|
+
// Disable SSL verification for MITM proxies
|
|
317
|
+
if (this.proxyConfiguration.isManInTheMiddle) {
|
|
318
|
+
/**
|
|
319
|
+
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
320
|
+
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
321
|
+
*/
|
|
322
|
+
newPageOptions.pageOptions = {
|
|
323
|
+
ignoreHTTPSErrors: true,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
const page = await this.browserPool.newPage(newPageOptions);
|
|
328
|
+
(0, timeout_1.tryCancel)();
|
|
329
|
+
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
|
|
310
330
|
if (this.useSessionPool) {
|
|
311
331
|
const sessionCookies = session.getPuppeteerCookies(request.url);
|
|
312
332
|
if (sessionCookies.length) {
|
|
313
333
|
await crawlingContext.browserController.setCookies(page, sessionCookies);
|
|
334
|
+
(0, timeout_1.tryCancel)();
|
|
314
335
|
}
|
|
315
336
|
}
|
|
316
337
|
try {
|
|
317
338
|
await this._handleNavigation(crawlingContext);
|
|
339
|
+
(0, timeout_1.tryCancel)();
|
|
318
340
|
await this._responseHandler(crawlingContext);
|
|
341
|
+
(0, timeout_1.tryCancel)();
|
|
319
342
|
// save cookies
|
|
320
343
|
// @TODO: Should we save the cookies also after/only the handle page?
|
|
321
344
|
if (this.persistCookiesPerSession) {
|
|
322
345
|
const cookies = await crawlingContext.browserController.getCookies(page);
|
|
346
|
+
(0, timeout_1.tryCancel)();
|
|
323
347
|
session.setPuppeteerCookies(cookies, request.loadedUrl);
|
|
324
348
|
}
|
|
325
|
-
await (0,
|
|
349
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this.handlePageFunction(crawlingContext), this.handlePageTimeoutMillis, `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
|
|
350
|
+
(0, timeout_1.tryCancel)();
|
|
326
351
|
if (session)
|
|
327
352
|
session.markGood();
|
|
328
353
|
}
|
|
@@ -333,11 +358,12 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
333
358
|
/**
|
|
334
359
|
* @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
|
|
335
360
|
* @param {*} page
|
|
361
|
+
* @param {boolean} useIncognitoPages
|
|
336
362
|
* @ignore
|
|
337
363
|
* @protected
|
|
338
364
|
* @internal
|
|
339
365
|
*/
|
|
340
|
-
_enhanceCrawlingContextWithPageInfo(crawlingContext, page) {
|
|
366
|
+
_enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages) {
|
|
341
367
|
crawlingContext.page = page;
|
|
342
368
|
// This switch is because the crawlingContexts are created on per request basis.
|
|
343
369
|
// However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
|
|
@@ -345,8 +371,12 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
345
371
|
// which configures proxy and session for every new request
|
|
346
372
|
const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
|
|
347
373
|
crawlingContext.browserController = browserControllerInstance;
|
|
348
|
-
|
|
349
|
-
|
|
374
|
+
if (!useIncognitoPages) {
|
|
375
|
+
crawlingContext.session = browserControllerInstance.launchContext.session;
|
|
376
|
+
}
|
|
377
|
+
if (!crawlingContext.proxyInfo) {
|
|
378
|
+
crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
|
|
379
|
+
}
|
|
350
380
|
}
|
|
351
381
|
/**
|
|
352
382
|
* @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
|
|
@@ -358,6 +388,7 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
358
388
|
/** @type {*} */
|
|
359
389
|
const gotoOptions = { ...this.defaultGotoOptions };
|
|
360
390
|
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
|
|
391
|
+
(0, timeout_1.tryCancel)();
|
|
361
392
|
try {
|
|
362
393
|
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
|
|
363
394
|
}
|
|
@@ -365,6 +396,7 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
365
396
|
this._handleNavigationTimeout(crawlingContext, error);
|
|
366
397
|
throw error;
|
|
367
398
|
}
|
|
399
|
+
(0, timeout_1.tryCancel)();
|
|
368
400
|
await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
|
|
369
401
|
}
|
|
370
402
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,
|
|
1
|
+
{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,4CAAgE;AAChE,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,8CAA2C;AAC3C,mDAGyB;AAEzB,kFAAkF;AAClF,mDAAgE;AAUhE,iFAAiF;AAEjF;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAqB,cAAe,SAAQ,4BAAY;IA0BpD;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,kBAAkB,EAClB,qBAAqB,GAAG,EAAE,EAC1B,qBAAqB,GAAG,EAAE,EAC1B,YAAY,EAAE,aAAa;QAC3B,eAAe,EAAE,aAAa;QAC9B,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,IAAI,CAAC;YACxE,wBAAwB,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,6CAAiC;SAC9G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,8EAA8E,CAAC,CAAC;SACvG;QAED,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QACjE,IAAI,CAAC,uBAAuB,GAAG,CAAC,eAAe,IAAI,qBAAqB,CAAC,GAAG,IAAI,CAAC;QAEjF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACxC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,0BAA0B;QAC1B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,0BAA0B;QAC1B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAAC;YAC/B,GAAG,IAAI;YACP,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,MAAM,cAAc,GAAG;YACnB,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC;QACrF,IAAI,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9E,eAAe,CAAC,OAAO,GAAG,OAAO,CAAC;YAClC,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YAExC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;iBAC1B,CAAC;aACL;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,cAAc,CAAC,MAAM,EAAE;gBACvB,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;aACf;SACJ;QAED,IAAI;YACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,IAAA,mBAAS,GAAE,CAAC;YAEZ,eAAe;YACf,qEAAqE;YACrE,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;gBACZ,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aAC3D;YAED,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EAC9C,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SACxF;IACL,CAAC;IAED;;;;;;;OAOG;IACH,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB;QACxE,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC;QACpF,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,iBAAiB,EAAE;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAO,CAAC;SAC7E;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC5B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAS,CAAC;SACjF;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,gBAAgB;QAChB,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1F;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,MAAM,KAAK,CAAC;SACf;QAED,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;OAOG;IACH,wBAAwB,CAAC,eAAe,EAAE,KAAK;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,oCAAoB,EAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;SAChD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,oDAAoD;YACpD,oFAAoF;YACpF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;SACvE;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,eAAe;QAClC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE;YAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAA,qCAAqB,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACrD;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAM,EAAE,aAAa;QAC5C,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxH,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,aAAa,CAAC,aAAa,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxD;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;OAOG;IACH,+BAA+B,CAAC,MAAM,EAAE,iBAAiB;QACrD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAO,EAAE,EAAE;gBACzB,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAK,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;oBACzC,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;iBAC/D;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YACtD,iBAAiB,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AApVL,iCAqVC;AApVG;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAElC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC"}
|
|
@@ -141,7 +141,7 @@ export type CheerioCrawlerOptions = {
|
|
|
141
141
|
ignoreSslErrors?: boolean | undefined;
|
|
142
142
|
/**
|
|
143
143
|
* If set, `CheerioCrawler` will be configured for all connections to use
|
|
144
|
-
* [Apify Proxy](https://
|
|
144
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
145
145
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
146
146
|
*/
|
|
147
147
|
proxyConfiguration?: ProxyConfiguration | undefined;
|
|
@@ -471,7 +471,7 @@ export type CheerioHandlePage = (inputs: CheerioHandlePageInputs) => Promise<voi
|
|
|
471
471
|
* If set to true, SSL certificate errors will be ignored.
|
|
472
472
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
473
473
|
* If set, `CheerioCrawler` will be configured for all connections to use
|
|
474
|
-
* [Apify Proxy](https://
|
|
474
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
475
475
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
476
476
|
* @property {HandleFailedRequest} [handleFailedRequestFunction]
|
|
477
477
|
* A function to handle requests that failed more than `option.maxRequestRetries` times.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"cheerio_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAkDc,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAu3BjB,OAAO;;;;;;;;;;;;sCAYV,oBAAoB,KAClB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;cAKnB,CAAC,eAAe,GAAC,QAAQ,CAAC;;;;aAC1B,OAAO;;;;;;;;;;aAON,cAAc;;oCAKlB,kBAAkB,KAChB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;;;;;UAOnB,CAAC,MAAM,GAAC,MAAM,CAAC;;;;;;;;aAIf,OAAO;;;;iBAEP;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;;;;cAElC,eAAe;aAEf,OAAO;;;;;eACP,SAAS;aAGT,cAAc;;yCAKjB,uBAAuB,KACrB,QAAQ,IAAI,CAAC;AAp7B1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EA2E/B;IAlCG,gCAA2F;IAO3F,gCAA2D;IAC3D,6BAAqD;IACrD,yBAAsC;IACtC,4CAAsD;IACtD,0CAAkD;IAClD,mDAAoD;IACpD,+CAAgD;IAChD,mDAA4C;IAC5C;;;SAGK;IACL,oBAHU,UAAW,CAGuB;IAC5C;;;SAGK;IACL,qBAHU,UAAW,CAMpB;IAGG,kCAAwG;IAMhH;;;;OAIG;IACH,eAFW,gBAAgB,QA+B1B;IAyED;;;;;OAKG;IACH,6CALW,eAAe,iBAwCzB;IAED;;;;;;;;;;;;;;OAcG;IACH,gCASC;IAED;;;;;;;;;;;;;;OAcG;IACH;QAT4B,OAAO,EAAxB,OAAO;QACU,OAAO,EAAxB,OAAO;QACS,QAAQ,EAAxB,MAAM;QAC2B,uBAAuB;QACtD,QAAQ,eAAe,GAAC,QAAQ,CAAC,CAoB7C;IAED;;;;;;;;;;OAUG;IACH,4BAUC;IAED;;;;;;;;OAQG;IACH,kCAPW,OAAO,kBACP,eAAe,GAAC,QAAQ,GACtB,QAAQ,MAAM,CAAC,CAgC3B;IAED;;;;;;;;;OASG;IACH,sCARW,OAAO,oGAsCjB;IAED;;;;;;;OAOG;IACH;;;MA+BC;IAED;;;;;OAKG;IACH,uDAYC;IAED;;;;;;OAMG;IACH,yDALW,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC,QAchC;IAED;;;;;;;OAOG;IACH,0CANW,OAAO,cACP,MAAM,QAWhB;IAED;;;;;;OAMG;IACH,yCALW,OAAO,QAQjB;IAED;;;;OAIG;IACH,6BAcC;CACJ"}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
/* eslint-disable class-methods-use-this */
|
|
5
5
|
const utilities_1 = require("@apify/utilities");
|
|
6
|
+
const timeout_1 = require("@apify/timeout");
|
|
6
7
|
const cheerio_1 = (0, tslib_1.__importDefault)(require("cheerio"));
|
|
7
8
|
const content_type_1 = (0, tslib_1.__importDefault)(require("content-type"));
|
|
8
9
|
const htmlparser2_1 = require("htmlparser2");
|
|
@@ -161,7 +162,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
|
|
|
161
162
|
* If set to true, SSL certificate errors will be ignored.
|
|
162
163
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
163
164
|
* If set, `CheerioCrawler` will be configured for all connections to use
|
|
164
|
-
* [Apify Proxy](https://
|
|
165
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
165
166
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
166
167
|
* @property {HandleFailedRequest} [handleFailedRequestFunction]
|
|
167
168
|
* A function to handle requests that failed more than `option.maxRequestRetries` times.
|
|
@@ -472,7 +473,9 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
472
473
|
crawlingContext.proxyInfo = this.proxyConfiguration.newProxyInfo(sessionId);
|
|
473
474
|
}
|
|
474
475
|
await this._handleNavigation(crawlingContext);
|
|
476
|
+
(0, timeout_1.tryCancel)();
|
|
475
477
|
const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
|
|
478
|
+
(0, timeout_1.tryCancel)();
|
|
476
479
|
if (this.useSessionPool) {
|
|
477
480
|
this._throwOnBlockedRequest(session, response.statusCode);
|
|
478
481
|
}
|
|
@@ -511,7 +514,7 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
511
514
|
return body;
|
|
512
515
|
},
|
|
513
516
|
});
|
|
514
|
-
return (0,
|
|
517
|
+
return (0, timeout_1.addTimeoutToPromise)(() => this.userProvidedHandler(crawlingContext), this.handlePageTimeoutMillis, `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
|
|
515
518
|
}
|
|
516
519
|
/**
|
|
517
520
|
* @param {CrawlingContext} crawlingContext
|
|
@@ -523,6 +526,7 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
523
526
|
if (this.prepareRequestFunction) {
|
|
524
527
|
this.log.deprecated('Option "prepareRequestFunction" is deprecated. Use "preNavigationHooks" instead.');
|
|
525
528
|
await this.prepareRequestFunction(crawlingContext);
|
|
529
|
+
(0, timeout_1.tryCancel)();
|
|
526
530
|
}
|
|
527
531
|
const requestAsBrowserOptions = {};
|
|
528
532
|
if (this.useSessionPool) {
|
|
@@ -531,13 +535,17 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
531
535
|
const { request, session } = crawlingContext;
|
|
532
536
|
const cookieSnapshot = request.headers.Cookie ?? request.headers.cookie;
|
|
533
537
|
await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);
|
|
538
|
+
(0, timeout_1.tryCancel)();
|
|
534
539
|
const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;
|
|
535
540
|
this._mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions);
|
|
536
|
-
crawlingContext.response = await (0,
|
|
541
|
+
crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }), this.requestTimeoutMillis, `request timed out after ${this.requestTimeoutMillis / 1000} seconds.`);
|
|
542
|
+
(0, timeout_1.tryCancel)();
|
|
537
543
|
await this._executeHooks(this.postNavigationHooks, crawlingContext, requestAsBrowserOptions);
|
|
544
|
+
(0, timeout_1.tryCancel)();
|
|
538
545
|
if (this.postResponseFunction) {
|
|
539
546
|
this.log.deprecated('Option "postResponseFunction" is deprecated. Use "postNavigationHooks" instead.');
|
|
540
547
|
await this.postResponseFunction(crawlingContext);
|
|
548
|
+
(0, timeout_1.tryCancel)();
|
|
541
549
|
}
|
|
542
550
|
}
|
|
543
551
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio_crawler.js","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;AAAA,2CAA2C;AAC3C,gDAA4E;AAC5E,mEAA8B;AAC9B,6EAA6C;AAC7C,6CAAyC;AACzC,mEAAgE;AAChE,yEAA+B;AAC/B,yDAAoB;AACpB,6DAAwB;AACxB,+CAA4C;AAC5C,4CAAiE;AACjE,oCAA6E;AAC7E,oDAAoD;AACpD,mDAA4D;AAC5D,mDAA+C,CAAC,2CAA2C;AAC3F,yFAAmD;AAcnD,8CAA2C;AAC3C,iFAAiF;AAEjF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,yCAAyC,GAAG;IAC9C,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH,MAAM,cAAe,SAAQ,4BAAY;IA0BrC;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QAExF,MAAM,EACF,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,yCAAyC,EACjE,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,gDAAgD;YAChD,qBAAqB,EAAE,kBAAkB;YACzC,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,wBAAwB,EAAE,kBAAkB,GAAG,qBAAqB,GAAG,6CAAiC;SAC3G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;SAC5H;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACtD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;QACrD,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC;YACvE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAAS;QACT,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,2BAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACvD,gDAAgD;QAChD,gBAAgB,CAAC,mBAAmB,GAAG,gBAAgB,CAAC,kBAAkB,CAAC;QAC3E,OAAO,gBAAgB,CAAC,kBAAkB,CAAC;QAE3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,sBAAsB;YACvE,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YAEjC,IAAI,CAAC,cAAc,EAAE,EAAE,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,wDAAwD,CAAC,CAAC;aAC5H;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,sBAAsB,GAAG,KAAK,YAAY,IAAI,CAC5H,CAAC;aACL;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,kCAAkC,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE5G,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACrB;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SAC/E;QAED,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;QAE9C,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEjH,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;SAC7D;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;SAC5C;QAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEjC,MAAM,CAAC,GAAG,GAAG;YACT,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,OAAO,EAAE,KAAK;gBACd,yGAAyG;gBACzG,gFAAgF;gBAChF,2GAA2G;gBAC3G,eAAe,EAAE,IAAI;aACxB,CAAC;YACF,CAAC,CAAC,IAAI,CAAC;QAEX,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC;QACtB,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACpC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;SACJ,CAAC,CAAC;QACH,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,6GAA6G;gBAC7G,mFAAmF;gBACnF,sEAAsE;gBACtE,IAAI,GAAG,EAAE;oBACL,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;iBAC9D;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,IAAA,2BAAmB,EACtB,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,EACzC,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kFAAkF,CAAC,CAAC;YACxG,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;SACtD;QAED,MAAM,uBAAuB,GAAG,EAAE,CAAC;QAEnC,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,mBAAmB,CAAC,eAAe,EAAE,uBAAuB,CAAC,CAAC;SACtE;QAED,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACxE,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC;QAC5E,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE/E,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,2BAAmB,EAChD,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,CAAC,EAC9E,IAAI,CAAC,oBAAoB,EACzB,2BAA2B,IAAI,CAAC,oBAAoB,GAAG,IAAI,WAAW,CACzE,CAAC;QAEF,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAE7F,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iFAAiF,CAAC,CAAC;YACvG,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB;QACpE,MAAM,UAAU,GAAG,IAAA,2BAAW,EAAC,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE9G,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE;gBAC/D,uBAAuB,CAAC,OAAO,CAAC,MAAM;gBACtC,UAAU;aACb,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE;QAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,CAAC,CAAC;QAC1F,IAAI,kBAAkB,CAAC;QAEvB,IAAI;YACA,kBAAkB,GAAG,MAAM,IAAA,gCAAgB,EAAC,IAAI,CAAC,CAAC;SACrD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,YAAY,2BAAY,EAAE;gBAC3B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;aACvC;iBAAM;gBACH,MAAM,CAAC,CAAC;aACX;SACJ;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,uBAAuB;QAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACpE,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,CAAC;QAE7E,0CAA0C;QAC1C,IAAI,aAAa,EAAE;YACf,uBAAuB,CAAC,OAAO,KAA/B,uBAAuB,CAAC,OAAO,GAAK,EAAE,EAAC;YACvC,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC;SAC1D;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,cAAc;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,oCAA4B,EAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAU,IAAI,GAAG,EAAE;YACnB,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE;gBACrC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;aACjD;YAED,6EAA6E;YAC7E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;SACpF;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;SACxE;aAAM;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SAC1C;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB;QAClE,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE;YAC/C,YAAY,EAAE,OAAO;YACrB,GAAG,uBAAuB;YAC1B,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,uBAAuB,CAAC,OAAO,EAAE;YACnE,KAAK,EAAE;gBACH,GAAG,uBAAuB,CAAC,KAAK;gBAChC,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,2EAA2E;QAC3E,oEAAoE;QACpE,yEAAyE;QACzE,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YACrE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;SACL;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAEjF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ;QACvC,IAAI,IAAI,CAAC,qBAAqB,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACzC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE;YAClD,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC;SAC3C;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAChC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvE,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAe;gBACzB,QAAQ,EAAE,IAAI;aACjB,CAAC;SACL;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,QAAQ;QAC1B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3C,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ;iBACH,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,yBAAyB,CAAC,mBAAmB;QACzC,mBAAmB,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrC,IAAI;gBACA,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;aAC9F;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,sBAAsB,CAAC,OAAO,EAAE,UAAU;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAEjE,IAAI,SAAS,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,cAAc,CAAC,CAAC;SAC3E;IACL,CAAC;IAED;;;;;;OAMG;IACH,qBAAqB,CAAC,OAAO;QACzB,IAAI,OAAO;YAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,QAAQ;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAA,oCAA4B,EAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE;YACpB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;SAChI;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,UAAU,GAAG,GAAG,EAAE;YACxD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBAC7D,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;SAC3G;IACL,CAAC;;AArhBD;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;QAC9E,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC5C,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC1C,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;GAAA;AAigBL,kBAAe,cAAc,CAAC;AAE9B;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;GAWG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;GAIG"}
|
|
1
|
+
{"version":3,"file":"cheerio_crawler.js","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;AAAA,2CAA2C;AAC3C,gDAA4E;AAC5E,4CAAgE;AAChE,mEAA8B;AAC9B,6EAA6C;AAC7C,6CAAyC;AACzC,mEAAgE;AAChE,yEAA+B;AAC/B,yDAAoB;AACpB,6DAAwB;AACxB,+CAA4C;AAC5C,4CAAiE;AACjE,oCAAwD;AACxD,oDAAoD;AACpD,mDAA4D;AAC5D,mDAA+C,CAAC,2CAA2C;AAC3F,yFAAmD;AAcnD,8CAA2C;AAC3C,iFAAiF;AAEjF;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,yCAAyC,GAAG;IAC9C,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH,MAAM,cAAe,SAAQ,4BAAY;IA0BrC;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QAExF,MAAM,EACF,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QAExB,eAAe;QACf,qBAAqB,GAAG,yCAAyC,EACjE,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,gDAAgD;YAChD,qBAAqB,EAAE,kBAAkB;YACzC,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,wBAAwB,EAAE,kBAAkB,GAAG,qBAAqB,GAAG,6CAAiC;SAC3G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,sGAAsG,CAAC,CAAC;SAC5H;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACtD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;QACrD,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C;;;aAGK;QACL,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC;YACvE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAAS;QACT,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,2BAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QACvD,gDAAgD;QAChD,gBAAgB,CAAC,mBAAmB,GAAG,gBAAgB,CAAC,kBAAkB,CAAC;QAC3E,OAAO,gBAAgB,CAAC,kBAAkB,CAAC;QAE3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE;YACzD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,sBAAsB;YACvE,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;YAEjC,IAAI,CAAC,cAAc,EAAE,EAAE,wDAAwD;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,wDAAwD,CAAC,CAAC;aAC5H;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,0DAA0D;gBACnF,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,sBAAsB,GAAG,KAAK,YAAY,IAAI,CAC5H,CAAC;aACL;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,kCAAkC,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE5G,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACrB;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;SAC/E;QAED,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;QAC9C,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QACjH,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;SAC7D;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;SAC5C;QAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEjC,MAAM,CAAC,GAAG,GAAG;YACT,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBAChB,OAAO,EAAE,KAAK;gBACd,yGAAyG;gBACzG,gFAAgF;gBAChF,2GAA2G;gBAC3G,eAAe,EAAE,IAAI;aACxB,CAAC;YACF,CAAC,CAAC,IAAI,CAAC;QAEX,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC;QACtB,eAAe,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1C,eAAe,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACpC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;SACJ,CAAC,CAAC;QACH,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;YAC3C,GAAG;gBACC,6GAA6G;gBAC7G,mFAAmF;gBACnF,sEAAsE;gBACtE,IAAI,GAAG,EAAE;oBACL,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;iBAC9D;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC;SACJ,CAAC,CAAC;QAEH,OAAO,IAAA,6BAAmB,EACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,EAC/C,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,IAAI,IAAI,CAAC,sBAAsB,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kFAAkF,CAAC,CAAC;YACxG,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,CAAC;YACnD,IAAA,mBAAS,GAAE,CAAC;SACf;QAED,MAAM,uBAAuB,GAAG,EAAE,CAAC;QAEnC,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,mBAAmB,CAAC,eAAe,EAAE,uBAAuB,CAAC,CAAC;SACtE;QAED,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACxE,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC5F,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,IAAI,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC;QAC5E,IAAI,CAAC,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB,CAAC,CAAC;QAE/E,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,CAAC,EACpF,IAAI,CAAC,oBAAoB,EACzB,2BAA2B,IAAI,CAAC,oBAAoB,GAAG,IAAI,WAAW,CACzE,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,uBAAuB,CAAC,CAAC;QAC7F,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iFAAiF,CAAC,CAAC;YACvG,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;YACjD,IAAA,mBAAS,GAAE,CAAC;SACf;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,uBAAuB,CAAC,OAAO,EAAE,cAAc,EAAE,uBAAuB;QACpE,MAAM,UAAU,GAAG,IAAA,2BAAW,EAAC,OAAO,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE9G,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YACvB,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE;gBAC/D,uBAAuB,CAAC,OAAO,CAAC,MAAM;gBACtC,UAAU;aACb,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE;QAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB,CAAC,CAAC;QAC1F,IAAI,kBAAkB,CAAC;QAEvB,IAAI;YACA,kBAAkB,GAAG,MAAM,IAAA,gCAAgB,EAAC,IAAI,CAAC,CAAC;SACrD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,YAAY,2BAAY,EAAE;gBAC3B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;aACvC;iBAAM;gBACH,MAAM,CAAC,CAAC;aACX;SACJ;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;OAUG;IACH,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,uBAAuB;QAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QACpE,MAAM,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,GAAG,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC,CAAC;QAE7E,0CAA0C;QAC1C,IAAI,aAAa,EAAE;YACf,uBAAuB,CAAC,OAAO,KAA/B,uBAAuB,CAAC,OAAO,GAAK,EAAE,EAAC;YACvC,uBAAuB,CAAC,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC;SAC1D;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,cAAc;QACxC,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAA,oCAA4B,EAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAU,IAAI,GAAG,EAAE;YACnB,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE;gBACrC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;aACjD;YAED,6EAA6E;YAC7E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;SACpF;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;SACxE;aAAM;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;SAC1C;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,uBAAuB;QAClE,MAAM,cAAc,GAAG;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,oBAAoB,EAAE;YAC/C,YAAY,EAAE,OAAO;YACrB,GAAG,uBAAuB;YAC1B,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,uBAAuB,CAAC,OAAO,EAAE;YACnE,KAAK,EAAE;gBACH,GAAG,uBAAuB,CAAC,KAAK;gBAChC,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,2EAA2E;QAC3E,oEAAoE;QACpE,yEAAyE;QACzE,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YACrE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;SACL;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;QAEjF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ;QACvC,IAAI,IAAI,CAAC,qBAAqB,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACzC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE;YAClD,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC;SAC3C;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE;YAChC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/D,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvE,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAe;gBACzB,QAAQ,EAAE,IAAI;aACjB,CAAC;SACL;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,QAAQ;QAC1B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,wBAAU,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3C,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,+BAAc,CAAC,UAAU,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,QAAQ;iBACH,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;OAMG;IACH,yBAAyB,CAAC,mBAAmB;QACzC,mBAAmB,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACrC,IAAI;gBACA,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAChD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;aAC9F;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,sBAAsB,CAAC,OAAO,EAAE,UAAU;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAEjE,IAAI,SAAS,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,cAAc,CAAC,CAAC;SAC3E;IACL,CAAC;IAED;;;;;;OAMG;IACH,qBAAqB,CAAC,OAAO;QACzB,IAAI,OAAO;YAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,QAAQ;QAClC,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,IAAA,oCAA4B,EAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,UAAU,KAAK,GAAG,EAAE;YACpB,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,oFAAoF,CAAC,CAAC;SAChI;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,UAAU,GAAG,GAAG,EAAE;YACxD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;kBAC7D,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;SAC3G;IACL,CAAC;;AA5hBD;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;QAC9E,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC5C,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC1C,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;GAAC;AAwgBN,kBAAe,cAAc,CAAC;AAE9B;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;GAWG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;GAIG"}
|
|
@@ -12,7 +12,7 @@ export type PlaywrightGotoOptions = {
|
|
|
12
12
|
* - `'load'` - consider operation to be finished when the `load` event is fired.
|
|
13
13
|
* - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
|
|
14
14
|
*/
|
|
15
|
-
waitUntil?: "
|
|
15
|
+
waitUntil?: "domcontentloaded" | "load" | "networkidle" | undefined;
|
|
16
16
|
/**
|
|
17
17
|
* Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
|
|
18
18
|
*/
|
|
@@ -129,14 +129,14 @@ export type PlaywrightCrawlerOptions = {
|
|
|
129
129
|
* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
|
|
130
130
|
* You can tweak those to fine-tune browser management.
|
|
131
131
|
*/
|
|
132
|
-
browserPoolOptions?: BrowserPoolOptions<import("browser-pool
|
|
132
|
+
browserPoolOptions?: BrowserPoolOptions<import("browser-pool").BrowserPlugin<import("browser-pool/dist/abstract-classes/browser-plugin").CommonLibrary, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonBrowser, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonPage>> | undefined;
|
|
133
133
|
/**
|
|
134
134
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
135
135
|
*/
|
|
136
136
|
persistCookiesPerSession?: boolean | undefined;
|
|
137
137
|
/**
|
|
138
138
|
* If set, `PlaywrightCrawler` will be configured for all connections to use
|
|
139
|
-
* [Apify Proxy](https://
|
|
139
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
140
140
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
141
141
|
*/
|
|
142
142
|
proxyConfiguration?: ProxyConfiguration | undefined;
|
|
@@ -310,7 +310,7 @@ export type PlaywrightCrawlerOptions = {
|
|
|
310
310
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
311
311
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
312
312
|
* If set, `PlaywrightCrawler` will be configured for all connections to use
|
|
313
|
-
* [Apify Proxy](https://
|
|
313
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
314
314
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
315
315
|
* @property {RequestList} [requestList]
|
|
316
316
|
* Static list of URLs to be processed.
|
|
@@ -433,8 +433,7 @@ declare class PlaywrightCrawler extends BrowserCrawler {
|
|
|
433
433
|
constructor(options: PlaywrightCrawlerOptions);
|
|
434
434
|
launchContext: PlaywrightLaunchContext;
|
|
435
435
|
}
|
|
436
|
-
|
|
437
|
-
import { Page } from "playwright/types/types";
|
|
436
|
+
import { Page } from "playwright-core";
|
|
438
437
|
import { BrowserCrawlingContext } from "./browser_crawler";
|
|
439
438
|
import { CrawlingContext } from "./basic_crawler";
|
|
440
439
|
import { HandleFailedRequest } from "./basic_crawler";
|
|
@@ -128,7 +128,7 @@ const playwright_utils_1 = require("../playwright_utils");
|
|
|
128
128
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
129
129
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
130
130
|
* If set, `PlaywrightCrawler` will be configured for all connections to use
|
|
131
|
-
* [Apify Proxy](https://
|
|
131
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
132
132
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
133
133
|
* @property {RequestList} [requestList]
|
|
134
134
|
* Static list of URLs to be processed.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright_crawler.js","sourceRoot":"","sources":["../../src/crawlers/playwright_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AAKpB,kFAAuG;AACvG,qFAA2E;AAS3E,iFAAiF;AAEjF,0DAAmD;AAEnD;;;;;;;;;;;;;GAaG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmFG;AACH,MAAM,iBAAkB,SAAQ,yBAAc;IAY1C;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,0BAA0B,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE9F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAE,EACvB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,sFAAsF;kBAChG,iDAAiD,CAAC,CAAC;SAC5D;QACD,MAAM,kBAAkB,GAAG,IAAI,wCAAkB,CAAC,aAAa,CAAC,CAAC;QAEjE,kBAAkB,CAAC,cAAc,GAAG;YAChC,kBAAkB,CAAC,mBAAmB,EAAE;SAC3C,CAAC;QAEF,KAAK,CAAC;YACF,GAAG,qBAAqB;YACxB,kBAAkB;SACrB,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,2GAA2G,CAAC,CAAC;YACjI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1D;QACD,OAAO,IAAA,+BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAvDD;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,yBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;
|
|
1
|
+
{"version":3,"file":"playwright_crawler.js","sourceRoot":"","sources":["../../src/crawlers/playwright_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AAKpB,kFAAuG;AACvG,qFAA2E;AAS3E,iFAAiF;AAEjF,0DAAmD;AAEnD;;;;;;;;;;;;;GAaG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmFG;AACH,MAAM,iBAAkB,SAAQ,yBAAc;IAY1C;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,0BAA0B,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE9F,MAAM,EACF,aAAa,GAAG,EAAE,EAClB,kBAAkB,GAAG,EAAE,EACvB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,sFAAsF;kBAChG,iDAAiD,CAAC,CAAC;SAC5D;QACD,MAAM,kBAAkB,GAAG,IAAI,wCAAkB,CAAC,aAAa,CAAC,CAAC;QAEjE,kBAAkB,CAAC,cAAc,GAAG;YAChC,kBAAkB,CAAC,mBAAmB,EAAE;SAC3C,CAAC;QAEF,KAAK,CAAC;YACF,GAAG,qBAAqB;YACxB,kBAAkB;SACrB,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,2GAA2G,CAAC,CAAC;YACjI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1D;QACD,OAAO,IAAA,+BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAvDD;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,yBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;GAAC;AAiDN,kBAAe,iBAAiB,CAAC"}
|
|
@@ -85,14 +85,14 @@ export type PuppeteerCrawlerOptions = {
|
|
|
85
85
|
* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.
|
|
86
86
|
* You can tweak those to fine-tune browser management.
|
|
87
87
|
*/
|
|
88
|
-
browserPoolOptions?: BrowserPoolOptions<import("browser-pool
|
|
88
|
+
browserPoolOptions?: BrowserPoolOptions<import("browser-pool").BrowserPlugin<import("browser-pool/dist/abstract-classes/browser-plugin").CommonLibrary, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonBrowser, unknown, import("browser-pool/dist/abstract-classes/browser-plugin").CommonPage>> | undefined;
|
|
89
89
|
/**
|
|
90
90
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
91
91
|
*/
|
|
92
92
|
persistCookiesPerSession?: boolean | undefined;
|
|
93
93
|
/**
|
|
94
94
|
* If set, `PuppeteerCrawler` will be configured for all connections to use
|
|
95
|
-
* [Apify Proxy](https://
|
|
95
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
96
96
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
97
97
|
*/
|
|
98
98
|
proxyConfiguration?: ProxyConfiguration | undefined;
|
|
@@ -259,7 +259,7 @@ export type PuppeteerCrawlerOptions = {
|
|
|
259
259
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
260
260
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
261
261
|
* If set, `PuppeteerCrawler` will be configured for all connections to use
|
|
262
|
-
* [Apify Proxy](https://
|
|
262
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
263
263
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
264
264
|
* @property {Array<PuppeteerHook>} [preNavigationHooks]
|
|
265
265
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
@@ -92,7 +92,7 @@ const browser_crawler_1 = (0, tslib_1.__importDefault)(require("./browser_crawle
|
|
|
92
92
|
* Automatically saves cookies to Session. Works only if Session Pool is used.
|
|
93
93
|
* @property {ProxyConfiguration} [proxyConfiguration]
|
|
94
94
|
* If set, `PuppeteerCrawler` will be configured for all connections to use
|
|
95
|
-
* [Apify Proxy](https://
|
|
95
|
+
* [Apify Proxy](https://console.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
|
|
96
96
|
* For more information, see the [documentation](https://docs.apify.com/proxy).
|
|
97
97
|
* @property {Array<PuppeteerHook>} [preNavigationHooks]
|
|
98
98
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"puppeteer_crawler.js","sourceRoot":"","sources":["../../src/crawlers/puppeteer_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AAEpB,wDAAkD;AAClD,8EAAuD;AAEvD,kFAAkF;AAClF,gFAAoG;AAEpG,qFAA2E;AAS3E,iFAAiF;AAEjF;;;;;GAKG;AACH;;;;GAIG;AACH;;;;GAIG;AACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkFG;AACH,MAAM,gBAAiB,SAAQ,yBAAc;IAWzC;;;MAGE;IACF,YAAY,OAAO,GAAG,EAAE;QACpB,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAAE,qDAAqD;QACzE,kBAAkB,GAAG,EAAE,EACvB,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,EACF,OAAO,GAAG,KAAK,GAClB,GAAG,aAAa,CAAC;QAElB,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QACD,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC;YACF,GAAG,qBAAqB;YACxB,kBAAkB;YAClB,kBAAkB;SACrB,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE;YACT,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE;gBACtE,8EAA8E;gBAC9E,mDAAmD;gBACnD,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,iBAAiB,CAAC,cAAc,CAAC;gBACjF,MAAM,IAAA,iBAAqB,EAAC,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAC9E,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1D;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAtED;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,yBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;
|
|
1
|
+
{"version":3,"file":"puppeteer_crawler.js","sourceRoot":"","sources":["../../src/crawlers/puppeteer_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AAEpB,wDAAkD;AAClD,8EAAuD;AAEvD,kFAAkF;AAClF,gFAAoG;AAEpG,qFAA2E;AAS3E,iFAAiF;AAEjF;;;;;GAKG;AACH;;;;GAIG;AACH;;;;GAIG;AACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6HG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkFG;AACH,MAAM,gBAAiB,SAAQ,yBAAc;IAWzC;;;MAGE;IACF,YAAY,OAAO,GAAG,EAAE;QACpB,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE5F,MAAM,EACF,aAAa,GAAG,EAAE,EAAE,qDAAqD;QACzE,kBAAkB,GAAG,EAAE,EACvB,kBAAkB,EAClB,GAAG,qBAAqB,EAC3B,GAAG,OAAO,CAAC;QAEZ,MAAM,EACF,OAAO,GAAG,KAAK,GAClB,GAAG,aAAa,CAAC;QAElB,IAAI,aAAa,CAAC,QAAQ,EAAE;YACxB,MAAM,IAAI,KAAK,CAAC,oFAAoF;kBAC9F,gDAAgD,CAAC,CAAC;SAC3D;QACD,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,aAAa,CAAC,CAAC;QAE/D,kBAAkB,CAAC,cAAc,GAAG;YAChC,iBAAiB,CAAC,mBAAmB,EAAE;SAC1C,CAAC;QAEF,KAAK,CAAC;YACF,GAAG,qBAAqB;YACxB,kBAAkB;YAClB,kBAAkB;SACrB,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE;YACT,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE;gBACtE,8EAA8E;gBAC9E,mDAAmD;gBACnD,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,EAAE,GAAG,iBAAiB,CAAC,cAAc,CAAC;gBACjF,MAAM,IAAA,iBAAqB,EAAC,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAC9E,CAAC,CAAC,CAAC;SACN;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,iHAAiH,CAAC,CAAC;YAEvI,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1D;QACD,OAAO,IAAA,8BAAY,EAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAtED;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,yBAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACpC;GAAC;AAiEN,kBAAe,gBAAgB,CAAC"}
|
|
@@ -131,7 +131,7 @@ declare class Statistics {
|
|
|
131
131
|
* displaying the current state in predefined intervals
|
|
132
132
|
*/
|
|
133
133
|
startCapturing(): Promise<void>;
|
|
134
|
-
logInterval: NodeJS.
|
|
134
|
+
logInterval: NodeJS.Timer | null | undefined;
|
|
135
135
|
/**
|
|
136
136
|
* Stops logging and remove event listeners, then persist
|
|
137
137
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.js"],"names":[],"mappings":";;;;;;;;;2BAsVc,MAAM,EAAE;aACR,MAAM;oCACN,MAAM;sCACN,MAAM;+BACN,MAAM;6BACN,MAAM;gCACN,MAAM;mBACN,MAAM;0BACN,MAAM;+BACN,MAAM;sBACN,MAAM;;;;;;sBAON,MAAM;oBACN,MAAM;qBACN,MAAM;6BACN,MAAM;+BACN,MAAM;8BACN,MAAM;8BACN,MAAM;sCACN,MAAM;wCACN,MAAM;sBACN,IAAI,GAAC,MAAM,GAAC,IAAI;uBAChB,IAAI,GAAC,MAAM,GAAC,IAAI;0BAChB,MAAM;sBACN,IAAI,GAAC,MAAM,GAAC,IAAI;;AApV9B;;;;;;;;;;;;;;;;GAgBG;AACH;IACI;;;OAGG;IACH,qDA6BC;IAlBG,kCAAqD;IACrD,0BAA+C;IAC/C,mBAA4B;IAC5B,0EAAyB;IAEzB,WAAyB;IACzB,wBAA0D;IAC1D,8BAA4C;IAC5C,6BAA+B;IAE/B;;;OAGG;IACH,2BAAmC;IAMvC;;OAEG;IACH,cAsBC;IArBG;;;;;;;;;;;;;;kBAcC;IAID,kCAA+B;IAKnC;;;;;OAKG;IACH,aAHW,MAAM,GAAC,MAAM,QAQvB;IAED;;;;;OAKG;IACH,cAHW,MAAM,GAAC,MAAM,QAavB;IAED;;;;;OAKG;IACH,YAHW,MAAM,GAAC,MAAM,QAUvB;IAED;;OAEG;IACH;;;;;;;;MAmBC;IAED;;;OAGG;IACH,gCAiBC;IANG
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.js"],"names":[],"mappings":";;;;;;;;;2BAsVc,MAAM,EAAE;aACR,MAAM;oCACN,MAAM;sCACN,MAAM;+BACN,MAAM;6BACN,MAAM;gCACN,MAAM;mBACN,MAAM;0BACN,MAAM;+BACN,MAAM;sBACN,MAAM;;;;;;sBAON,MAAM;oBACN,MAAM;qBACN,MAAM;6BACN,MAAM;+BACN,MAAM;8BACN,MAAM;8BACN,MAAM;sCACN,MAAM;wCACN,MAAM;sBACN,IAAI,GAAC,MAAM,GAAC,IAAI;uBAChB,IAAI,GAAC,MAAM,GAAC,IAAI;0BAChB,MAAM;sBACN,IAAI,GAAC,MAAM,GAAC,IAAI;;AApV9B;;;;;;;;;;;;;;;;GAgBG;AACH;IACI;;;OAGG;IACH,qDA6BC;IAlBG,kCAAqD;IACrD,0BAA+C;IAC/C,mBAA4B;IAC5B,0EAAyB;IAEzB,WAAyB;IACzB,wBAA0D;IAC1D,8BAA4C;IAC5C,6BAA+B;IAE/B;;;OAGG;IACH,2BAAmC;IAMvC;;OAEG;IACH,cAsBC;IArBG;;;;;;;;;;;;;;kBAcC;IAID,kCAA+B;IAKnC;;;;;OAKG;IACH,aAHW,MAAM,GAAC,MAAM,QAQvB;IAED;;;;;OAKG;IACH,cAHW,MAAM,GAAC,MAAM,QAavB;IAED;;;;;OAKG;IACH,YAHW,MAAM,GAAC,MAAM,QAUvB;IAED;;OAEG;IACH;;;;;;;;MAmBC;IAED;;;OAGG;IACH,gCAiBC;IANG,6CAK0B;IAG9B;;OAEG;IACH,+BAMC;IAED;;;;;OAKG;IACH,qCALW,GAAG,QAWb;IAED;;OAEG;IACH,8BASC;IAED;;;;;OAKG;IACH,gDAsCC;IAED;;;;OAIG;IACH,4BAQC;IAED;;;;;OAKG;IACH,UAFa,uBAAuB,GAAG,cAAc,CAgBpD;CACJ;;;;AA9TD;;GAEG;AACH;IAEQ,yBAAqB;IACrB,aAAa;IAGjB,cAGC;IAED,iBAGC;IAFG,mCAAiD;IAIrD,qBAEC;CACJ"}
|
|
@@ -109,8 +109,7 @@ export function extractUrlsFromPage(page: PuppeteerPage | PlaywrightPage, select
|
|
|
109
109
|
export function extractUrlsFromCheerio($: (any | any), selector: string, baseUrl: string): string[];
|
|
110
110
|
// @ts-ignore optional peer dependency
|
|
111
111
|
import { Page as PuppeteerPage } from "puppeteer";
|
|
112
|
-
|
|
113
|
-
import { Page as PlaywrightPage } from "playwright/types/types";
|
|
112
|
+
import { Page as PlaywrightPage } from "playwright-core";
|
|
114
113
|
import { RequestQueue } from "../storages/request_queue";
|
|
115
114
|
import { RequestTransform } from "./shared";
|
|
116
115
|
import { QueueOperationInfo } from "../storages/request_queue";
|
package/build/main.d.ts
CHANGED
|
@@ -109,8 +109,7 @@ declare const exportedUtils: {
|
|
|
109
109
|
} | undefined) => Promise<void>;
|
|
110
110
|
};
|
|
111
111
|
playwright: {
|
|
112
|
-
|
|
113
|
-
gotoExtended: (page: import("playwright").Page, request: Request, gotoOptions?: import("./playwright_utils").DirectNavigationOptions | undefined) => Promise<import("playwright").Response | null>;
|
|
112
|
+
gotoExtended: (page: import("playwright-core").Page, request: Request, gotoOptions?: import("./playwright_utils").DirectNavigationOptions | undefined) => Promise<import("playwright-core").Response | null>;
|
|
114
113
|
};
|
|
115
114
|
social: {
|
|
116
115
|
emailsFromText: (text: string) => string[];
|
|
@@ -16,15 +16,13 @@ export type DirectNavigationOptions = {
|
|
|
16
16
|
* - `'load'` - consider operation to be finished when the `load` event is fired.
|
|
17
17
|
* - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
|
|
18
18
|
*/
|
|
19
|
-
waitUntil?: "
|
|
19
|
+
waitUntil?: "domcontentloaded" | "load" | "networkidle" | undefined;
|
|
20
20
|
/**
|
|
21
21
|
* Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
|
|
22
22
|
*/
|
|
23
23
|
referer?: string | undefined;
|
|
24
24
|
};
|
|
25
|
-
|
|
26
|
-
import { Page } from "playwright/types/types";
|
|
25
|
+
import { Page } from "playwright-core";
|
|
27
26
|
import Request from "./request";
|
|
28
|
-
|
|
29
|
-
import { Response } from "playwright/types/types";
|
|
27
|
+
import { Response } from "playwright-core";
|
|
30
28
|
//# sourceMappingURL=playwright_utils.d.ts.map
|
package/build/request_list.d.ts
CHANGED
|
@@ -345,15 +345,6 @@ export class RequestList {
|
|
|
345
345
|
* @internal
|
|
346
346
|
*/
|
|
347
347
|
protected _fetchRequestsFromUrl(source: any): Promise<Array<RequestOptions>>;
|
|
348
|
-
/**
|
|
349
|
-
* Tries to detect wrong urls and fix them. Currently detects only sharing url instead of csv download one.
|
|
350
|
-
* @param {string} url
|
|
351
|
-
* @return {string}
|
|
352
|
-
* @ignore
|
|
353
|
-
* @internal
|
|
354
|
-
* @private
|
|
355
|
-
*/
|
|
356
|
-
private _ensureCorrectRemoteRequestListUrl;
|
|
357
348
|
/**
|
|
358
349
|
* Adds given request.
|
|
359
350
|
* If the `source` parameter is a string or plain object and not an instance
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../src/request_list.js"],"names":[],"mappings":"AAeA,yDAA0D;AAC1D,+DAAgE;AAIhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyHG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH;IACI;;OAEG;IACH,sBAFW,kBAAkB,EAoE5B;IAxCG,kCAA+C;IAI/C,6BAA6B;IAC7B,UADW,MAAM,OAAO,CAAC,CACP;IAGlB,kBAAkB;IAGlB,qBAA0B;IAK1B,eAAoB;IAKpB,cAAmB;IAEnB,oCAAmF;IACnF,uCAA+F;IAE/F,2CAAyB;IAGzB,2BAA0C;IAG1C,0BAA4B;IAE5B,8BAAiC;IACjC,mBAAsB;IACtB,uBAA0B;IAE1B;yBAhQ0D,MAAM;SAgQpC;IAC5B,wDAAsC;IAG1C;;;;;OAKG;IACH,cAFa,QAAQ,IAAI,CAAC,CAwBzB;IAED;;;;;;;;OAQG;IACH,mDALW,MAAM,iBAkBhB;IAED;;;;;;;;;OASG;IACH,qCALa,QAAQ,IAAI,CAAC,CAqCzB;IAED;;;;;;;;OAQG;IACH,gBAFY,QAAQ,IAAI,CAAC,CAaxB;IAED;;;;;;;;;OASG;IACH,8BALY,QAAQ,IAAI,CAAC,CASxB;IAED;;;;;;;OAOG;IACH,+BALW,gBAAgB,QA0D1B;IAED;;;;;;;;OAQG;IACH,4CALY,QAAQ,MAAM,CAAC,gBAAgB,GAAC,IAAI,CAAC,CAAC,CAAC,CAqBlD;IAED;;;;;OAKG;IACH,YAFa,gBAAgB,CAY5B;IAED;;;;;;OAMG;IACH,WAFa,QAAQ,OAAO,CAAC,CAM5B;IAED;;;;OAIG;IACH,cAFa,QAAQ,OAAO,CAAC,CAM5B;IAED;;;;;;;;;OASG;IACH,oBAFa,QAAQ,CAAC,OAAO,GAAC,IAAI,CAAC,CAAC,CAuBnC;IAED;;;;;OAKG;IACH,4BAHW,OAAO,GACL,QAAQ,IAAI,CAAC,CAWzB;IAED;;;;;;OAMG;IACH,wBAHW,OAAO,GACL,QAAQ,IAAI,CAAC,CAUzB;IAED;;;;;;OAMG;IACH,gFAiBC;IAED;;;;;;;OAOG;IACH,8CALY,QAAQ,MAAM,cAAc,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"request_list.d.ts","sourceRoot":"","sources":["../src/request_list.js"],"names":[],"mappings":"AAeA,yDAA0D;AAC1D,+DAAgE;AAIhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyHG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH;IACI;;OAEG;IACH,sBAFW,kBAAkB,EAoE5B;IAxCG,kCAA+C;IAI/C,6BAA6B;IAC7B,UADW,MAAM,OAAO,CAAC,CACP;IAGlB,kBAAkB;IAGlB,qBAA0B;IAK1B,eAAoB;IAKpB,cAAmB;IAEnB,oCAAmF;IACnF,uCAA+F;IAE/F,2CAAyB;IAGzB,2BAA0C;IAG1C,0BAA4B;IAE5B,8BAAiC;IACjC,mBAAsB;IACtB,uBAA0B;IAE1B;yBAhQ0D,MAAM;SAgQpC;IAC5B,wDAAsC;IAG1C;;;;;OAKG;IACH,cAFa,QAAQ,IAAI,CAAC,CAwBzB;IAED;;;;;;;;OAQG;IACH,mDALW,MAAM,iBAkBhB;IAED;;;;;;;;;OASG;IACH,qCALa,QAAQ,IAAI,CAAC,CAqCzB;IAED;;;;;;;;OAQG;IACH,gBAFY,QAAQ,IAAI,CAAC,CAaxB;IAED;;;;;;;;;OASG;IACH,8BALY,QAAQ,IAAI,CAAC,CASxB;IAED;;;;;;;OAOG;IACH,+BALW,gBAAgB,QA0D1B;IAED;;;;;;;;OAQG;IACH,4CALY,QAAQ,MAAM,CAAC,gBAAgB,GAAC,IAAI,CAAC,CAAC,CAAC,CAqBlD;IAED;;;;;OAKG;IACH,YAFa,gBAAgB,CAY5B;IAED;;;;;;OAMG;IACH,WAFa,QAAQ,OAAO,CAAC,CAM5B;IAED;;;;OAIG;IACH,cAFa,QAAQ,OAAO,CAAC,CAM5B;IAED;;;;;;;;;OASG;IACH,oBAFa,QAAQ,CAAC,OAAO,GAAC,IAAI,CAAC,CAAC,CAuBnC;IAED;;;;;OAKG;IACH,4BAHW,OAAO,GACL,QAAQ,IAAI,CAAC,CAWzB;IAED;;;;;;OAMG;IACH,wBAHW,OAAO,GACL,QAAQ,IAAI,CAAC,CAUzB;IAED;;;;;;OAMG;IACH,gFAiBC;IAED;;;;;;;OAOG;IACH,8CALY,QAAQ,MAAM,cAAc,CAAC,CAAC,CAwBzC;IAED;;;;;;;;;OASG;IACH,8BALW,CAAC,MAAM,GAAC,OAAO,GAAC,MAAM,CAAC,QAmCjC;IAED;;;;;;;OAOG;IACH,sDAIC;IAED;;;;;;OAMG;IACH,iEAOC;IAED;;;;;;OAMG;IACH,uCAIC;IAED;;;;OAIG;IACH,UAFa,MAAM,CAMlB;IAED;;;;OAIG;IACH,gBAFa,MAAM,CAMlB;CACJ;AA2DM,0CAlCI,MAAM,GAAC,IAAI,WAWX,MAAM,cAAc,GAAC,OAAO,GAAC,MAAM,CAAC,6CAkBlC,QAAQ,WAAW,CAAC,CAkBhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA5zBiE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA+0B1D,MAAM;;;;mBAEN,MAAM;;;;;YAEC,MAAM,GAAC,OAAO;;;+CAMvB,QAAQ,MAAM,CAAC,cAAc,GAAC,OAAO,GAAC,MAAM,CAAC,CAAC,CAAC"}
|
package/build/request_list.js
CHANGED
|
@@ -601,40 +601,21 @@ class RequestList {
|
|
|
601
601
|
async _fetchRequestsFromUrl(source) {
|
|
602
602
|
const { requestsFromUrl, regex, ...sharedOpts } = source;
|
|
603
603
|
const { downloadListOfUrls } = utils_1.publicUtils;
|
|
604
|
-
const fixedRequestsFromUrl = this._ensureCorrectRemoteRequestListUrl(requestsFromUrl);
|
|
605
|
-
if (requestsFromUrl !== fixedRequestsFromUrl) {
|
|
606
|
-
this.log.warning('Detected wrong request list url format, fixing automatically.', { old: requestsFromUrl, new: fixedRequestsFromUrl });
|
|
607
|
-
}
|
|
608
604
|
// Download remote resource and parse URLs.
|
|
609
605
|
let urlsArr;
|
|
610
606
|
try {
|
|
611
|
-
urlsArr = await downloadListOfUrls({ url:
|
|
607
|
+
urlsArr = await downloadListOfUrls({ url: requestsFromUrl, urlRegExp: regex });
|
|
612
608
|
}
|
|
613
609
|
catch (err) {
|
|
614
|
-
throw new Error(`Cannot fetch a request list from ${
|
|
610
|
+
throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);
|
|
615
611
|
}
|
|
616
612
|
// Skip if resource contained no URLs.
|
|
617
613
|
if (!urlsArr.length) {
|
|
618
|
-
this.log.warning('list fetched, but it is empty.', { requestsFromUrl
|
|
614
|
+
this.log.warning('list fetched, but it is empty.', { requestsFromUrl, regex });
|
|
619
615
|
return [];
|
|
620
616
|
}
|
|
621
617
|
return urlsArr.map((url) => ({ url, ...sharedOpts }));
|
|
622
618
|
}
|
|
623
|
-
/**
|
|
624
|
-
* Tries to detect wrong urls and fix them. Currently detects only sharing url instead of csv download one.
|
|
625
|
-
* @param {string} url
|
|
626
|
-
* @return {string}
|
|
627
|
-
* @ignore
|
|
628
|
-
* @internal
|
|
629
|
-
* @private
|
|
630
|
-
*/
|
|
631
|
-
_ensureCorrectRemoteRequestListUrl(url) {
|
|
632
|
-
const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/edit/);
|
|
633
|
-
if (match) {
|
|
634
|
-
return `${match[1]}/gviz/tq?tqx=out:csv`;
|
|
635
|
-
}
|
|
636
|
-
return url;
|
|
637
|
-
}
|
|
638
619
|
/**
|
|
639
620
|
* Adds given request.
|
|
640
621
|
* If the `source` parameter is a string or plain object and not an instance
|