apify 2.1.1-beta.18 → 2.1.1-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/apify.d.ts +11 -11
- package/build/apify.d.ts.map +1 -1
- package/build/autoscaling/autoscaled_pool.d.ts +1 -1
- package/build/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/build/autoscaling/autoscaled_pool.js +2 -2
- package/build/autoscaling/autoscaled_pool.js.map +1 -1
- package/build/autoscaling/snapshotter.d.ts +1 -1
- package/build/autoscaling/snapshotter.d.ts.map +1 -1
- package/build/crawlers/basic_crawler.d.ts +5 -5
- package/build/crawlers/basic_crawler.d.ts.map +1 -1
- package/build/crawlers/basic_crawler.js +9 -5
- package/build/crawlers/basic_crawler.js.map +1 -1
- package/build/crawlers/browser_crawler.d.ts +2 -0
- package/build/crawlers/browser_crawler.d.ts.map +1 -1
- package/build/crawlers/browser_crawler.js +10 -2
- package/build/crawlers/browser_crawler.js.map +1 -1
- package/build/crawlers/cheerio_crawler.d.ts +6 -6
- package/build/crawlers/cheerio_crawler.d.ts.map +1 -1
- package/build/crawlers/cheerio_crawler.js +10 -2
- package/build/crawlers/cheerio_crawler.js.map +1 -1
- package/build/crawlers/playwright_crawler.d.ts +1 -1
- package/build/crawlers/puppeteer_crawler.d.ts +1 -1
- package/build/crawlers/puppeteer_crawler.d.ts.map +1 -1
- package/build/enqueue_links/enqueue_links.d.ts +2 -2
- package/build/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/build/playwright_utils.d.ts +1 -1
- package/build/puppeteer_request_interception.d.ts +1 -1
- package/build/puppeteer_utils.d.ts +1 -1
- package/build/puppeteer_utils.d.ts.map +1 -1
- package/build/session_pool/session.d.ts +5 -5
- package/build/session_pool/session.d.ts.map +1 -1
- package/build/session_pool/session_utils.d.ts +1 -1
- package/build/session_pool/session_utils.d.ts.map +1 -1
- package/build/tsconfig.tsbuildinfo +1 -1
- package/build/utils.d.ts +2 -3
- package/build/utils.d.ts.map +1 -1
- package/build/utils.js +1 -26
- package/build/utils.js.map +1 -1
- package/build/utils_request.d.ts +1 -1
- package/build/utils_request.d.ts.map +1 -1
- package/package.json +4 -3
package/build/apify.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export class Apify {
|
|
|
74
74
|
* the promise will be awaited. The user function is called with no arguments.
|
|
75
75
|
* @return {Promise<unknown>}
|
|
76
76
|
*/
|
|
77
|
-
main(userFunc:
|
|
77
|
+
main(userFunc: any): Promise<unknown>;
|
|
78
78
|
/**
|
|
79
79
|
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
80
80
|
* waits for the actor to finish and fetches its output.
|
|
@@ -154,8 +154,8 @@ export class Apify {
|
|
|
154
154
|
waitSecs?: number | undefined;
|
|
155
155
|
fetchOutput?: boolean | undefined;
|
|
156
156
|
disableBodyParser?: boolean | undefined;
|
|
157
|
-
webhooks?:
|
|
158
|
-
} | undefined): Promise<
|
|
157
|
+
webhooks?: any[] | undefined;
|
|
158
|
+
} | undefined): Promise<any>;
|
|
159
159
|
/**
|
|
160
160
|
* Runs an actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
161
161
|
* waits for the task to finish and fetches its output.
|
|
@@ -222,8 +222,8 @@ export class Apify {
|
|
|
222
222
|
timeoutSecs?: number | undefined;
|
|
223
223
|
build?: string | undefined;
|
|
224
224
|
waitSecs?: number | undefined;
|
|
225
|
-
webhooks?:
|
|
226
|
-
} | undefined): Promise<
|
|
225
|
+
webhooks?: any[] | undefined;
|
|
226
|
+
} | undefined): Promise<any>;
|
|
227
227
|
/**
|
|
228
228
|
* Transforms this actor run to an actor run of a given actor. The system stops the current container and starts
|
|
229
229
|
* the new container instead. All the default storages are preserved and the new input is stored under the `INPUT-METAMORPH-1` key
|
|
@@ -283,11 +283,11 @@ export class Apify {
|
|
|
283
283
|
* For more information, see the [Get webhook](https://apify.com/docs/api/v2#/reference/webhooks/webhook-object/get-webhook) API endpoint.
|
|
284
284
|
*/
|
|
285
285
|
addWebhook(options: {
|
|
286
|
-
eventTypes:
|
|
286
|
+
eventTypes: any;
|
|
287
287
|
requestUrl: string;
|
|
288
288
|
payloadTemplate?: string | undefined;
|
|
289
289
|
idempotencyKey?: string | undefined;
|
|
290
|
-
}): Promise<
|
|
290
|
+
}): Promise<any | undefined>;
|
|
291
291
|
/**
|
|
292
292
|
* Stores an object or an array of objects to the default {@link Dataset} of the current actor run.
|
|
293
293
|
*
|
|
@@ -507,7 +507,7 @@ export class Apify {
|
|
|
507
507
|
* options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
|
|
508
508
|
* @returns {Promise<RequestList>}
|
|
509
509
|
*/
|
|
510
|
-
openRequestList(listName: string | null, sources: Array<
|
|
510
|
+
openRequestList(listName: string | null, sources: Array<any | Request | string>, options?: any): Promise<RequestList>;
|
|
511
511
|
/**
|
|
512
512
|
* Opens a request queue and returns a promise resolving to an instance
|
|
513
513
|
* of the {@link RequestQueue} class.
|
|
@@ -540,7 +540,7 @@ export class Apify {
|
|
|
540
540
|
* @param {SessionPoolOptions} sessionPoolOptions
|
|
541
541
|
* @return {Promise<SessionPool>}
|
|
542
542
|
*/
|
|
543
|
-
openSessionPool(sessionPoolOptions:
|
|
543
|
+
openSessionPool(sessionPoolOptions: any): Promise<SessionPool>;
|
|
544
544
|
/**
|
|
545
545
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
546
546
|
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
@@ -588,7 +588,7 @@ export class Apify {
|
|
|
588
588
|
* If some of the variables are not defined or are invalid, the corresponding value in the resulting object will be null.
|
|
589
589
|
* @returns {ApifyEnv}
|
|
590
590
|
*/
|
|
591
|
-
getEnv():
|
|
591
|
+
getEnv(): any;
|
|
592
592
|
/**
|
|
593
593
|
* Returns a new instance of the Apify API client. The `ApifyClient` class is provided
|
|
594
594
|
* by the <a href="https://www.npmjs.com/package/apify-client" target="_blank">apify-client</a>
|
|
@@ -606,7 +606,7 @@ export class Apify {
|
|
|
606
606
|
token?: string | undefined;
|
|
607
607
|
maxRetries?: string | undefined;
|
|
608
608
|
minDelayBetweenRetriesMillis?: string | undefined;
|
|
609
|
-
} | undefined):
|
|
609
|
+
} | undefined): any;
|
|
610
610
|
/**
|
|
611
611
|
* Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
|
|
612
612
|
*
|
package/build/apify.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"apify.d.ts","sourceRoot":"","sources":["../src/apify.js"],"names":[],"mappings":"AAsBA;;;;;;;GAOG;AACH;IACI,0BAGC;IAFG,sBAAwC;IACxC,gCAAiC;IAGrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+DG;IACH,
|
|
1
|
+
{"version":3,"file":"apify.d.ts","sourceRoot":"","sources":["../src/apify.js"],"names":[],"mappings":"AAsBA;;;;;;;GAOG;AACH;IACI,0BAGC;IAFG,sBAAwC;IACxC,gCAAiC;IAGrC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+DG;IACH,qBAFY,QAAQ,OAAO,CAAC,CA+C3B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmEG;IACH,YAxCW,MAAM;;;;;;;;;;;;oBAqCJ,YAAiB,CAgE7B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyDG;IACH,iBA9BW,MAAM;;;;;;;;;oBA2BJ,YAAiB,CA0D7B;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,yBAlBW,MAAM;;;;;oBAgBJ,QAAQ,IAAI,CAAC,CA+BzB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH;QApB+B,UAAU;QAGb,UAAU,EAA3B,MAAM;QAEW,eAAe;QAOf,cAAc;QAK9B,QAAQ,MAAW,SAAS,CAAC,CAiCxC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,eAJW,MAAM,GAEJ,QAAQ,IAAI,CAAC,CAKzB;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH;;oBAFa,QAAQ,OAAO,CAAC,CAS5B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,cARW,MAAM,GAEJ,QAAQ;YAAO,MAAM;QAAK,MAAM,GAAC,MAAM,GAAC,IAAI,CAAC,CASzD;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,cAbW,MAAM;;oBAWL,QAAQ,IAAI,CAAC,CAKxB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,YANa,QAAQ;YAAO,MAAM;QAAK,MAAM,GAAC,MAAM,GAAC,IAAI,CAAC,CAQzD;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH;;oBAFa,QAAQ,aAAa,CAAC,CASlC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqDG;IACH,0BA/BW,MAAM,GAAC,IAAI,WAWX,MAAM,MAAe,OAAO,GAAC,MAAM,CAAC,kBAkBlC,QAAQ,WAAW,CAAC,CAgBhC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH;;oBAFa,QAAQ,YAAY,CAAC,CASjC;IAED;;;;;;;;OAQG;IACH,0CAFY,QAAQ,WAAW,CAAC,CAO/B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqCG;IACH,2DAFa,QAAQ,kBAAkB,GAAC,SAAS,CAAC,CAgBjD;IAED;;;;;;;OAOG;IACH,cAEC;IAED;;;;;;;;;;;;OAYG;IACH;;;;wBAEC;IAED;;;;OAIG;IACH,YAFa,OAAO,CAInB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAUC;IAED;;;;OAIG;IACH,2BAOC;IAED;;;;OAIG;IACH,2BAIC;CACJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH;IACI;;;OAGG;IACH,qBAHW,qBAAqB,EA4E/B;IAlCG;;OAEG;IACH,
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.d.ts","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH;IACI;;;OAGG;IACH,qBAHW,qBAAqB,EA4E/B;IAlCG;;OAEG;IACH,SAAkD;IAGlD,gCAAsD;IACtD,yBAAwC;IACxC,2BAA4C;IAC5C,+BAAyD;IACzD,8BAAuD;IACvD,gCAA2D;IAC3D,0BAA+C;IAC/C,0BAAsC;IACtC,6BAA4C;IAC5C,8BAA8C;IAG9C,wBAAqC;IACrC,wBAAqC;IACrC,4BAAuG;IACvG,4BAA4B;IAC5B,mBAAsB;IACtB,wBAAwB;IACxB,uCAAmB;IACnB,wCAAkB;IA2RtB;;;;;;;;OAQG;IACH,iDAuCC;IAtJD;;;;;;;;;;;OAWG;IACH,6DAwFC;IAlRG,yBAAsC;IACtC,2BAA6C;IAYjD;;;;;;;OAOG;IACH,gCAGC;IApBD;;;;OAIG;IACH,6BAEC;IAwBD;;;;OAIG;IACH,gCAGC;IAjBD;;;;OAIG;IACH,6BAEC;IAsBD;;;;;OAKG;IACH,oCAGC;IAnBD;;;;;OAKG;IACH,iCAEC;IAaD;;;;OAIG;IACH,iCAEC;IAED;;;;;OAKG;IACH,OAFY,QAAQ,IAAI,CAAC,CAwBxB;IArBG,sCAGE;IAKF;;kBAAyF;IAKzF;;kBAA0F;IAU9F;;;;;;;;;;;;OAYG;IACH,SAFY,QAAQ,IAAI,CAAC,CAQxB;IAED;;;;;;;;;;;;;OAaG;IACH,yCAFY,QAAQ,IAAI,CAAC,CAwBxB;IAED;;;;;OAKG;IACH,eAEC;IA2CG,yCAA+B;IA+GnC;;;;;;;;OAQG;IACH,iCALW,MAAM,QAahB;IAED;;;;;;;;OAQG;IACH,mCALW,MAAM,QAahB;IAED;;;;;;;;;OASG;IACH,wCAiBC;IAbG,wCAA8B;IAelC;;;;;;OAMG;IACH,oCAOC;CACJ"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
|
+
const timeout_1 = require("@apify/timeout");
|
|
4
5
|
const utilities_1 = require("@apify/utilities");
|
|
5
6
|
const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
|
|
6
7
|
const snapshotter_1 = (0, tslib_1.__importDefault)(require("./snapshotter")); // eslint-disable-line import/named,no-unused-vars
|
|
7
8
|
const system_status_1 = (0, tslib_1.__importDefault)(require("./system_status")); // eslint-disable-line import/named,no-unused-vars
|
|
8
9
|
const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
|
|
9
|
-
const utils_1 = require("../utils");
|
|
10
10
|
/**
|
|
11
11
|
* @typedef AutoscaledPoolOptions
|
|
12
12
|
* @property {Function} [runTaskFunction]
|
|
@@ -403,7 +403,7 @@ class AutoscaledPool {
|
|
|
403
403
|
// Execute the current task.
|
|
404
404
|
this.log.perf('Running a task.');
|
|
405
405
|
if (this.taskTimeoutMillis > 0) {
|
|
406
|
-
await (0,
|
|
406
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this.runTaskFunction(), this.taskTimeoutMillis, `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`);
|
|
407
407
|
}
|
|
408
408
|
else {
|
|
409
409
|
await this.runTaskFunction();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.js"],"names":[],"mappings":";;;AAAA,gDAA0E;AAC1E,yDAAoB;AACpB,6EAAgE,CAAC,kDAAkD;AACnH,iFAAoE,CAAC,kDAAkD;AACvH,0EAAsC;AACtC,oCAA+C;AAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,cAAc;IAChB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,mBAAU,GACnB,GAAG,OAAO,CAAC;QAEZ;;WAEG;QACH,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,OAAO,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC;QACxG,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,WAAW;YAAE,OAAO,CAAC,WAAW,GAAG,IAAI,qBAAW,CAAC,EAAE,GAAG,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,uBAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,IAAI,cAAc,CAAC,KAAK;QACpB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc,CAAC,KAAK;QACpB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,IAAI,kBAAkB,CAAC,KAAK;QACxB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,KAAK,CAAC,WAAW;QACnB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAO,CAAC;YACZ,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,aAAa,CAAC,gBAAgB;QAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,2BAAmB,EACrB,IAAI,CAAC,eAAe,EAAE,EACtB,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,gBAAgB;QACvB,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;OAQG;IACH,QAAQ,CAAC,YAAY;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,YAAY;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;CACJ;AAED,kBAAe,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.js"],"names":[],"mappings":";;;AAAA,4CAAqD;AACrD,gDAA0E;AAC1E,yDAAoB;AACpB,6EAAgE,CAAC,kDAAkD;AACnH,iFAAoE,CAAC,kDAAkD;AACvH,0EAAsC;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,cAAc;IAChB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,YAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACxC,mBAAmB,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,eAAe,CAAC;YAC1D,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,IAAI,EACrB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,IAAI,EAC9B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,mBAAU,GACnB,GAAG,OAAO,CAAC;QAEZ;;WAEG;QACH,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,OAAO,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC;QACxG,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEnD,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,WAAW;YAAE,OAAO,CAAC,WAAW,GAAG,IAAI,qBAAW,CAAC,EAAE,GAAG,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,uBAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,IAAI,cAAc,CAAC,KAAK;QACpB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,IAAI,cAAc,CAAC,KAAK;QACpB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,IAAI,kBAAkB,CAAC,KAAK;QACxB,IAAA,YAAE,EAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,0FAA0F;QAC1F,IAAI,CAAC,gBAAgB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI;YACA,MAAM,IAAI,CAAC,WAAW,CAAC;SAC1B;gBAAS;YACN,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SAC3C;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE;YACd,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;SACzB;IACL,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,KAAK,CAAC,WAAW;QACnB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,OAAO,CAAC;YACZ,IAAI,WAAW,EAAE;gBACb,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0CAA0C;0BAC1D,MAAM,WAAW,sCAAsC,CAAC,CAAC;oBAC/D,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;aACnB;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE;oBAC/B,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;iBACb;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,aAAa,CAAC,gBAAgB;QAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YACtD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE;YACnE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;SACjB;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI;YACA,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;SACpC;QACD,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9B;QAED,IAAI;YACA,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE;gBAC5B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAC5B,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;aACL;iBAAM;gBACH,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aAChC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,gBAAgB;QACvB,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE;gBACzD,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;aACN;SACJ;QAED,8BAA8B;QAC9B,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;OAQG;IACH,QAAQ,CAAC,YAAY;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,YAAY;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI;YACA,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACpB;SACJ;gBAAS;YACN,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;SACnC;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;CACJ;AAED,kBAAe,cAAc,CAAC"}
|
|
@@ -112,7 +112,7 @@ declare class Snapshotter {
|
|
|
112
112
|
*/
|
|
113
113
|
constructor(options?: SnapshotterOptions | undefined);
|
|
114
114
|
/** @type {defaultLog.Log} */
|
|
115
|
-
log:
|
|
115
|
+
log: any;
|
|
116
116
|
eventLoopSnapshotIntervalMillis: number;
|
|
117
117
|
memorySnapshotIntervalMillis: number;
|
|
118
118
|
clientSnapshotIntervalMillis: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAYA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH;IACI;;OAEG;IACH,sDAwDC;IA7BG,6BAA6B;IAC7B,
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAYA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH;IACI;;OAEG;IACH,sDAwDC;IA7BG,6BAA6B;IAC7B,SAA+C;IAE/C,wCAA2E;IAC3E,qCAAqE;IACrE,qCAAqE;IACrE,kCAA+D;IAC/D,8BAAuD;IACvD,yBAAwC;IACxC,2BAA4C;IAC5C,wBAAsC;IACtC,wBAAsC;IACtC,8BAA+F;IAE/F,oBAAsB;IACtB,0BAA4B;IAC5B,uBAAyB;IACzB,uBAAyB;IAEzB;;aAA6B;IAC7B;;aAA0B;IAC1B;;aAA0B;IAC1B;;aAAuB;IAEvB,gDAA8C;IAsNlD;;;;;;;OAOG;IACH,6CALW,MAAM,QAehB;IA9HD;;;;;;;OAOG;IACH,2DAYC;IAvHD;;;OAGG;IACH,SAFY,QAAQ,IAAI,CAAC,CAexB;IAED;;;OAGG;IACH,QAFY,QAAQ,IAAI,CAAC,CAWxB;IAED;;;;;OAKG;IACH,4DAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,+DAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,yDAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,sCAHW,MAAM,GACL,UAAQ,CAInB;IAED;;;;;;;;OAQG;IACH,gCAPW,UAAQ,8CAEP,UAAQ,CAmBnB;IAwBD;;;;;;;;OAQG;IACH,8DALY,QAAQ,IAAI,CAAC,CAyBxB;IAED;;;;;;OAMG;IACH,iEAgBC;IAED;;;;;;OAMG;IACH,+DAqBC;IAsBD;;;;;;;OAOG;IACH,gEAyBC;IAED;;;;OAIG;IACH;;;MASC;IAED;;;;;;;;;;;;OAYG;IACH,uDAsBC;IAED;;;;;;;;OAQG;IACH,qCANW,UAAQ,OACR,IAAI,QAad;IAED;;;;;OAKG;IACH,mDAUC;CACJ"}
|
|
@@ -158,7 +158,7 @@ export class BasicCrawler {
|
|
|
158
158
|
*/
|
|
159
159
|
constructor(options: BasicCrawlerOptions);
|
|
160
160
|
/** @type {defaultLog.Log} */
|
|
161
|
-
log:
|
|
161
|
+
log: any;
|
|
162
162
|
requestList: RequestList | undefined;
|
|
163
163
|
requestQueue: RequestQueue | undefined;
|
|
164
164
|
userProvidedHandler: HandleRequest;
|
|
@@ -216,10 +216,10 @@ export class BasicCrawler {
|
|
|
216
216
|
* Wrapper around _runTaskFunction to set a timeout on the entire function.
|
|
217
217
|
* Temporary solution to see whether this causes the 0 concurrency bug.
|
|
218
218
|
* @ignore
|
|
219
|
-
* @
|
|
220
|
-
* @
|
|
219
|
+
* @internal
|
|
220
|
+
* @private
|
|
221
221
|
*/
|
|
222
|
-
|
|
222
|
+
private timeoutForRunTaskFunction;
|
|
223
223
|
/**
|
|
224
224
|
* Wrapper around handleRequestFunction that fetches requests from RequestList/RequestQueue
|
|
225
225
|
* then retries them in a case of an error, etc.
|
|
@@ -293,7 +293,7 @@ export class BasicCrawler {
|
|
|
293
293
|
* @protected
|
|
294
294
|
* @internal
|
|
295
295
|
*/
|
|
296
|
-
protected _executeHooks(hooks: Array<
|
|
296
|
+
protected _executeHooks(hooks: Array<any>, ...args: any): Promise<void>;
|
|
297
297
|
/**
|
|
298
298
|
* Function for cleaning up after all request are processed.
|
|
299
299
|
* @ignore
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":"AA8CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH;IACI;;;OAGG;IACH,qBAFS,GAAG,CAuBV;IAEF;;;OAGG;IACH,qBAHW,mBAAmB,EA+G7B;IA/EG,6BAA6B;IAC7B,
|
|
1
|
+
{"version":3,"file":"basic_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":"AA8CA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH;IACI;;;OAGG;IACH,qBAFS,GAAG,CAuBV;IAEF;;;OAGG;IACH,qBAHW,mBAAmB,EA+G7B;IA/EG,6BAA6B;IAC7B,SAAc;IACd,qCAA8B;IAC9B,uCAAgC;IAChC,mCAAgD;IAChD,sDAAuD;IACvD,mCAAiE;IACjE,6DAA8D;IAC9D,0BAA0C;IAC1C,6BAA6B;IAC7B,kBAA6F;IAC7F,iCAAiC;IACjC,oBADW,kBAAkB,CAI5B;IACD,wBAAoC;IACpC,gCAAiC;IAuDjC,2BAA2G;IAE3G,uCAA4B;IAOhC;;;;OAIG;IACH,OAFY,QAAQ,IAAI,CAAC,CAuBxB;IAED;;;;;OAKG;IACH,mBALY,QAAQ,IAAI,CAAC,CAkBxB;IATG,2CAAoE;IAGhE,4EAAiE;IAQzE;;;;;;OAMG;IACH,kDANW,eAAe,GACd,QAAQ,IAAI,CAAC,CAOxB;IAED;;;;OAIG;IACH,6CAmCC;IAED;;;;;;;OAOG;IACH,uDAoBC;IAED;;;;;;OAMG;IACH,kCAMC;IAED;;;;;;;OAOG;IACH,4CA6DC;IAED;;;;;;OAMG;IACH,mDAOC;IAED;;;;;;OAMG;IACH,yDAUC;IAED;;;;;;;;;;OAUG;IACH,8CATW,KAAK;QAEoB,OAAO,EAAhC,OAAO;eACP,CAAC,WAAW,GAAC,YAAY,CAAC,GACzB,QAAQ,IAAI,CAAC,CA4BxB;IAED;;;;;;;;OAQG;IACH;QAPkC,KAAK,EAA5B,KAAK;QACoB,OAAO,EAAhC,OAAO;QACN,QAAQ,IAAI,CAAC,CAgBxB;IAED;;;;;;;;;;;;OAYG;IACH,sCALY,QAAQ,IAAI,CAAC,CAWxB;IAED;;;;;;OAMG;IACH,+BANW,UAAW,+BAYrB;IAED;;;OAGG;IACH,0BAIC;CACJ;;QA3pBa,MAAM;aACN,OAAO;aACP,OAAO;eACP,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BAmBT,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qCAyoBhB,mBAAmB,KACjB,QAAQ,IAAI,CAAC;;;;;;;;;;aAIZ,OAAO;;;;2CAYV,wBAAwB,KACtB,QAAQ,IAAI,CAAC;;;;;WAKZ,KAAK;;;;aACL,OAAO;aACP,OAAO;eACP,SAAS"}
|
|
@@ -6,11 +6,11 @@ const consts_1 = require("@apify/consts");
|
|
|
6
6
|
const utilities_1 = require("@apify/utilities");
|
|
7
7
|
const ow_1 = (0, tslib_1.__importStar)(require("ow"));
|
|
8
8
|
const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
|
|
9
|
+
const timeout_1 = require("@apify/timeout");
|
|
9
10
|
const autoscaled_pool_1 = (0, tslib_1.__importDefault)(require("../autoscaling/autoscaled_pool")); // eslint-disable-line import/no-duplicates
|
|
10
11
|
const events_1 = (0, tslib_1.__importDefault)(require("../events"));
|
|
11
12
|
const session_pool_1 = require("../session_pool/session_pool"); // eslint-disable-line import/no-duplicates
|
|
12
13
|
const statistics_1 = (0, tslib_1.__importDefault)(require("./statistics"));
|
|
13
|
-
const utils_1 = require("../utils");
|
|
14
14
|
const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
|
|
15
15
|
const validators_1 = require("../validators");
|
|
16
16
|
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
|
|
@@ -400,11 +400,11 @@ class BasicCrawler {
|
|
|
400
400
|
* Wrapper around _runTaskFunction to set a timeout on the entire function.
|
|
401
401
|
* Temporary solution to see whether this causes the 0 concurrency bug.
|
|
402
402
|
* @ignore
|
|
403
|
-
* @
|
|
404
|
-
* @
|
|
403
|
+
* @internal
|
|
404
|
+
* @private
|
|
405
405
|
*/
|
|
406
406
|
async timeoutForRunTaskFunction() {
|
|
407
|
-
await (0,
|
|
407
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this._runTaskFunction(), this.handleRequestTimeoutMillis * 2, `_runTaskFunction timed out after ${(this.handleRequestTimeoutMillis * 2) / 1000} seconds.`);
|
|
408
408
|
}
|
|
409
409
|
/**
|
|
410
410
|
* Wrapper around handleRequestFunction that fetches requests from RequestList/RequestQueue
|
|
@@ -424,6 +424,7 @@ class BasicCrawler {
|
|
|
424
424
|
else {
|
|
425
425
|
request = await this._fetchNextRequest();
|
|
426
426
|
}
|
|
427
|
+
(0, timeout_1.tryCancel)();
|
|
427
428
|
if (!request)
|
|
428
429
|
return;
|
|
429
430
|
// Reset loadedUrl so an old one is not carried over to retries.
|
|
@@ -439,8 +440,10 @@ class BasicCrawler {
|
|
|
439
440
|
};
|
|
440
441
|
this.crawlingContexts.set(crawlingContext.id, crawlingContext);
|
|
441
442
|
try {
|
|
442
|
-
await (0,
|
|
443
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this._handleRequestFunction(crawlingContext), this.handleRequestTimeoutMillis, `handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`);
|
|
444
|
+
(0, timeout_1.tryCancel)();
|
|
443
445
|
await source.markRequestHandled(request);
|
|
446
|
+
(0, timeout_1.tryCancel)();
|
|
444
447
|
this.stats.finishJob(statisticsId);
|
|
445
448
|
this.handledRequestsCount++;
|
|
446
449
|
// reclaim session if request finishes successfully
|
|
@@ -463,6 +466,7 @@ class BasicCrawler {
|
|
|
463
466
|
finally {
|
|
464
467
|
this.crawlingContexts.delete(crawlingContext.id);
|
|
465
468
|
}
|
|
469
|
+
(0, timeout_1.tryCancel)();
|
|
466
470
|
}
|
|
467
471
|
/**
|
|
468
472
|
* Returns true if either RequestList or RequestQueue have a request ready for processing.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic_crawler.js","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":";;;;AAAA,0CAAkD;AAClD,gDAAwD;AACxD,sDAAuC;AACvC,yEAA2B;AAC3B,kGAA4D,CAAC,2CAA2C;AACxG,oEAA+B;AAC/B,+DAA+D,CAAC,2CAA2C;AAC3G,2EAAsC;AACtC,oCAA+C;AAC/C,0EAAsC;AACtC,8CAA2C;AAY3C,iFAAiF;AAEjF;;;;;;;GAOG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH,MAAa,YAAY;IA4BrB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,qBAAqB,EACrB,wBAAwB,GAAG,EAAE,EAC7B,2BAA2B,EAC3B,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc;QAEd,WAAW;QACX,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,GAC5D,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE;YAC/B,MAAM,GAAG,GAAG,mGAAmG,CAAC;YAChH,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SAClD;QAED,6DAA6D;QAC7D,6BAA6B;QAC7B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,2BAA2B,CAAC;QACxD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,GAAG,IAAI,CAAC;QAClE,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,CAAC,CAAC;QAC9F,iCAAiC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,EAAE;YACpD,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,0BAA0B,EAAE;kBACrE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,CAAC;SACtD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAG;YAC5C,cAAc;YACd,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1D,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,oBAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,uCAAuC,EAAE,qBAAqB,CAAC,CAAC;QAE5G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,wEAAwE;QACxE,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1E,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC;QAExD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,gBAAgB,CAAC;SAC/B;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,gBAAgB;gBAChB,cAAc;gBACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBAChD,GAAG,UAAU;aAChB,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,IAAA,8BAAe,EAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,gHAAgH;8BAChH,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,+CAA+C;8BACjE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE;YACpC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC;SAC/C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,yBAAyB;QAC3B,MAAM,IAAA,2BAAmB,EACrB,IAAI,CAAC,gBAAgB,EAAE,EACvB,IAAI,CAAC,0BAA0B,GAAG,CAAC,EACnC,oCAAoC,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,CAAC,GAAG,IAAI,WAAW,CAC9F,CAAC;IACN,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAErD,IAAI,OAAO,CAAC;QACZ,IAAI,OAAO,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SACrG;aAAM;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;SAC5C;QAED,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,MAAM,eAAe,GAAG;YACpB,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,OAAO;YACP,OAAO;SACV,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,2BAAmB,EACrB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,EAC5C,IAAI,CAAC,0BAA0B,EAC/B,yCAAyC,IAAI,CAAC,0BAA0B,GAAG,IAAI,WAAW,CAC7F,CAAC;YACF,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;aACzE;YAAC,OAAO,cAAc,EAAE;gBACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,oDAAoD;sBACjF,6GAA6G;sBAC7G,qGAAqG;sBACrG,4GAA4G;sBAC5G,sGAAsG,CAAC,CAAC;gBAC9G,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB;QACtB,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,0BAA0B;QAC5B,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,4BAA4B,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC3F,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,KAAK,EACL,mFAAmF,EACnF,oBAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9C,eAAe,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,MAAM,IAAI,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC,CAAC,yCAAyC;SACtG;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,4BAA4B,CAAC,eAAe;QAC9C,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,eAAe,CAAC,KAAK,EACrB,4CAA4C,EAC5C,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;SACL;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,wBAAwB;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,IAAI;QAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;SACrC;IACL,CAAC;;AA3eL,oCA4eC;AA3eG;;;GAGG;AACI;;;;WAAe;QAClB,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,qBAAqB,EAAE,YAAE,CAAC,QAAQ;QAClC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAElC,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC;AAodN;;;;GAIG;AACH;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;GAMG"}
|
|
1
|
+
{"version":3,"file":"basic_crawler.js","sourceRoot":"","sources":["../../src/crawlers/basic_crawler.js"],"names":[],"mappings":";;;;AAAA,0CAAkD;AAClD,gDAAwD;AACxD,sDAAuC;AACvC,yEAA2B;AAC3B,4CAAgE;AAChE,kGAA4D,CAAC,2CAA2C;AACxG,oEAA+B;AAC/B,+DAA+D,CAAC,2CAA2C;AAC3G,2EAAsC;AACtC,0EAAsC;AACtC,8CAA2C;AAY3C,iFAAiF;AAEjF;;;;;;;GAOG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyEG;AACH,MAAa,YAAY;IA4BrB;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,qBAAqB,EACrB,wBAAwB,GAAG,EAAE,EAC7B,2BAA2B,EAC3B,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc;QAEd,WAAW;QACX,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,GAC5D,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,IAAI,CAAC,YAAY,EAAE;YAC/B,MAAM,GAAG,GAAG,mGAAmG,CAAC;YAChH,MAAM,IAAI,kBAAa,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SAClD;QAED,6DAA6D;QAC7D,6BAA6B;QAC7B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,mBAAmB,GAAG,qBAAqB,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,2BAA2B,CAAC;QACxD,IAAI,CAAC,0BAA0B,GAAG,wBAAwB,GAAG,IAAI,CAAC;QAClE,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,oBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,CAAC,CAAC;QAC9F,iCAAiC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,EAAE;YACpD,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,0BAA0B,EAAE;kBACrE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,0BAA0B,GAAG,gBAAgB,CAAC;SACtD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAG;YAC5C,cAAc;YACd,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1D,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,oBAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,uCAAuC,EAAE,qBAAqB,CAAC,CAAC;QAE5G,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,wEAAwE;QACxE,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1E,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC;QAExD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,gBAAgB,CAAC;SAC/B;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,gBAAgB;gBAChB,cAAc;gBACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;gBAChD,GAAG,UAAU;aAChB,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,IAAA,8BAAe,EAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,gHAAgH;8BAChH,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,+CAA+C;8BACjE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE;YACpC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC;SAC/C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,yBAAyB;QAC3B,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAC7B,IAAI,CAAC,0BAA0B,GAAG,CAAC,EACnC,oCAAoC,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,CAAC,GAAG,IAAI,WAAW,CAC9F,CAAC;IACN,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAErD,IAAI,OAAO,CAAC;QACZ,IAAI,OAAO,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;SACrG;aAAM;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;SAC5C;QAED,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,MAAM,eAAe,GAAG;YACpB,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,OAAO;YACP,OAAO;SACV,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,EAClD,IAAI,CAAC,0BAA0B,EAC/B,yCAAyC,IAAI,CAAC,0BAA0B,GAAG,IAAI,WAAW,CAC7F,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YACZ,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAA,mBAAS,GAAE,CAAC;YACZ,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAI,CAAC,4BAA4B,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;aACzE;YAAC,OAAO,cAAc,EAAE;gBACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,oDAAoD;sBACjF,6GAA6G;sBAC7G,qGAAqG;sBACrG,4GAA4G;sBAC5G,sGAAsG,CAAC,CAAC;gBAC9G,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;QAED,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB;QACtB,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,0BAA0B;QAC5B,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,4BAA4B,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM;QAC7D,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC3F,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,KAAK,EACL,mFAAmF,EACnF,oBAAC,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9C,eAAe,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,MAAM,IAAI,CAAC,4BAA4B,CAAC,eAAe,CAAC,CAAC,CAAC,yCAAyC;SACtG;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,4BAA4B,CAAC,eAAe;QAC9C,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,eAAe,CAAC,KAAK,EACrB,4CAA4C,EAC5C,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;SACL;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,wBAAwB;QAC1B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,GAAG,IAAI;QAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;SACrC;IACL,CAAC;;AAjfL,oCAkfC;AAjfG;;;GAGG;AACI;;;;WAAe;QAClB,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,qBAAqB,EAAE,YAAE,CAAC,QAAQ;QAClC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAElC,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC;AA0dN;;;;GAIG;AACH;;;;;;;;;;GAUG;AAEH;;;;GAIG;AAEH;;;;;;GAMG"}
|
|
@@ -255,6 +255,8 @@ export default class BrowserCrawler extends BasicCrawler {
|
|
|
255
255
|
retireBrowserAfterPageCount?: number | undefined;
|
|
256
256
|
operationTimeoutSecs?: number | undefined;
|
|
257
257
|
closeInactiveBrowserAfterSecs?: number | undefined;
|
|
258
|
+
useFingerprints?: boolean | undefined;
|
|
259
|
+
fingerprintsOptions?: import("browser-pool").FingerprintsOptions | undefined;
|
|
258
260
|
}, [], never, never, never, never>;
|
|
259
261
|
/**
|
|
260
262
|
* @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G
|
|
1
|
+
{"version":3,"file":"browser_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":"AAwBA;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EAqE/B;IAnCG,8CAA4C;IAC5C,8BAAkD;IAClD,gCAAgE;IAChE,gCAAgF;IAEhF,uCAAgC;IAChC;;MAEC;IAED,mDAA4C;IAE5C,0BAA0B;IAC1B,oBADW,MAAM,IAAI,CAAC,CACsB;IAC5C,0BAA0B;IAC1B,qBADW,MAAM,IAAI,CAAC,CACwB;IAG1C,kCAAwG;IAM5G;;;;;;;;;;uCAUE;IA8EN;;;;;;;OAOG;IACH,+DAPW,sBAAsB,GAAG,eAAe,gCAExC,OAAO,QAsBjB;IAED;;;;;OAKG;IACH,6CALW,sBAAsB,GAAG,eAAe,iBAqBlD;IAED;;;;;;;OAOG;IACH,oDANW,sBAAsB,GAAG,eAAe,SACxC,KAAK,QAWf;IAED;;;;;;OAMG;IACH,8CANW,sBAAsB,GAAG,eAAe;YACjC,MAAM;qBAYvB;IAED;;;;;;;;OAQG;IACH,4CANW,sBAAsB,GAAG,eAAe,GACvC,QAAQ,IAAI,CAAC,CAiBxB;IAED;;;;;;OAMG;IACH,uCANW,MAAM,qCA6BhB;IAED;;;;;;;OAOG;IACH,kDANW,MAAM,qBACN,iBAAiB,QAiB3B;CAUJ;;uBA1jBa,iBAAiB;;qCAIpB,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,QAAQ,IAAI,CAAC;kDAIf,sBAAsB,GAAG,eAAe,KACtC,QAAQ,IAAI,CAAC;qCAIf,sBAAsB,GAAG,eAAe;QACjC,MAAM;MACX,YAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAKT,yBAAyB"}
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
|
|
5
|
+
const timeout_1 = require("@apify/timeout");
|
|
5
6
|
const browser_pool_1 = require("browser-pool"); // eslint-disable-line import/no-duplicates,no-unused-vars
|
|
6
7
|
const constants_1 = require("../constants");
|
|
7
8
|
const events_1 = (0, tslib_1.__importDefault)(require("../session_pool/events")); // eslint-disable-line import/no-duplicates
|
|
8
|
-
const utils_1 = require("../utils");
|
|
9
9
|
const validators_1 = require("../validators");
|
|
10
10
|
const crawler_utils_1 = require("./crawler_utils");
|
|
11
11
|
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
|
|
@@ -325,23 +325,29 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
325
325
|
}
|
|
326
326
|
}
|
|
327
327
|
const page = await this.browserPool.newPage(newPageOptions);
|
|
328
|
+
(0, timeout_1.tryCancel)();
|
|
328
329
|
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
|
|
329
330
|
if (this.useSessionPool) {
|
|
330
331
|
const sessionCookies = session.getPuppeteerCookies(request.url);
|
|
331
332
|
if (sessionCookies.length) {
|
|
332
333
|
await crawlingContext.browserController.setCookies(page, sessionCookies);
|
|
334
|
+
(0, timeout_1.tryCancel)();
|
|
333
335
|
}
|
|
334
336
|
}
|
|
335
337
|
try {
|
|
336
338
|
await this._handleNavigation(crawlingContext);
|
|
339
|
+
(0, timeout_1.tryCancel)();
|
|
337
340
|
await this._responseHandler(crawlingContext);
|
|
341
|
+
(0, timeout_1.tryCancel)();
|
|
338
342
|
// save cookies
|
|
339
343
|
// @TODO: Should we save the cookies also after/only the handle page?
|
|
340
344
|
if (this.persistCookiesPerSession) {
|
|
341
345
|
const cookies = await crawlingContext.browserController.getCookies(page);
|
|
346
|
+
(0, timeout_1.tryCancel)();
|
|
342
347
|
session.setPuppeteerCookies(cookies, request.loadedUrl);
|
|
343
348
|
}
|
|
344
|
-
await (0,
|
|
349
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this.handlePageFunction(crawlingContext), this.handlePageTimeoutMillis, `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
|
|
350
|
+
(0, timeout_1.tryCancel)();
|
|
345
351
|
if (session)
|
|
346
352
|
session.markGood();
|
|
347
353
|
}
|
|
@@ -382,6 +388,7 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
382
388
|
/** @type {*} */
|
|
383
389
|
const gotoOptions = { ...this.defaultGotoOptions };
|
|
384
390
|
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
|
|
391
|
+
(0, timeout_1.tryCancel)();
|
|
385
392
|
try {
|
|
386
393
|
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
|
|
387
394
|
}
|
|
@@ -389,6 +396,7 @@ class BrowserCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
389
396
|
this._handleNavigationTimeout(crawlingContext, error);
|
|
390
397
|
throw error;
|
|
391
398
|
}
|
|
399
|
+
(0, timeout_1.tryCancel)();
|
|
392
400
|
await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
|
|
393
401
|
}
|
|
394
402
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,
|
|
1
|
+
{"version":3,"file":"browser_crawler.js","sourceRoot":"","sources":["../../src/crawlers/browser_crawler.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,4CAAgE;AAChE,+CAA8D,CAAC,0DAA0D;AACzH,4CAAiE;AACjE,iFAA4C,CAAC,2CAA2C;AACxF,8CAA2C;AAC3C,mDAGyB;AAEzB,kFAAkF;AAClF,mDAAgE;AAUhE,iFAAiF;AAEjF;;;GAGG;AACH;;;;;GAKG;AACH;;;;GAIG;AACH;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmJG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AACH,MAAqB,cAAe,SAAQ,4BAAY;IA0BpD;;;OAGG;IACH,YAAY,OAAO;QACf,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,kBAAkB,EAClB,qBAAqB,GAAG,EAAE,EAC1B,qBAAqB,GAAG,EAAE,EAC1B,YAAY,EAAE,aAAa;QAC3B,eAAe,EAAE,aAAa;QAC9B,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,qBAAqB,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,IAAI,CAAC;YACxE,wBAAwB,EAAE,qBAAqB,GAAG,qBAAqB,GAAG,6CAAiC;SAC9G,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,eAAe,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,8EAA8E,CAAC,CAAC;SACvG;QAED,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QACjE,IAAI,CAAC,uBAAuB,GAAG,CAAC,eAAe,IAAI,qBAAqB,CAAC,GAAG,IAAI,CAAC;QAEjF,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,kBAAkB,GAAG;YACtB,OAAO,EAAE,IAAI,CAAC,uBAAuB;SACxC,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,0BAA0B;QAC1B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,0BAA0B;QAC1B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAAC;YAC/B,GAAG,IAAI;YACP,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,sBAAsB,CAAC,eAAe;QACxC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,MAAM,cAAc,GAAG;YACnB,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC;QACrF,IAAI,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9E,eAAe,CAAC,OAAO,GAAG,OAAO,CAAC;YAClC,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YAExC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;iBAC1B,CAAC;aACL;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,cAAc,CAAC,MAAM,EAAE;gBACvB,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;aACf;SACJ;QAED,IAAI;YACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,IAAA,mBAAS,GAAE,CAAC;YAEZ,eAAe;YACf,qEAAqE;YACrE,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,IAAA,mBAAS,GAAE,CAAC;gBACZ,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aAC3D;YAED,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EAC9C,IAAI,CAAC,uBAAuB,EAC5B,sCAAsC,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CACvF,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SACxF;IACL,CAAC;IAED;;;;;;;OAOG;IACH,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB;QACxE,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAI,CAAC,CAAC;QACpF,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,iBAAiB,EAAE;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAO,CAAC;SAC7E;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC5B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAS,CAAC;SACjF;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,iBAAiB,CAAC,eAAe;QACnC,gBAAgB;QAChB,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACnD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;SAC1F;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,MAAM,KAAK,CAAC;SACf;QAED,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;OAOG;IACH,wBAAwB,CAAC,eAAe,EAAE,KAAK;QAC3C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,oCAAoB,EAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;SAChD;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW;QACjD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;YACpB,oDAAoD;YACpD,oFAAoF;YACpF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;SACvE;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,gBAAgB,CAAC,eAAe;QAClC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,EAAE;YAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAA,qCAAqB,EAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACrD;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAM,EAAE,aAAa;QAC5C,MAAM,oBAAoB,GAAG,EAAE,CAAC;QAEhC,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACxH,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,aAAa,CAAC,aAAa,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxD;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;OAOG;IACH,+BAA+B,CAAC,MAAM,EAAE,iBAAiB;QACrD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAO,EAAE,EAAE;gBACzB,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAK,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE;oBACzC,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,iBAAiB,CAAC,CAAC;iBAC/D;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YACtD,iBAAiB,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,gBAAM,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AApVL,iCAqVC;AApVG;;;GAGG;AACI;;;;WAAe;QAClB,GAAG,4BAAY,CAAC,YAAY;QAC5B,gDAAgD;QAChD,qBAAqB,EAAE,YAAE,CAAC,SAAS;QAEnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAElC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAClD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC"}
|
|
@@ -182,7 +182,7 @@ export type CheerioCrawlerOptions = {
|
|
|
182
182
|
* ]
|
|
183
183
|
* ```
|
|
184
184
|
*/
|
|
185
|
-
preNavigationHooks?:
|
|
185
|
+
preNavigationHooks?: any[] | undefined;
|
|
186
186
|
/**
|
|
187
187
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
188
188
|
* The function accepts `crawlingContext` as the only parameter.
|
|
@@ -195,7 +195,7 @@ export type CheerioCrawlerOptions = {
|
|
|
195
195
|
* ]
|
|
196
196
|
* ```
|
|
197
197
|
*/
|
|
198
|
-
postNavigationHooks?:
|
|
198
|
+
postNavigationHooks?: any[] | undefined;
|
|
199
199
|
/**
|
|
200
200
|
* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"
|
|
201
201
|
* target="_blank">MIME types</a> you want the crawler to load and process.
|
|
@@ -310,7 +310,7 @@ export type CheerioHandlePageInputs = {
|
|
|
310
310
|
/**
|
|
311
311
|
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
|
|
312
312
|
*/
|
|
313
|
-
$:
|
|
313
|
+
$: any;
|
|
314
314
|
/**
|
|
315
315
|
* The request body of the web page.
|
|
316
316
|
*/
|
|
@@ -702,12 +702,12 @@ declare class CheerioCrawler extends BasicCrawler {
|
|
|
702
702
|
* @type {Array<Hook>}
|
|
703
703
|
* @ignore
|
|
704
704
|
* */
|
|
705
|
-
preNavigationHooks: Array<
|
|
705
|
+
preNavigationHooks: Array<any>;
|
|
706
706
|
/**
|
|
707
707
|
* @type {Array<Hook>}
|
|
708
708
|
* @ignore
|
|
709
709
|
* */
|
|
710
|
-
postNavigationHooks: Array<
|
|
710
|
+
postNavigationHooks: Array<any>;
|
|
711
711
|
persistCookiesPerSession: boolean;
|
|
712
712
|
/**
|
|
713
713
|
* **EXPERIMENTAL**
|
|
@@ -757,7 +757,7 @@ declare class CheerioCrawler extends BasicCrawler {
|
|
|
757
757
|
request: Request;
|
|
758
758
|
session: Session;
|
|
759
759
|
proxyUrl: string;
|
|
760
|
-
requestAsBrowserOptions:
|
|
760
|
+
requestAsBrowserOptions: any;
|
|
761
761
|
}): Promise<IncomingMessage | Readable>;
|
|
762
762
|
/**
|
|
763
763
|
* Sets the cookie header to `requestAsBrowserOptions` based on provided session and request. If some cookies were already set,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"cheerio_crawler.d.ts","sourceRoot":"","sources":["../../src/crawlers/cheerio_crawler.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBAkDc,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;aAu3BjB,OAAO;;;;;;;;;;;;sCAYV,oBAAoB,KAClB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;cAKnB,CAAC,eAAe,GAAC,QAAQ,CAAC;;;;aAC1B,OAAO;;;;;;;;;;aAON,cAAc;;oCAKlB,kBAAkB,KAChB,CAAC,IAAI,GAAC,QAAQ,IAAI,CAAC,CAAC;;;;;;;;;UAOnB,CAAC,MAAM,GAAC,MAAM,CAAC;;;;;;;;aAIf,OAAO;;;;iBAEP;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;;;;cAElC,eAAe;aAEf,OAAO;;;;;eACP,SAAS;aAGT,cAAc;;yCAKjB,uBAAuB,KACrB,QAAQ,IAAI,CAAC;AAp7B1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkOG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgHG;AACH;IA0BI;;;OAGG;IACH,qBAHW,qBAAqB,EA2E/B;IAlCG,gCAA2F;IAO3F,gCAA2D;IAC3D,6BAAqD;IACrD,yBAAsC;IACtC,4CAAsD;IACtD,0CAAkD;IAClD,mDAAoD;IACpD,+CAAgD;IAChD,mDAA4C;IAC5C;;;SAGK;IACL,oBAHU,UAAW,CAGuB;IAC5C;;;SAGK;IACL,qBAHU,UAAW,CAMpB;IAGG,kCAAwG;IAMhH;;;;OAIG;IACH,eAFW,gBAAgB,QA+B1B;IAyED;;;;;OAKG;IACH,6CALW,eAAe,iBAwCzB;IAED;;;;;;;;;;;;;;OAcG;IACH,gCASC;IAED;;;;;;;;;;;;;;OAcG;IACH;QAT4B,OAAO,EAAxB,OAAO;QACU,OAAO,EAAxB,OAAO;QACS,QAAQ,EAAxB,MAAM;QAC2B,uBAAuB;QACtD,QAAQ,eAAe,GAAC,QAAQ,CAAC,CAoB7C;IAED;;;;;;;;;;OAUG;IACH,4BAUC;IAED;;;;;;;;OAQG;IACH,kCAPW,OAAO,kBACP,eAAe,GAAC,QAAQ,GACtB,QAAQ,MAAM,CAAC,CAgC3B;IAED;;;;;;;;;OASG;IACH,sCARW,OAAO,oGAsCjB;IAED;;;;;;;OAOG;IACH;;;MA+BC;IAED;;;;;OAKG;IACH,uDAYC;IAED;;;;;;OAMG;IACH,yDALW,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC,QAchC;IAED;;;;;;;OAOG;IACH,0CANW,OAAO,cACP,MAAM,QAWhB;IAED;;;;;;OAMG;IACH,yCALW,OAAO,QAQjB;IAED;;;;OAIG;IACH,6BAcC;CACJ"}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const tslib_1 = require("tslib");
|
|
4
4
|
/* eslint-disable class-methods-use-this */
|
|
5
5
|
const utilities_1 = require("@apify/utilities");
|
|
6
|
+
const timeout_1 = require("@apify/timeout");
|
|
6
7
|
const cheerio_1 = (0, tslib_1.__importDefault)(require("cheerio"));
|
|
7
8
|
const content_type_1 = (0, tslib_1.__importDefault)(require("content-type"));
|
|
8
9
|
const htmlparser2_1 = require("htmlparser2");
|
|
@@ -472,7 +473,9 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
472
473
|
crawlingContext.proxyInfo = this.proxyConfiguration.newProxyInfo(sessionId);
|
|
473
474
|
}
|
|
474
475
|
await this._handleNavigation(crawlingContext);
|
|
476
|
+
(0, timeout_1.tryCancel)();
|
|
475
477
|
const { dom, isXml, body, contentType, response } = await this._parseResponse(request, crawlingContext.response);
|
|
478
|
+
(0, timeout_1.tryCancel)();
|
|
476
479
|
if (this.useSessionPool) {
|
|
477
480
|
this._throwOnBlockedRequest(session, response.statusCode);
|
|
478
481
|
}
|
|
@@ -511,7 +514,7 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
511
514
|
return body;
|
|
512
515
|
},
|
|
513
516
|
});
|
|
514
|
-
return (0,
|
|
517
|
+
return (0, timeout_1.addTimeoutToPromise)(() => this.userProvidedHandler(crawlingContext), this.handlePageTimeoutMillis, `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
|
|
515
518
|
}
|
|
516
519
|
/**
|
|
517
520
|
* @param {CrawlingContext} crawlingContext
|
|
@@ -523,6 +526,7 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
523
526
|
if (this.prepareRequestFunction) {
|
|
524
527
|
this.log.deprecated('Option "prepareRequestFunction" is deprecated. Use "preNavigationHooks" instead.');
|
|
525
528
|
await this.prepareRequestFunction(crawlingContext);
|
|
529
|
+
(0, timeout_1.tryCancel)();
|
|
526
530
|
}
|
|
527
531
|
const requestAsBrowserOptions = {};
|
|
528
532
|
if (this.useSessionPool) {
|
|
@@ -531,13 +535,17 @@ class CheerioCrawler extends basic_crawler_1.BasicCrawler {
|
|
|
531
535
|
const { request, session } = crawlingContext;
|
|
532
536
|
const cookieSnapshot = request.headers.Cookie ?? request.headers.cookie;
|
|
533
537
|
await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);
|
|
538
|
+
(0, timeout_1.tryCancel)();
|
|
534
539
|
const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;
|
|
535
540
|
this._mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions);
|
|
536
|
-
crawlingContext.response = await (0,
|
|
541
|
+
crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }), this.requestTimeoutMillis, `request timed out after ${this.requestTimeoutMillis / 1000} seconds.`);
|
|
542
|
+
(0, timeout_1.tryCancel)();
|
|
537
543
|
await this._executeHooks(this.postNavigationHooks, crawlingContext, requestAsBrowserOptions);
|
|
544
|
+
(0, timeout_1.tryCancel)();
|
|
538
545
|
if (this.postResponseFunction) {
|
|
539
546
|
this.log.deprecated('Option "postResponseFunction" is deprecated. Use "postNavigationHooks" instead.');
|
|
540
547
|
await this.postResponseFunction(crawlingContext);
|
|
548
|
+
(0, timeout_1.tryCancel)();
|
|
541
549
|
}
|
|
542
550
|
}
|
|
543
551
|
/**
|