@crawlee/playwright 4.0.0-beta.13 → 4.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/internals/adaptive-playwright-crawler.d.ts +14 -8
- package/internals/adaptive-playwright-crawler.d.ts.map +1 -1
- package/internals/adaptive-playwright-crawler.js +61 -19
- package/internals/adaptive-playwright-crawler.js.map +1 -1
- package/internals/enqueue-links/click-elements.d.ts +10 -0
- package/internals/enqueue-links/click-elements.d.ts.map +1 -1
- package/internals/enqueue-links/click-elements.js +14 -2
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/playwright-crawler.d.ts +3 -1
- package/internals/playwright-crawler.d.ts.map +1 -1
- package/internals/utils/playwright-utils.d.ts +9 -0
- package/internals/utils/playwright-utils.d.ts.map +1 -1
- package/internals/utils/playwright-utils.js +35 -3
- package/internals/utils/playwright-utils.js.map +1 -1
- package/internals/utils/rendering-type-prediction.d.ts +8 -3
- package/internals/utils/rendering-type-prediction.d.ts.map +1 -1
- package/internals/utils/rendering-type-prediction.js +22 -10
- package/internals/utils/rendering-type-prediction.js.map +1 -1
- package/package.json +12 -8
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
+
<p align=center>
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
12
16
|
<p align=center>
|
|
13
17
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
18
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { BasicCrawler } from '@crawlee/basic';
|
|
2
2
|
import type { BasicCrawlerOptions, BrowserHook, LoadedRequest, Request } from '@crawlee/browser';
|
|
3
|
-
import type { CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
|
|
3
|
+
import type { CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
|
|
4
4
|
import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
|
|
5
|
-
import type { Dictionary } from '@crawlee/types';
|
|
5
|
+
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
6
6
|
import { type CheerioRoot } from '@crawlee/utils';
|
|
7
7
|
import { type Cheerio } from 'cheerio';
|
|
8
8
|
import type { AnyNode } from 'domhandler';
|
|
9
9
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
10
10
|
import type { Page } from 'playwright';
|
|
11
|
-
import type {
|
|
11
|
+
import type { SetRequired } from 'type-fest';
|
|
12
|
+
import type { PlaywrightGotoOptions } from './playwright-crawler.js';
|
|
12
13
|
import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
|
|
13
14
|
interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
|
|
14
15
|
httpOnlyRequestHandlerRuns?: number;
|
|
@@ -99,15 +100,18 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
|
|
|
99
100
|
resultChecker?: (result: RequestHandlerResult) => boolean;
|
|
100
101
|
/**
|
|
101
102
|
* An optional callback used in rendering type detection. On each detection, the result of the plain HTTP run is compared to that of the browser one.
|
|
102
|
-
* If
|
|
103
|
+
* If a callback is provided, the contract is as follows:
|
|
104
|
+
* It the callback returns true or 'equal', the results are considered equal and the target site is considered static.
|
|
105
|
+
* If it returns false or 'different', the target site is considered client-rendered.
|
|
106
|
+
* If it returns 'inconclusive', the detection result won't be used.
|
|
103
107
|
* If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
|
|
104
108
|
* If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
|
|
105
109
|
*/
|
|
106
|
-
resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean;
|
|
110
|
+
resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
|
|
107
111
|
/**
|
|
108
112
|
* A custom rendering type predictor
|
|
109
113
|
*/
|
|
110
|
-
renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult'>;
|
|
114
|
+
renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;
|
|
111
115
|
/**
|
|
112
116
|
* Prevent direct access to storage in request handlers (only allow using context helpers).
|
|
113
117
|
* Defaults to `true`
|
|
@@ -156,12 +160,14 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
|
|
|
156
160
|
private resultObjects;
|
|
157
161
|
private teardownHooks;
|
|
158
162
|
constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>, config?: Configuration);
|
|
163
|
+
protected _init(): Promise<void>;
|
|
159
164
|
private adaptCheerioContext;
|
|
160
165
|
private adaptPlaywrightContext;
|
|
161
166
|
private crawlOne;
|
|
162
|
-
protected runRequestHandler(crawlingContext:
|
|
163
|
-
protected commitResult(crawlingContext:
|
|
167
|
+
protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
|
|
168
|
+
protected commitResult(crawlingContext: CrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
|
|
164
169
|
protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
|
|
170
|
+
protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
|
|
165
171
|
private createLogProxy;
|
|
166
172
|
teardown(): Promise<void>;
|
|
167
173
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAIjG,OAAO,KAAK,EAER,eAAe,EACf,mBAAmB,EACnB,sBAAsB,
|
|
1
|
+
{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAIjG,OAAO,KAAK,EAER,eAAe,EACf,mBAAmB,EACnB,sBAAsB,EAEtB,yBAAyB,EACzB,YAAY,EAEZ,iBAAiB,EACjB,cAAc,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,aAAa,EAEb,oBAAoB,EAGpB,UAAU,EAEb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAK7C,OAAO,KAAK,EAA6B,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEhG,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAMlG,UAAU,uCAAwC,SAAQ,cAAc;IACpE,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACxC;AAQD,cAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,EAAE,uCAAuC,CAAe;gBAE1D,OAAO,GAAE,iBAAsB;IAKlC,KAAK,IAAI,IAAI;cAOG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9D,8BAA8B,IAAI,IAAI;IAKtC,6BAA6B,IAAI,IAAI;IAKrC,+BAA+B,IAAI,IAAI;CAI1C;AAED,MAAM,WAAW,gCAAgC,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtF,SAAQ,eAAe,CAAC,QAAQ,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAE/E;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,UAAU,YACN,SAAQ,WAAW,CACf,IAAI,CAAC,gCAAgC,EAAE,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,KAAK,CAAC,GAAG;IAC7E,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;CACpB,EACD,qBAAqB,CACxB;CAAG;AAER,MAAM,WAAW,gCAAgC,CAC7C,eAAe,SAAS,gCAAgC,GAAG,gCAAgC,CAC7F,SAAQ,IAAI,CACN,mBAAmB,CAAC,gCAAgC,EAAE,eAAe,CAAC,EACtE,oBAAoB,GAAG,qBAAqB,CAC/C;IACD;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,YAAY,EAAE,CAAC;IAEpC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,YAAY,EAAE,CAAC;IAErC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE1D;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,CACf,OAAO,EAAE,oBAAoB,EAC7B,OAAO,EAAE,oBAAoB,KAC5B,OAAO,GAAG,OAAO,GAAG,WAAW,GAAG,cAAc,CAAC;IAEtD;;OAEG;IACH,sBAAsB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,aAAa,GAAG,YAAY,CAAC,CAAC;IAEhG;;;OAGG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACxC;AAeD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAAyB,CAClC,eAAe,SAAS,gCAAgC,GAAG,gCAAgC,CAC7F,SAAQ,YAAY,CAAC,gCAAgC,EAAE,eAAe,CAAC;aAe/C,MAAM;IAd5B,OAAO,CAAC,sBAAsB,CAA0E;IACxG,OAAO,CAAC,aAAa,CAAiE;IACtF,OAAO,CAAC,gBAAgB,CAAoE;IAC5F,OAAO,CAAC,0BAA0B,CAAU;IAC5C,OAAO,CAAC,qBAAqB,CAAoD;IACjF,OAAO,CAAC,sBAAsB,CAAoD;IAClF,OAAO,CAAC,qCAAqC,CAAS;IACtD,SAAiB,KAAK,EAAE,mCAAmC,CAAC;IAC5D,OAAO,CAAC,aAAa,CAAwD;IAE7E,OAAO,CAAC,aAAa,CAAkC;gBAGnD,OAAO,GAAE,gCAAgC,CAAC,eAAe,CAAM,EAC7C,MAAM,gBAAkC;cAuIrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjC,mBAAmB;YAgCnB,sBAAsB;YAiDtB,QAAQ;cA6DG,iBAAiB,CAAC,eAAe,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;cAiH3E,YAAY,CACxB,eAAe,EAAE,eAAe,EAChC,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GACtD,OAAO,CAAC,IAAI,CAAC;IAehB,SAAS,CAAC,kBAAkB,CAAC,CAAC,EAAE,KAAK,SAAS,GAAG,EAAE,EAC/C,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GACrC,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;cAQjB,YAAY,CACxB,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EACjD,OAAO,EAAE,yBAAyB,CAAC,SAAS,CAAC,EAC7C,MAAM,EAAE,oBAAoB,GAC7B,OAAO,CAAC,sBAAsB,CAAC;IA2BlC,OAAO,CAAC,cAAc;IAaP,QAAQ;CAM1B;AAED,wBAAgB,8BAA8B,CAC1C,OAAO,SAAS,gCAAgC,GAAG,gCAAgC,EACnF,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -2,7 +2,7 @@ import { isDeepStrictEqual } from 'node:util';
|
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import { extractUrlsFromPage } from '@crawlee/browser';
|
|
4
4
|
import { CheerioCrawler } from '@crawlee/cheerio';
|
|
5
|
-
import { Configuration, RequestHandlerError, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
|
|
5
|
+
import { Configuration, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
|
|
6
6
|
import { extractUrlsFromCheerio } from '@crawlee/utils';
|
|
7
7
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
8
8
|
import { PlaywrightCrawler } from './playwright-crawler.js';
|
|
@@ -189,9 +189,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
189
189
|
});
|
|
190
190
|
this.preventDirectStorageAccess = preventDirectStorageAccess;
|
|
191
191
|
}
|
|
192
|
+
async _init() {
|
|
193
|
+
await this.renderingTypePredictor.initialize();
|
|
194
|
+
return await super._init();
|
|
195
|
+
}
|
|
192
196
|
async adaptCheerioContext(cheerioContext) {
|
|
193
|
-
|
|
194
|
-
|
|
197
|
+
// Capture the original response to avoid infinite recursion when the getter is copied to the context
|
|
198
|
+
const result = this.resultObjects.get(cheerioContext);
|
|
199
|
+
if (result === undefined) {
|
|
195
200
|
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
196
201
|
}
|
|
197
202
|
return {
|
|
@@ -201,18 +206,18 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
201
206
|
async querySelector(selector) {
|
|
202
207
|
return cheerioContext.$(selector);
|
|
203
208
|
},
|
|
204
|
-
async
|
|
209
|
+
enqueueLinks: async (options = {}) => {
|
|
205
210
|
const urls = options.urls ??
|
|
206
211
|
extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
|
|
207
|
-
await enqueueLinks({ ...options, urls });
|
|
212
|
+
return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result));
|
|
208
213
|
},
|
|
209
214
|
response: cheerioContext.response,
|
|
210
215
|
};
|
|
211
216
|
}
|
|
212
217
|
async adaptPlaywrightContext(playwrightContext) {
|
|
213
218
|
const originalResponse = playwrightContext.response;
|
|
214
|
-
const
|
|
215
|
-
if (
|
|
219
|
+
const result = this.resultObjects.get(playwrightContext);
|
|
220
|
+
if (result === undefined) {
|
|
216
221
|
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
217
222
|
}
|
|
218
223
|
return {
|
|
@@ -227,14 +232,21 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
227
232
|
const $ = await playwrightContext.parseWithCheerio();
|
|
228
233
|
return $(selector);
|
|
229
234
|
},
|
|
230
|
-
async
|
|
231
|
-
const selector = options.selector ?? 'a';
|
|
232
|
-
const locator = playwrightContext.page.locator(selector).first();
|
|
233
|
-
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
235
|
+
enqueueLinks: async (options = {}, timeoutMs = 5000) => {
|
|
234
236
|
// TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
237
|
+
let urls;
|
|
238
|
+
if (options.urls === undefined) {
|
|
239
|
+
const selector = options.selector ?? 'a';
|
|
240
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
241
|
+
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
242
|
+
urls =
|
|
243
|
+
options.urls ??
|
|
244
|
+
(await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
urls = options.urls;
|
|
248
|
+
}
|
|
249
|
+
return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result));
|
|
238
250
|
},
|
|
239
251
|
};
|
|
240
252
|
}
|
|
@@ -247,7 +259,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
247
259
|
pushData: result.pushData,
|
|
248
260
|
useState: this.allowStorageAccess(useStateFunction),
|
|
249
261
|
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
|
|
250
|
-
enqueueLinks:
|
|
262
|
+
enqueueLinks: async (options) => {
|
|
263
|
+
return await this.enqueueLinks(options, context.request, result);
|
|
264
|
+
},
|
|
251
265
|
log: this.createLogProxy(context.log, logs),
|
|
252
266
|
registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
|
|
253
267
|
};
|
|
@@ -338,19 +352,24 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
338
352
|
if (!plainHTTPRun.ok) {
|
|
339
353
|
return 'clientOnly';
|
|
340
354
|
}
|
|
341
|
-
|
|
355
|
+
const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
|
|
356
|
+
if (comparisonResult === true || comparisonResult === 'equal') {
|
|
342
357
|
return 'static';
|
|
343
358
|
}
|
|
344
|
-
|
|
359
|
+
if (comparisonResult === false || comparisonResult === 'different') {
|
|
360
|
+
return 'clientOnly';
|
|
361
|
+
}
|
|
362
|
+
return undefined;
|
|
345
363
|
})();
|
|
346
364
|
crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
|
|
347
|
-
|
|
365
|
+
if (detectionResult !== undefined) {
|
|
366
|
+
this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
|
|
367
|
+
}
|
|
348
368
|
}
|
|
349
369
|
}
|
|
350
370
|
async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
|
|
351
371
|
await Promise.all([
|
|
352
372
|
...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
|
|
353
|
-
...calls.enqueueLinks.map(async (params) => await crawlingContext.enqueueLinks(...params)),
|
|
354
373
|
...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
|
|
355
374
|
...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
|
|
356
375
|
const store = await crawlingContext.getKeyValueStore(storeIdOrName);
|
|
@@ -361,6 +380,29 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
|
361
380
|
allowStorageAccess(func) {
|
|
362
381
|
return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
|
|
363
382
|
}
|
|
383
|
+
async enqueueLinks(options, request, result) {
|
|
384
|
+
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
385
|
+
enqueueStrategy: options?.strategy,
|
|
386
|
+
finalRequestUrl: request.loadedUrl,
|
|
387
|
+
originalRequestUrl: request.url,
|
|
388
|
+
userProvidedBaseUrl: options?.baseUrl,
|
|
389
|
+
});
|
|
390
|
+
const addRequestsBatched = async (requests) => {
|
|
391
|
+
await result.addRequests(requests);
|
|
392
|
+
return {
|
|
393
|
+
addedRequests: requests.map(({ uniqueKey, id }) => ({
|
|
394
|
+
uniqueKey,
|
|
395
|
+
requestId: id ?? '',
|
|
396
|
+
wasAlreadyPresent: false,
|
|
397
|
+
wasAlreadyHandled: false,
|
|
398
|
+
})),
|
|
399
|
+
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
400
|
+
};
|
|
401
|
+
};
|
|
402
|
+
// We need to use a mock request queue implementation, in order to add the requests into our result object
|
|
403
|
+
const mockRequestQueue = { addRequestsBatched };
|
|
404
|
+
return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
|
|
405
|
+
}
|
|
364
406
|
createLogProxy(log, logs) {
|
|
365
407
|
return new Proxy(log, {
|
|
366
408
|
get(target, propertyName, receiver) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adaptive-playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAWlD,OAAO,EACH,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,MAAM,EACN,UAAU,EACV,wBAAwB,GAC3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAoB,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAM1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAkBlG,MAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,GAA4C,IAAW,CAAC,CAAC,+HAA+H;IAEtM,YAAY,UAA6B,EAAE;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAEQ,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC/C,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CACjD,IAAI,CAAC,eAAe,CACvB,CAAC;QAEF,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,UAAU,CAAC,0BAA0B,CAAC;QAC9E,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;IACpF,CAAC;IAED,8BAA8B;QAC1B,IAAI,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,6BAA6B;QACzB,IAAI,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;QAC3B,IAAI,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAmHD,MAAM,eAAe,GAAG;IACpB,OAAO;IACP,WAAW;IACX,UAAU;IACV,MAAM;IACN,OAAO;IACP,MAAM;IACN,aAAa;IACb,YAAY;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,yBAEX,SAAQ,YAA+D;IAe/C;IAdd,sBAAsB,CAA0E;IAChG,aAAa,CAAiE;IAC9E,gBAAgB,CAAoE;IACpF,0BAA0B,CAAU;IACpC,qBAAqB,CAAoD;IACzE,sBAAsB,CAAoD;IAC1E,qCAAqC,CAAS;IAE9C,aAAa,GAAG,IAAI,OAAO,EAAyC,CAAC;IAErE,aAAa,GAA+B,EAAE,CAAC;IAEvD,YACI,UAA6D,EAAE,EAC7C,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,MAAM,EACF,cAAc,EACd,2BAA2B,GAAG,GAAG,EACjC,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,GAAG,IAAI,EACjC,yBAAyB,GAAG,EAAE,EAC9B,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,GAAG,IAAI,EACV,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,IAAI;YACP,uHAAuH;YACvH,YAAY;YACZ,oBAAoB;YACpB,2BAA2B;YAC3B,cAAc;YACd,0JAA0J;YAC1J,gDAAgD;YAChD,sBAAsB,EAAE,GAAG,EAAE,CACzB,IAAqF;SAC5F,EACD,MAAM,CACT,CAAC;QAlCgB,WAAM,GAAN,MAAM,CAAkC;QAmC1D,IAAI,CAAC,qCAAqC,GAAG,yBAAyB,GAAG,IAAI,CAAC;QAE9E,IAAI,CAAC,sBAAsB;YACvB,sBAAsB,IAAI,IAAI,sBAAsB,CAAC,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEnD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC7C,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,OAAO,CACH,OAAO,CAAC,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,MAAM;oBAC3D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;wBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;wBACtC,OAAO,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBAC3C,CAAC,CAAC,CACL,CAAC;YACN,CAAC,CAAC;QACN,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,cAAc,CACpC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,iBAAiB,CACxC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAEtE,IAAI,CAAC,qBAAqB,GAAG,aAAa,CAAC,eAAe;aACrD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9C,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,sBAAsB,GAAG,cAAc,CAAC,eAAe;aACvD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,KAAK,GAAG,IAAI,mCAAmC,CAAC;YACjD,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB;YACjE,MAAM;YACN,GAAG,iBAAiB;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;IACjE,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,cAAsC;QACpE,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC;QAC1E,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACzE,CAAC;YACD,KAAK,CAAC,aAAa,CAAC,QAAgB;gBAChC,OAAO,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YACD,KAAK,CAAC,YAAY,CAAC,UAA+B,EAAE;gBAChD,MAAM,IAAI,GACN,OAAO,CAAC,IAAI;oBACZ,sBAAsB,CAClB,cAAc,CAAC,CAAC,EAChB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,SAAS,CACtD,CAAC;gBACN,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,QAAQ,EAAE,cAAc,CAAC,QAAQ;SACpC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,iBAA4C;QAC7E,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,QAAQ,CAAC;QACpD,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,YAAY,CAAC;QAC7E,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,QAAQ,EAAE,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE;gBACnE,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE;gBACnC,MAAM,EAAE,gBAAgB,CAAC,MAAM,EAAE;gBACjC,UAAU,EAAE,gBAAgB,CAAC,UAAU,EAAE;aAC5C,CAAC;YACF,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,SAAS,GAAG,IAAI;gBAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBACjE,MAAM,CAAC,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;gBAErD,OAAO,CAAC,CAAC,QAAQ,CAAiB,CAAC;YACvC,CAAC;YACD,KAAK,CAAC,YAAY,CAAC,UAA+B,EAAE,EAAE,SAAS,GAAG,IAAI;gBAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;gBACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBAEjE,mGAAmG;gBACnG,MAAM,IAAI,GACN,OAAO,CAAC,IAAI;oBACZ,CAAC,MAAM,mBAAmB,CACtB,iBAAiB,CAAC,IAAI,EACtB,QAAQ,EACR,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CACzD,CAAC,CAAC;gBACP,MAAM,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,QAAQ,CAClB,aAA4B,EAC5B,OAAwB,EACxB,gBAAoE;QAEpE,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAClG,MAAM,IAAI,GAAmB,EAAE,CAAC;QAEhC,MAAM,eAAe,GAA+B,EAAE,CAAC;QAEvD,MAAM,yBAAyB,GAAG;YAC9B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC;YACnD,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAClE,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC;YAC3C,uBAAuB,EAAE,CAAC,OAA+B,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;SAC9F,CAAC;QAEF,MAAM,iBAAiB,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,yBAAyB,EAAE,CAAC;QACvE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,CAAC;YACD,MAAM,0BAA0B,GAAG,KAAK,IAAI,EAAE;gBAC1C,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CACjC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;qBAAM,IAAI,aAAa,KAAK,YAAY,EAAE,CAAC;oBACxC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAClC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CACP,wBAAwB,CAAC,GAAG,EAAE;gBAC1B,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CACX,6FAA6F,CAChG,CAAC;gBACN,CAAC;YACL,CAAC,EAAE,0BAA0B,CAAC,EAClC,IAAI,CAAC,qCAAqC,EAC1C,2BAA2B,CAC9B,CAAC;YAEF,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;gBAAS,CAAC;YACP,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,iBAAiB,CAAC,eAA0C;QACjF,MAAM,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,uBAAuB,CAAC,kCAAkC,CAAC;QAE7G,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAC7B,eAAe,CAAC,GAAG,CAAC,KAAK,CACrB,4BAA4B,uBAAuB,CAAC,aAAa,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACzG,CAAC;QACN,CAAC;QAED,IAAI,uBAAuB,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnF,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,yCAAyC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClG,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC;YAE5C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;YAE9F,IAAI,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAI,IAAmB,CAAC,CAAC,CAAC;gBAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YAED,iFAAiF;YACjF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACnB,MAAM,WAAW,GACb,YAAY,CAAC,KAAK,YAAY,mBAAmB;oBAC7C,CAAC,CAAE,YAAY,CAAC,KAAK,CAAC,KAAe;oBACrC,CAAC,CAAE,YAAY,CAAC,KAAe,CAAC;gBAExC,eAAe,CAAC,GAAG,CAAC,SAAS,CACzB,WAAW,EACX,wCAAwC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACxE,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,GAAG,CAAC,OAAO,CACvB,8DAA8D,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAC9F,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC;QAE3C,qGAAqG;QACrG,oGAAoG;QACpG,oGAAoG;QACpG,wEAAwE;QACxE,MAAM,YAAY,GAAG;YACjB,SAAS,EAAE,IAAI;YACf,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAE3D,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAED,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,OAAO,YAAY,CAAC;gBACxB,CAAC;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC;YAC1B,CAAC;SACJ,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAClC,YAAY,EACZ,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,CAAC,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,yBAAyB,EAAE,CAAC;YAC5B,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACzF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpC,QAAQ,EACR,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;YAEF,MAAM,eAAe,GAAkB,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChE,OAAO,QAAQ,CAAC;gBACpB,CAAC;gBAED,OAAO,YAAY,CAAC;YACxB,CAAC,CAAC,EAAE,CAAC;YAEL,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,eAAe,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC3G,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,eAA0C,EAC1C,EAAE,KAAK,EAAE,oBAAoB,EAAwB;QAErD,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5E,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1F,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC;YAClF,GAAG,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE;gBAC3E,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CACnE,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAC7C,CACJ,CAAC;YACN,CAAC,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IAES,kBAAkB,CACxB,IAAoC;QAEpC,OAAO,KAAK,EAAE,GAAG,IAAW,EAAE,EAAE,CAC5B,wBAAwB,CACpB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAC5B,CAAC;IACV,CAAC;IAEO,cAAc,CAAC,GAAQ,EAAE,IAAoB;QACjD,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE;YAClB,GAAG,CAAC,MAAW,EAAE,YAA8C,EAAE,QAAa;gBAC1E,IAAI,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;wBAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;oBAC/C,CAAC,CAAC;gBACN,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;YACvD,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEQ,KAAK,CAAC,QAAQ;QACnB,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpC,MAAM,IAAI,EAAE,CAAC;QACjB,CAAC;IACL,CAAC;CACJ;AAED,MAAM,UAAU,8BAA8B,CAG5C,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"adaptive-playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAalD,OAAO,EACH,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,sCAAsC,EACtC,MAAM,EACN,UAAU,EACV,wBAAwB,GAC3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAoB,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAO1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAkBlG,MAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,GAA4C,IAAW,CAAC,CAAC,+HAA+H;IAEtM,YAAY,UAA6B,EAAE;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAEQ,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC/C,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CACjD,IAAI,CAAC,eAAe,CACvB,CAAC;QAEF,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,UAAU,CAAC,0BAA0B,CAAC;QAC9E,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;IACpF,CAAC;IAED,8BAA8B;QAC1B,IAAI,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,6BAA6B;QACzB,IAAI,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;QAC3B,IAAI,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAyHD,MAAM,eAAe,GAAG;IACpB,OAAO;IACP,WAAW;IACX,UAAU;IACV,MAAM;IACN,OAAO;IACP,MAAM;IACN,aAAa;IACb,YAAY;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,yBAEX,SAAQ,YAA+D;IAe/C;IAdd,sBAAsB,CAA0E;IAChG,aAAa,CAAiE;IAC9E,gBAAgB,CAAoE;IACpF,0BAA0B,CAAU;IACpC,qBAAqB,CAAoD;IACzE,sBAAsB,CAAoD;IAC1E,qCAAqC,CAAS;IAE9C,aAAa,GAAG,IAAI,OAAO,EAAyC,CAAC;IAErE,aAAa,GAA+B,EAAE,CAAC;IAEvD,YACI,UAA6D,EAAE,EAC7C,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,MAAM,EACF,cAAc,EACd,2BAA2B,GAAG,GAAG,EACjC,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,GAAG,IAAI,EACjC,yBAAyB,GAAG,EAAE,EAC9B,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,GAAG,IAAI,EACV,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,IAAI;YACP,uHAAuH;YACvH,YAAY;YACZ,oBAAoB;YACpB,2BAA2B;YAC3B,cAAc;YACd,0JAA0J;YAC1J,gDAAgD;YAChD,sBAAsB,EAAE,GAAG,EAAE,CACzB,IAAqF;SAC5F,EACD,MAAM,CACT,CAAC;QAlCgB,WAAM,GAAN,MAAM,CAAkC;QAmC1D,IAAI,CAAC,qCAAqC,GAAG,yBAAyB,GAAG,IAAI,CAAC;QAE9E,IAAI,CAAC,sBAAsB;YACvB,sBAAsB,IAAI,IAAI,sBAAsB,CAAC,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEnD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC7C,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,OAAO,CACH,OAAO,CAAC,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,MAAM;oBAC3D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;wBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;wBACtC,OAAO,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBAC3C,CAAC,CAAC,CACL,CAAC;YACN,CAAC,CAAC;QACN,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,cAAc,CACpC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,iBAAiB,CACxC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAEtE,IAAI,CAAC,qBAAqB,GAAG,aAAa,CAAC,eAAe;aACrD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9C,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,sBAAsB,GAAG,cAAc,CAAC,eAAe;aACvD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,KAAK,GAAG,IAAI,mCAAmC,CAAC;YACjD,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB;YACjE,MAAM;YACN,GAAG,iBAAiB;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;IACjE,CAAC;IAEkB,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,cAAsC;QACpE,qGAAqG;QACrG,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACzE,CAAC;YACD,KAAK,CAAC,aAAa,CAAC,QAAgB;gBAChC,OAAO,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,UAA+B,EAAE,EAAE,EAAE;gBACtD,MAAM,IAAI,GACN,OAAO,CAAC,IAAI;oBACZ,sBAAsB,CAClB,cAAc,CAAC,CAAC,EAChB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,SAAS,CACtD,CAAC;gBACN,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAC3B,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,EACpB,cAAc,CAAC,OAAO,EACtB,MAAM,CACT,CAAoB,CAAC;YAC1B,CAAC;YACD,QAAQ,EAAE,cAAc,CAAC,QAAQ;SACpC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,iBAA4C;QAC7E,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,QAAQ,CAAC;QAEpD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACzD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,QAAQ,EAAE,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE;gBACnE,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE;gBACnC,MAAM,EAAE,gBAAgB,CAAC,MAAM,EAAE;gBACjC,UAAU,EAAE,gBAAgB,CAAC,UAAU,EAAE;aAC5C,CAAC;YACF,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,SAAS,GAAG,IAAI;gBAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBACjE,MAAM,CAAC,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;gBAErD,OAAO,CAAC,CAAC,QAAQ,CAAiB,CAAC;YACvC,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,UAA+B,EAAE,EAAE,SAAS,GAAG,IAAI,EAAE,EAAE;gBACxE,mGAAmG;gBACnG,IAAI,IAAuB,CAAC;gBAE5B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;oBACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;oBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;oBACjE,IAAI;wBACA,OAAO,CAAC,IAAI;4BACZ,CAAC,MAAM,mBAAmB,CACtB,iBAAiB,CAAC,IAAI,EACtB,QAAQ,EACR,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CACzD,CAAC,CAAC;gBACX,CAAC;qBAAM,CAAC;oBACJ,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;gBACxB,CAAC;gBAED,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAC3B,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,EACpB,iBAAiB,CAAC,OAAO,EACzB,MAAM,CACT,CAAoB,CAAC;YAC1B,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,QAAQ,CAClB,aAA4B,EAC5B,OAAwB,EACxB,gBAAoE;QAEpE,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAClG,MAAM,IAAI,GAAmB,EAAE,CAAC;QAEhC,MAAM,eAAe,GAA+B,EAAE,CAAC;QAEvD,MAAM,yBAAyB,GAAG;YAC9B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC;YACnD,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAClE,YAAY,EAAE,KAAK,EAAE,OAAiD,EAAE,EAAE;gBACtE,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrE,CAAC;YACD,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC;YAC3C,uBAAuB,EAAE,CAAC,OAA+B,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;SAC9F,CAAC;QAEF,MAAM,iBAAiB,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,yBAAyB,EAAE,CAAC;QACvE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,CAAC;YACD,MAAM,0BAA0B,GAAG,KAAK,IAAI,EAAE;gBAC1C,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CACjC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;qBAAM,IAAI,aAAa,KAAK,YAAY,EAAE,CAAC;oBACxC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAClC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CACP,wBAAwB,CAAC,GAAG,EAAE;gBAC1B,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CACX,6FAA6F,CAChG,CAAC;gBACN,CAAC;YACL,CAAC,EAAE,0BAA0B,CAAC,EAClC,IAAI,CAAC,qCAAqC,EAC1C,2BAA2B,CAC9B,CAAC;YAEF,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;gBAAS,CAAC;YACP,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,iBAAiB,CAAC,eAAgC;QACvE,MAAM,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,uBAAuB,CAAC,kCAAkC,CAAC;QAE7G,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAC7B,eAAe,CAAC,GAAG,CAAC,KAAK,CACrB,4BAA4B,uBAAuB,CAAC,aAAa,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACzG,CAAC;QACN,CAAC;QAED,IAAI,uBAAuB,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnF,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,yCAAyC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClG,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC;YAE5C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;YAE9F,IAAI,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAI,IAAmB,CAAC,CAAC,CAAC;gBAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YAED,iFAAiF;YACjF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACnB,MAAM,WAAW,GACb,YAAY,CAAC,KAAK,YAAY,mBAAmB;oBAC7C,CAAC,CAAE,YAAY,CAAC,KAAK,CAAC,KAAe;oBACrC,CAAC,CAAE,YAAY,CAAC,KAAe,CAAC;gBAExC,eAAe,CAAC,GAAG,CAAC,SAAS,CACzB,WAAW,EACX,wCAAwC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACxE,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,GAAG,CAAC,OAAO,CACvB,8DAA8D,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAC9F,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC;QAE3C,qGAAqG;QACrG,oGAAoG;QACpG,oGAAoG;QACpG,wEAAwE;QACxE,MAAM,YAAY,GAAG;YACjB,SAAS,EAAE,IAAI;YACf,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAE3D,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAED,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,OAAO,YAAY,CAAC;gBACxB,CAAC;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC;YAC1B,CAAC;SACJ,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAClC,YAAY,EACZ,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,CAAC,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,yBAAyB,EAAE,CAAC;YAC5B,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACzF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpC,QAAQ,EACR,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;YAEF,MAAM,eAAe,GAA8B,CAAC,GAAG,EAAE;gBACrD,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBACvF,IAAI,gBAAgB,KAAK,IAAI,IAAI,gBAAgB,KAAK,OAAO,EAAE,CAAC;oBAC5D,OAAO,QAAQ,CAAC;gBACpB,CAAC;gBAED,IAAI,gBAAgB,KAAK,KAAK,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;oBACjE,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,OAAO,SAAS,CAAC;YACrB,CAAC,CAAC,EAAE,CAAC;YAEL,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,eAAe,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAE3G,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;gBAChC,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;IACL,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,eAAgC,EAChC,EAAE,KAAK,EAAE,oBAAoB,EAAwB;QAErD,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5E,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC;YAClF,GAAG,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE;gBAC3E,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CACnE,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAC7C,CACJ,CAAC;YACN,CAAC,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IAES,kBAAkB,CACxB,IAAoC;QAEpC,OAAO,KAAK,EAAE,GAAG,IAAW,EAAE,EAAE,CAC5B,wBAAwB,CACpB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAC5B,CAAC;IACV,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,OAAiD,EACjD,OAA6C,EAC7C,MAA4B;QAE5B,MAAM,OAAO,GAAG,sCAAsC,CAAC;YACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;YAClC,eAAe,EAAE,OAAO,CAAC,SAAS;YAClC,kBAAkB,EAAE,OAAO,CAAC,GAAG;YAC/B,mBAAmB,EAAE,OAAO,EAAE,OAAO;SACxC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAuC,KAAK,EAAE,QAA+B,EAAE,EAAE;YACrG,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAEnC,OAAO;gBACH,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;oBAChD,SAAS;oBACT,SAAS,EAAE,EAAE,IAAI,EAAE;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,iBAAiB,EAAE,KAAK;iBAC3B,CAAC,CAAC;gBACH,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC,CAAC;QACF,0GAA0G;QAC1G,MAAM,gBAAgB,GAAG,EAAE,kBAAkB,EAAkB,CAAC;QAEhE,OAAO,MAAM,IAAI,CAAC,0BAA0B,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACrG,CAAC;IAEO,cAAc,CAAC,GAAQ,EAAE,IAAoB;QACjD,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE;YAClB,GAAG,CAAC,MAAW,EAAE,YAA8C,EAAE,QAAa;gBAC1E,IAAI,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;wBAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;oBAC/C,CAAC,CAAC;gBACN,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;YACvD,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEQ,KAAK,CAAC,QAAQ;QACnB,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpC,MAAM,IAAI,EAAE,CAAC;QACjB,CAAC;IACL,CAAC;CACJ;AAED,MAAM,UAAU,8BAA8B,CAG5C,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -40,6 +40,16 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
40
40
|
* after clicking on elements matching the provided CSS selector.
|
|
41
41
|
*/
|
|
42
42
|
globs?: GlobInput[];
|
|
43
|
+
/**
|
|
44
|
+
* An array of glob pattern strings, regexp patterns or plain objects
|
|
45
|
+
* containing patterns matching URLs that will **never** be enqueued.
|
|
46
|
+
*
|
|
47
|
+
* The plain objects must include either the `glob` property or the `regexp` property.
|
|
48
|
+
*
|
|
49
|
+
* Glob matching is always case-insensitive.
|
|
50
|
+
* If you need case-sensitive matching, provide a regexp.
|
|
51
|
+
*/
|
|
52
|
+
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
43
53
|
/**
|
|
44
54
|
* An array of regular expressions or plain objects
|
|
45
55
|
* containing regular expressions matching the URLs to be enqueued.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EAEX,eAAe,EACf,gBAAgB,EAEnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAS,IAAI,EAAkB,MAAM,YAAY,CAAC;AAO9D,KAAK,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjD,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,eAAe,CAAC;IAE9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAChD,OAAO,EAAE,qCAAqC,GAC/C,OAAO,CAAC,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EAEX,eAAe,EACf,gBAAgB,EAEnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAS,IAAI,EAAkB,MAAM,YAAY,CAAC;AAO9D,KAAK,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjD,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,eAAe,CAAC;IAE9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAChD,OAAO,EAAE,qCAAqC,GAC/C,OAAO,CAAC,sBAAsB,CAAC,CAmFjC;AAED,UAAU,sBAAsB;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,UAAU,kDAAmD,SAAQ,sBAAsB;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAsB,2CAA2C,CAC7D,OAAO,EAAE,kDAAkD,GAC5D,OAAO,CAAC,UAAU,EAAE,CAAC,CA4BvB;AAoHD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA2B5G"}
|
|
@@ -55,6 +55,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
55
55
|
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
56
56
|
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
57
57
|
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
58
|
+
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
58
59
|
transformRequestFunction: ow.optional.function,
|
|
59
60
|
waitForPageIdleSecs: ow.optional.number,
|
|
60
61
|
maxWaitForPageIdleSecs: ow.optional.number,
|
|
@@ -62,10 +63,21 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
62
63
|
forefront: ow.optional.boolean,
|
|
63
64
|
skipNavigation: ow.optional.boolean,
|
|
64
65
|
}));
|
|
65
|
-
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
|
|
66
|
+
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, } = options;
|
|
66
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
67
68
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
69
|
+
const urlExcludePatternObjects = [];
|
|
68
70
|
const urlPatternObjects = [];
|
|
71
|
+
if (exclude?.length) {
|
|
72
|
+
for (const excl of exclude) {
|
|
73
|
+
if (typeof excl === 'string' || 'glob' in excl) {
|
|
74
|
+
urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
|
|
75
|
+
}
|
|
76
|
+
else if (excl instanceof RegExp || 'regexp' in excl) {
|
|
77
|
+
urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
69
81
|
if (pseudoUrls?.length) {
|
|
70
82
|
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
71
83
|
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
@@ -87,7 +99,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
87
99
|
if (transformRequestFunction) {
|
|
88
100
|
requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
|
|
89
101
|
}
|
|
90
|
-
const requests = createRequests(requestOptions, urlPatternObjects);
|
|
102
|
+
const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);
|
|
91
103
|
const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
|
|
92
104
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
93
105
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAW/B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,GACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,IAAI,MAAM,YAAY,CAAC;AAE9B,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"click-elements.js","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAW/B,OAAO,EACH,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,cAAc,GACjB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,IAAI,MAAM,YAAY,CAAC;AAE9B,MAAM,gBAAgB,GAAG,UAAU,CAAC;AACpC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC,CAAC;AA8JhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAChD,OAA8C;IAE9C,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,QAAQ,EAAE,EAAE,CAAC,MAAM;QACnB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,YAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC;QAC/D,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,sBAAsB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC1C,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACtC,CAAC,CACL,CAAC;IAEF,MAAM,EACF,IAAI,EACJ,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,mBAAmB,GAAG,CAAC,EACvB,sBAAsB,GAAG,CAAC,EAC1B,SAAS,EACT,OAAO,GACV,GAAG,OAAO,CAAC;IAEZ,MAAM,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;IACzD,MAAM,wBAAwB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IAE/D,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,mBAAmB,GAAG,MAAM,2CAA2C,CAAC;QAC1E,IAAI;QACJ,QAAQ;QACR,qBAAqB;QACrB,wBAAwB;QACxB,YAAY;KACf,CAAC,CAAC;IACH,IAAI,cAAc,GAAG,oBAAoB,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAC;IACxE,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IACzG,CAAC;IACD,MAAM,QAAQ,GAAG,cAAc,CAAC,cAAc,EAAE,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;IAC7F,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAEzF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAaD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,2CAA2C,CAC7D,OAA2D;IAE3D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAElG,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAE/B,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,0BAA0B,CAAC,cAAc,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,2BAA2B,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAE3E,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;IAChD,qEAAqE;IACrE,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IAC5C,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAE1B,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,MAAM,eAAe,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAEjF,MAAM,2CAA2C,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAExE,oEAAoE;IACpE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;IAC7C,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,CAAC,CAAC;IAEjD,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,6BAA6B,CAClC,IAAU,EACV,QAAqB;IAErB,OAAO,KAAK,UAAU,oBAAoB,CAAC,KAAK,EAAE,OAAO;QACrD,IAAI,CAAC,2BAA2B,CAAC,IAAI,EAAE,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzE,QAAQ,CAAC,GAAG,CACR,IAAI,CAAC,SAAS,CAAC;YACX,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;YACxB,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,SAAS;SAC3C,CAAC,CACL,CAAC;QAEF,IAAI,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;YAC3B,OAAO,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,4BAA4B;QACpE,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,sBAAsB;IACzD,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,QAAqB;IACrD,OAAO,KAAK,UAAU,eAAe,CAAC,KAAK;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAEtC,iDAAiD;QACjD,sCAAsC;QACtC,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QAChH,CAAC;IACL,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,GAAY;IACzD,OAAO,GAAG,CAAC,mBAAmB,EAAE,IAAI,GAAG,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAU,EAAE,QAAqB;IAClE,OAAO,SAAS,gBAAgB,CAAC,KAAK;QAClC,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;QACxB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;AACN,CAAC;AAaD;;GAEG;AACH,KAAK,UAAU,wBAAwB,CAAC,IAAU;IAC9C,0BAA0B;IAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACrB,MAAgC,CAAC,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC;QACvE,OAAQ,MAAgC,CAAC,OAAO,CAAC,CAAC,iCAAiC;QAClF,MAAgC,CAAC,OAAO,GAAG;YACxC,YAAY,EAAE,EAAE;YAChB,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,EAAE;YACT,EAAE,KAAI,CAAC;YACP,IAAI,KAAI,CAAC;YACT,OAAO,KAAI,CAAC;YACZ,SAAS,CAAC,GAAG,IAAe;gBACxB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,IAAe;gBAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;SACW,CAAC;IACrB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,0BAA0B;AAC1B;;GAEG;AACH,SAAS,kCAAkC,CAAC,EAAW,EAAE,MAAc;IACnE,MAAM,MAAM,GAAG,EAAiB,CAAC;IACjC,MAAM,CAAC,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;IACpC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACxB,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC;IACvB,MAAM,YAAY,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;IACpD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACvD,IAAI,CAAC,YAAY,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC;AACzD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,QAAgB,EAAE,YAA2B;IACzF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC/C,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnG,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,gBAAgB,CAAC;IAC9B,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,MAAM,CAAC,QAAQ,CAAC,kCAAkC,EAAE,MAAM,EAAE,CAAC,CAAC;YACpE,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACjC,oBAAoB,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,GAAY,CAAC;YACvB,IAAI,gBAAgB,IAAI,CAAC,CAAC,KAAM,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;gBACrE,GAAG,CAAC,OAAO,CACP,4BAA4B,QAAQ,+DAA+D;oBAC/F,qHAAqH;oBACrH,uHAAuH,CAC9H,CAAC;gBACF,gBAAgB,GAAG,KAAK,CAAC;YAC7B,CAAC;YACD,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IACD,GAAG,CAAC,KAAK,CACL,wDAAwD,oBAAoB,oBAAoB,cAAc,CAAC,MAAM,EAAE,CAC1H,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,eAAe,CAAC,EAC3B,IAAI,EACJ,qBAAqB,EACrB,wBAAwB,GACH;IACrB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,IAAI,OAAuB,CAAC;QAC5B,IAAI,UAA0B,CAAC;QAE/B,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QAElC,SAAS,eAAe;YACpB,YAAY,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,YAAY,CAAC,UAAU,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACb,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC9B,CAAC;QAED,SAAS,iBAAiB;YACtB,GAAG,CAAC,KAAK,CACL,oEAAoE,wBAAwB,MAAM;gBAC9F,4GAA4G,CACnH,CAAC;YACF,MAAM,EAAE,CAAC;QACb,CAAC;QAED,SAAS,MAAM;YACX,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;YAC1G,OAAO,EAAE,CAAC;QACd,CAAC;QAED,UAAU,GAAG,UAAU,CAAC,iBAAiB,EAAE,wBAAwB,CAAC,CAAC;QACrE,eAAe,EAAE,CAAC,CAAC,wEAAwE;QAC3F,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;QACpC,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,2CAA2C,CAAC,IAAU,EAAE,QAAqB;IACxF,0BAA0B;IAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,OAAiC,CAAC;QACjE,MAAgC,CAAC,OAAO,GAAI,MAAgC,CAAC,mBAAmB,CAAC;QAClG,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC;YAC/C,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAG,GAAa,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7F,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -197,6 +197,8 @@ export declare class PlaywrightCrawler<ContextExtension = {}, ExtendedContext ex
|
|
|
197
197
|
maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
198
198
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
199
199
|
maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
200
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
201
|
+
maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
200
202
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
201
203
|
autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
202
204
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -206,7 +208,7 @@ export declare class PlaywrightCrawler<ContextExtension = {}, ExtendedContext ex
|
|
|
206
208
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
207
209
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
208
210
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
209
|
-
respectRobotsTxtFile: import("ow").
|
|
211
|
+
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
210
212
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
211
213
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
212
214
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAwB,MAAM,kBAAkB,CAAC;AACvF,OAAO,KAAK,EAAsB,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGhE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAExE,OAAO,KAAK,EAER,uBAAuB,EAIvB,sBAAsB,EAEzB,MAAM,6BAA6B,CAAC;AAGrC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC/E,SAAQ,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,EAC1E,sBAAsB;CAAG;AACjC,MAAM,WAAW,cAAe,SAAQ,WAAW,CAAC,yBAAyB,EAAE,qBAAqB,CAAC;CAAG;AACxG,MAAM,MAAM,qBAAqB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEhE,MAAM,WAAW,wBAAwB,CACrC,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,yBAAyB,GAAG,yBAAyB,GAAG,gBAAgB,CAClG,SAAQ,qBAAqB,CACvB,IAAI,EACJ,QAAQ,EACR,oBAAoB,EACpB,yBAAyB,EACzB,gBAAgB,EAChB,eAAe,EACf;IAAE,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAA;CAAE,CACzC;IACD;;OAEG;IACH,aAAa,CAAC,EAAE,uBAAuB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;;;;;;;;;;;;OAgBG;IACH,kBAAkB,CAAC,EAAE,cAAc,EAAE,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,iBAAiB,CAC1B,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,yBAAyB,GAAG,yBAAyB,GAAG,gBAAgB,CAClG,SAAQ,cAAc,CACpB,IAAI,EACJ,QAAQ,EACR,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAA;CAAE,EACtC,aAAa,EACb,yBAAyB,EACzB,gBAAgB,EAChB,eAAe,CAClB;aAcyB,MAAM;IAb5B,iBAA0B,YAAY
|
|
1
|
+
{"version":3,"file":"playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,qBAAqB,EACrB,sBAAsB,EACtB,WAAW,EACX,sBAAsB,EACtB,cAAc,EACd,YAAY,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAwB,MAAM,kBAAkB,CAAC;AACvF,OAAO,KAAK,EAAsB,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACxG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGhE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAExE,OAAO,KAAK,EAER,uBAAuB,EAIvB,sBAAsB,EAEzB,MAAM,6BAA6B,CAAC;AAGrC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAC/E,SAAQ,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,CAAC,EAC1E,sBAAsB;CAAG;AACjC,MAAM,WAAW,cAAe,SAAQ,WAAW,CAAC,yBAAyB,EAAE,qBAAqB,CAAC;CAAG;AACxG,MAAM,MAAM,qBAAqB,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEhE,MAAM,WAAW,wBAAwB,CACrC,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,yBAAyB,GAAG,yBAAyB,GAAG,gBAAgB,CAClG,SAAQ,qBAAqB,CACvB,IAAI,EACJ,QAAQ,EACR,oBAAoB,EACpB,yBAAyB,EACzB,gBAAgB,EAChB,eAAe,EACf;IAAE,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAA;CAAE,CACzC;IACD;;OAEG;IACH,aAAa,CAAC,EAAE,uBAAuB,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;;;;;;;;;;;;OAgBG;IACH,kBAAkB,CAAC,EAAE,cAAc,EAAE,CAAC;IAEtC;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,qBAAa,iBAAiB,CAC1B,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,yBAAyB,GAAG,yBAAyB,GAAG,gBAAgB,CAClG,SAAQ,cAAc,CACpB,IAAI,EACJ,QAAQ,EACR,oBAAoB,EACpB;IAAE,cAAc,EAAE,CAAC,gBAAgB,CAAC,CAAA;CAAE,EACtC,aAAa,EACb,yBAAyB,EACzB,gBAAgB,EAChB,eAAe,CAClB;aAcyB,MAAM;IAb5B,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAMpC;IAEF;;OAEG;gBAEC,OAAO,GAAE,wBAAwB,CAAC,eAAe,CAAM,EACrC,MAAM,gBAAkC;cA4CrC,kBAAkB,CACvC,eAAe,EAAE,yBAAyB,EAC1C,WAAW,EAAE,uBAAuB;YAK1B,cAAc;CAqD/B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,sBAAsB,CAClC,OAAO,SAAS,yBAAyB,GAAG,yBAAyB,EACrE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,qDAEzC"}
|
|
@@ -509,6 +509,15 @@ export interface PlaywrightContextUtils {
|
|
|
509
509
|
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
|
510
510
|
/**
|
|
511
511
|
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
512
|
+
*
|
|
513
|
+
* Note that this method requires the idcac-playwright package to be installed.
|
|
514
|
+
* Crawlee does not include it by default due to licensing issues.
|
|
515
|
+
*
|
|
516
|
+
* To use this method, please install the package manually by running:
|
|
517
|
+
*
|
|
518
|
+
* ```bash
|
|
519
|
+
* npm install idcac-playwright
|
|
520
|
+
* ```
|
|
512
521
|
*/
|
|
513
522
|
closeCookieModals(): Promise<void>;
|
|
514
523
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EAAE,aAAa,EAAiB,KAAK,OAAO,EAAE,KAAK,OAAO,EAA4B,MAAM,kBAAkB,CAAC;AACtH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAA4B,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EAAE,aAAa,EAAiB,KAAK,OAAO,EAAE,KAAK,OAAO,EAA4B,MAAM,kBAAkB,CAAC;AACtH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAA4B,MAAM,gBAAgB,CAAC;AAG7F,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAKxD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAUxE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CA0BhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAC9B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,OAAO,EAChB,WAAW,GAAE,uBAA4B,GAC1C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAiD1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+C/F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,gBAAgB,CAClC,IAAI,EAAE,IAAI,EACV,iBAAiB,UAAQ,EACzB,aAAa,UAAQ,GACtB,OAAO,CAAC,WAAW,CAAC,CA2CtB;AAwBD,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjE;AAED,MAAM,WAAW,gCAAgC;IAC7C,2FAA2F;IAC3F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uKAAuK;IACvK,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrF,sEAAsE;IACtE,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACxD;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,iBAAe,yBAAyB,CACpC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,GAAE,gCAAqC,GAC/C,OAAO,CAAC,IAAI,CAAC,CAgGf;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAC1B,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAC9E,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;;;;;;;;;;OAWG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,yBAAyB,CAAC,OAAO,CAAC,EAAE,gCAAgC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxF;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe;;;;;;;;;;;;;CAa3B,CAAC"}
|
|
@@ -23,7 +23,6 @@ import vm from 'node:vm';
|
|
|
23
23
|
import { Configuration, KeyValueStore, SessionError, validators } from '@crawlee/browser';
|
|
24
24
|
import { expandShadowRoots, sleep } from '@crawlee/utils';
|
|
25
25
|
import * as cheerio from 'cheerio';
|
|
26
|
-
import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
|
|
27
26
|
import ow from 'ow';
|
|
28
27
|
import { LruCache } from '@apify/datastructures';
|
|
29
28
|
import log_ from '@apify/log';
|
|
@@ -403,7 +402,15 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
403
402
|
try {
|
|
404
403
|
const iframe = await frame.contentFrame();
|
|
405
404
|
if (iframe) {
|
|
406
|
-
const
|
|
405
|
+
const getIframeHTML = async () => {
|
|
406
|
+
try {
|
|
407
|
+
return iframe.locator('body').first().innerHTML();
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
return iframe.content();
|
|
411
|
+
}
|
|
412
|
+
};
|
|
413
|
+
const contents = await getIframeHTML();
|
|
407
414
|
await frame.evaluate((f, c) => {
|
|
408
415
|
const replacementNode = document.createElement('div');
|
|
409
416
|
replacementNode.innerHTML = c;
|
|
@@ -423,9 +430,34 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
423
430
|
const pageContent = html || (await page.content());
|
|
424
431
|
return cheerio.load(pageContent);
|
|
425
432
|
}
|
|
433
|
+
let idcacPlaywright = null;
|
|
434
|
+
async function getIdcacPlaywright() {
|
|
435
|
+
if (idcacPlaywright)
|
|
436
|
+
return idcacPlaywright;
|
|
437
|
+
try {
|
|
438
|
+
idcacPlaywright = await import('idcac-playwright');
|
|
439
|
+
}
|
|
440
|
+
catch (error) {
|
|
441
|
+
log.warning(`Failed to import 'idcac-playwright'.
|
|
442
|
+
|
|
443
|
+
We recently made idcac-playwright an optional dependency due to licensing issues.
|
|
444
|
+
To use this feature, please install it manually by running
|
|
445
|
+
|
|
446
|
+
npm install idcac-playwright
|
|
447
|
+
|
|
448
|
+
Original error message follows:
|
|
449
|
+
|
|
450
|
+
${error.message}
|
|
451
|
+
`);
|
|
452
|
+
}
|
|
453
|
+
return idcacPlaywright;
|
|
454
|
+
}
|
|
426
455
|
export async function closeCookieModals(page) {
|
|
427
456
|
ow(page, ow.object.validate(validators.browserPage));
|
|
428
|
-
await
|
|
457
|
+
const idcac = await getIdcacPlaywright();
|
|
458
|
+
if (idcac?.getInjectableScript()) {
|
|
459
|
+
await page.evaluate(idcac.getInjectableScript());
|
|
460
|
+
}
|
|
429
461
|
}
|
|
430
462
|
/**
|
|
431
463
|
* This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
|