@crawlee/playwright 4.0.0-beta.4 → 4.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/internals/adaptive-playwright-crawler.d.ts +48 -56
- package/internals/adaptive-playwright-crawler.d.ts.map +1 -1
- package/internals/adaptive-playwright-crawler.js +309 -205
- package/internals/adaptive-playwright-crawler.js.map +1 -1
- package/internals/enqueue-links/click-elements.d.ts +29 -10
- package/internals/enqueue-links/click-elements.d.ts.map +1 -1
- package/internals/enqueue-links/click-elements.js +38 -15
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/playwright-crawler.d.ts +53 -49
- package/internals/playwright-crawler.d.ts.map +1 -1
- package/internals/playwright-crawler.js +52 -13
- package/internals/playwright-crawler.js.map +1 -1
- package/internals/playwright-launcher.d.ts +2 -0
- package/internals/playwright-launcher.d.ts.map +1 -1
- package/internals/utils/playwright-utils.d.ts +19 -6
- package/internals/utils/playwright-utils.d.ts.map +1 -1
- package/internals/utils/playwright-utils.js +103 -94
- package/internals/utils/playwright-utils.js.map +1 -1
- package/internals/utils/rendering-type-prediction.d.ts +9 -5
- package/internals/utils/rendering-type-prediction.d.ts.map +1 -1
- package/internals/utils/rendering-type-prediction.js +58 -23
- package/internals/utils/rendering-type-prediction.js.map +1 -1
- package/package.json +12 -9
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -17,14 +17,13 @@
|
|
|
17
17
|
* ```
|
|
18
18
|
* @module playwrightUtils
|
|
19
19
|
*/
|
|
20
|
-
import { Configuration, type Request
|
|
20
|
+
import { Configuration, type Request } from '@crawlee/browser';
|
|
21
21
|
import type { BatchAddRequestsResult } from '@crawlee/types';
|
|
22
22
|
import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
|
|
23
23
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
24
24
|
import type { Page, Response } from 'playwright';
|
|
25
25
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
|
|
26
26
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
27
|
-
import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from '../playwright-crawler.js';
|
|
28
27
|
import { RenderingTypePredictor } from './rendering-type-prediction.js';
|
|
29
28
|
export interface InjectFileOptions {
|
|
30
29
|
/**
|
|
@@ -290,7 +289,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
|
|
|
290
289
|
*/
|
|
291
290
|
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
|
|
292
291
|
export declare function closeCookieModals(page: Page): Promise<void>;
|
|
293
|
-
interface HandleCloudflareChallengeOptions {
|
|
292
|
+
export interface HandleCloudflareChallengeOptions {
|
|
294
293
|
/** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
|
|
295
294
|
verbose?: boolean;
|
|
296
295
|
/** How long should we wait after the challenge is completed for the final page to load. */
|
|
@@ -304,6 +303,13 @@ interface HandleCloudflareChallengeOptions {
|
|
|
304
303
|
isChallengeCallback?: (page: Page) => Promise<boolean>;
|
|
305
304
|
/** Allows overriding the detection of Cloudflare "blocked page". */
|
|
306
305
|
isBlockedCallback?: (page: Page) => Promise<boolean>;
|
|
306
|
+
/** Allows overriding how the checkbox click position is calculated. */
|
|
307
|
+
clickPositionCallback?: (page: Page) => Promise<{
|
|
308
|
+
x: number;
|
|
309
|
+
y: number;
|
|
310
|
+
} | null>;
|
|
311
|
+
/** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
|
|
312
|
+
preChallengeSleepSecs?: number;
|
|
307
313
|
}
|
|
308
314
|
/**
|
|
309
315
|
* This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
|
|
@@ -325,10 +331,9 @@ interface HandleCloudflareChallengeOptions {
|
|
|
325
331
|
*
|
|
326
332
|
* @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
|
|
327
333
|
* @param url current URL for request identification, only used for logging
|
|
328
|
-
* @param [session] current session object
|
|
329
334
|
* @param [options]
|
|
330
335
|
*/
|
|
331
|
-
declare function handleCloudflareChallenge(page: Page, url: string,
|
|
336
|
+
declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions, blockedStatusCodes?: Set<number>): Promise<void>;
|
|
332
337
|
/** @internal */
|
|
333
338
|
export interface PlaywrightContextUtils {
|
|
334
339
|
/**
|
|
@@ -510,6 +515,15 @@ export interface PlaywrightContextUtils {
|
|
|
510
515
|
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
|
511
516
|
/**
|
|
512
517
|
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
518
|
+
*
|
|
519
|
+
* Note that this method requires the idcac-playwright package to be installed.
|
|
520
|
+
* Crawlee does not include it by default due to licensing issues.
|
|
521
|
+
*
|
|
522
|
+
* To use this method, please install the package manually by running:
|
|
523
|
+
*
|
|
524
|
+
* ```bash
|
|
525
|
+
* npm install idcac-playwright
|
|
526
|
+
* ```
|
|
513
527
|
*/
|
|
514
528
|
closeCookieModals(): Promise<void>;
|
|
515
529
|
/**
|
|
@@ -534,7 +548,6 @@ export interface PlaywrightContextUtils {
|
|
|
534
548
|
*/
|
|
535
549
|
handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
|
|
536
550
|
}
|
|
537
|
-
export declare function registerUtilsToContext(context: PlaywrightCrawlingContext, crawlerOptions: PlaywrightCrawlerOptions): void;
|
|
538
551
|
export { enqueueLinksByClickingElements };
|
|
539
552
|
/** @internal */
|
|
540
553
|
export declare const playwrightUtils: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,
|
|
1
|
+
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EAAE,aAAa,EAAiB,KAAK,OAAO,EAA4C,MAAM,kBAAkB,CAAC;AACxH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAA4B,MAAM,gBAAgB,CAAC;AAG7F,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAIxD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAUxE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CA0BhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAC9B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,OAAO,EAChB,WAAW,GAAE,uBAA4B,GAC1C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAiD1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+C/F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,gBAAgB,CAClC,IAAI,EAAE,IAAI,EACV,iBAAiB,UAAQ,EACzB,aAAa,UAAQ,GACtB,OAAO,CAAC,WAAW,CAAC,CA2CtB;AAwBD,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjE;AAED,MAAM,WAAW,gCAAgC;IAC7C,2FAA2F;IAC3F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uKAAuK;IACvK,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrF,sEAAsE;IACtE,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACrD,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACjF,4GAA4G;IAC5G,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,iBAAe,yBAAyB,CACpC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,gCAAqC,EAC9C,kBAAkB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CA4Gf;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAC1B,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAC9E,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;;;;;;;;;;OAWG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,yBAAyB,CAAC,OAAO,CAAC,EAAE,gCAAgC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxF;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe;;;;;;;;;;;;;CAa3B,CAAC"}
|
|
@@ -20,16 +20,14 @@
|
|
|
20
20
|
import { readFile } from 'node:fs/promises';
|
|
21
21
|
import { createRequire } from 'node:module';
|
|
22
22
|
import vm from 'node:vm';
|
|
23
|
-
import { Configuration, KeyValueStore,
|
|
23
|
+
import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
|
|
24
24
|
import { expandShadowRoots, sleep } from '@crawlee/utils';
|
|
25
25
|
import * as cheerio from 'cheerio';
|
|
26
|
-
import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
|
|
27
26
|
import ow from 'ow';
|
|
28
27
|
import { LruCache } from '@apify/datastructures';
|
|
29
|
-
import log_ from '@apify/log';
|
|
30
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
31
29
|
import { RenderingTypePredictor } from './rendering-type-prediction.js';
|
|
32
|
-
const
|
|
30
|
+
const getLog = () => serviceLocator.getChildLog('Playwright Utils');
|
|
33
31
|
const require = createRequire(import.meta.url);
|
|
34
32
|
const jqueryPath = require.resolve('jquery');
|
|
35
33
|
const MAX_INJECT_FILE_CACHE_SIZE = 10;
|
|
@@ -64,7 +62,7 @@ export async function injectFile(page, filePath, options = {}) {
|
|
|
64
62
|
if (options.surviveNavigations) {
|
|
65
63
|
page.on('framenavigated', async () => page
|
|
66
64
|
.evaluate(contents)
|
|
67
|
-
.catch((error) =>
|
|
65
|
+
.catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
|
|
68
66
|
}
|
|
69
67
|
return evalP;
|
|
70
68
|
}
|
|
@@ -123,7 +121,7 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
|
123
121
|
const isEmpty = (o) => !o || Object.keys(o).length === 0;
|
|
124
122
|
if (method !== 'GET' || payload || !isEmpty(headers)) {
|
|
125
123
|
// This is not deprecated, we use it to log only once.
|
|
126
|
-
|
|
124
|
+
getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
|
|
127
125
|
'in recent versions of Playwright. Use only when necessary.');
|
|
128
126
|
let wasCalled = false;
|
|
129
127
|
const interceptRequestHandler = async (route) => {
|
|
@@ -144,7 +142,7 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
|
|
|
144
142
|
await route.continue(overrides);
|
|
145
143
|
}
|
|
146
144
|
catch (error) {
|
|
147
|
-
|
|
145
|
+
getLog().debug('Error inside request interceptor', { error });
|
|
148
146
|
}
|
|
149
147
|
return undefined;
|
|
150
148
|
};
|
|
@@ -213,7 +211,7 @@ export async function blockRequests(page, options = {}) {
|
|
|
213
211
|
await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
|
|
214
212
|
}
|
|
215
213
|
catch {
|
|
216
|
-
|
|
214
|
+
getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
|
|
217
215
|
}
|
|
218
216
|
}
|
|
219
217
|
/**
|
|
@@ -249,7 +247,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
|
|
|
249
247
|
func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
|
|
250
248
|
}
|
|
251
249
|
catch (err) {
|
|
252
|
-
|
|
250
|
+
getLog().exception(err, 'Cannot compile script!');
|
|
253
251
|
throw err;
|
|
254
252
|
}
|
|
255
253
|
if (typeof func !== 'function')
|
|
@@ -403,7 +401,15 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
403
401
|
try {
|
|
404
402
|
const iframe = await frame.contentFrame();
|
|
405
403
|
if (iframe) {
|
|
406
|
-
const
|
|
404
|
+
const getIframeHTML = async () => {
|
|
405
|
+
try {
|
|
406
|
+
return iframe.locator('body').first().innerHTML();
|
|
407
|
+
}
|
|
408
|
+
catch {
|
|
409
|
+
return iframe.content();
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
const contents = await getIframeHTML();
|
|
407
413
|
await frame.evaluate((f, c) => {
|
|
408
414
|
const replacementNode = document.createElement('div');
|
|
409
415
|
replacementNode.innerHTML = c;
|
|
@@ -413,7 +419,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
413
419
|
}
|
|
414
420
|
}
|
|
415
421
|
catch (error) {
|
|
416
|
-
|
|
422
|
+
getLog().warning(`Failed to extract iframe content: ${error}`);
|
|
417
423
|
}
|
|
418
424
|
}));
|
|
419
425
|
}
|
|
@@ -423,9 +429,34 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
423
429
|
const pageContent = html || (await page.content());
|
|
424
430
|
return cheerio.load(pageContent);
|
|
425
431
|
}
|
|
432
|
+
let idcacPlaywright = null;
|
|
433
|
+
async function getIdcacPlaywright() {
|
|
434
|
+
if (idcacPlaywright)
|
|
435
|
+
return idcacPlaywright;
|
|
436
|
+
try {
|
|
437
|
+
idcacPlaywright = await import('idcac-playwright');
|
|
438
|
+
}
|
|
439
|
+
catch (error) {
|
|
440
|
+
getLog().warning(`Failed to import 'idcac-playwright'.
|
|
441
|
+
|
|
442
|
+
We recently made idcac-playwright an optional dependency due to licensing issues.
|
|
443
|
+
To use this feature, please install it manually by running
|
|
444
|
+
|
|
445
|
+
npm install idcac-playwright
|
|
446
|
+
|
|
447
|
+
Original error message follows:
|
|
448
|
+
|
|
449
|
+
${error.message}
|
|
450
|
+
`);
|
|
451
|
+
}
|
|
452
|
+
return idcacPlaywright;
|
|
453
|
+
}
|
|
426
454
|
export async function closeCookieModals(page) {
|
|
427
455
|
ow(page, ow.object.validate(validators.browserPage));
|
|
428
|
-
await
|
|
456
|
+
const idcac = await getIdcacPlaywright();
|
|
457
|
+
if (idcac?.getInjectableScript()) {
|
|
458
|
+
await page.evaluate(idcac.getInjectableScript());
|
|
459
|
+
}
|
|
429
460
|
}
|
|
430
461
|
/**
|
|
431
462
|
* This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
|
|
@@ -447,17 +478,12 @@ export async function closeCookieModals(page) {
|
|
|
447
478
|
*
|
|
448
479
|
* @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
|
|
449
480
|
* @param url current URL for request identification, only used for logging
|
|
450
|
-
* @param [session] current session object
|
|
451
481
|
* @param [options]
|
|
452
482
|
*/
|
|
453
|
-
async function handleCloudflareChallenge(page, url,
|
|
454
|
-
//
|
|
455
|
-
const
|
|
456
|
-
|
|
457
|
-
if (blockedStatusCodes?.includes(403)) {
|
|
458
|
-
const idx = blockedStatusCodes.indexOf(403);
|
|
459
|
-
blockedStatusCodes.splice(idx, 1);
|
|
460
|
-
}
|
|
483
|
+
async function handleCloudflareChallenge(page, url, options = {}, blockedStatusCodes) {
|
|
484
|
+
// Cloudflare pages return 403, which is blocked by default — temporarily allow it during challenge handling
|
|
485
|
+
const had403 = blockedStatusCodes?.has(403);
|
|
486
|
+
blockedStatusCodes?.delete(403);
|
|
461
487
|
options.isBlockedCallback ??= async () => {
|
|
462
488
|
const isBlocked = await page.evaluate(() => {
|
|
463
489
|
return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
|
|
@@ -479,84 +505,67 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
|
|
|
479
505
|
const isChallenge = async () => {
|
|
480
506
|
return options.isChallengeCallback(page).catch(() => false);
|
|
481
507
|
};
|
|
482
|
-
|
|
483
|
-
await retryBlocked();
|
|
484
|
-
return;
|
|
485
|
-
}
|
|
486
|
-
const logLevel = options.verbose ? 'info' : 'debug';
|
|
487
|
-
log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
|
|
488
|
-
const bb = await page
|
|
489
|
-
.evaluate(() => {
|
|
490
|
-
const div = document.querySelector('.main-content div');
|
|
491
|
-
return div?.getBoundingClientRect();
|
|
492
|
-
})
|
|
493
|
-
.catch(() => undefined);
|
|
494
|
-
if (!bb) {
|
|
495
|
-
return;
|
|
496
|
-
}
|
|
497
|
-
const randomOffset = (range) => {
|
|
498
|
-
return Math.round(100 * range * Math.random()) / 100;
|
|
499
|
-
};
|
|
500
|
-
const x = bb.x + 30;
|
|
501
|
-
const y = bb.y + 25;
|
|
502
|
-
// try to click the checkbox every second
|
|
503
|
-
for (let i = 0; i < 10; i++) {
|
|
504
|
-
await sleep(1000);
|
|
505
|
-
// break early if we are no longer on the CF challenge page
|
|
508
|
+
try {
|
|
506
509
|
if (!(await isChallenge())) {
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
if (options.clickCallback) {
|
|
510
|
-
await options.clickCallback(page, { x, y });
|
|
511
|
-
continue;
|
|
510
|
+
await retryBlocked();
|
|
511
|
+
return;
|
|
512
512
|
}
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
const
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
if (await isChallenge()) {
|
|
523
|
-
throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
|
|
524
|
-
}
|
|
525
|
-
await retryBlocked();
|
|
526
|
-
}
|
|
527
|
-
export function registerUtilsToContext(context, crawlerOptions) {
|
|
528
|
-
context.injectFile = async (filePath, options) => injectFile(context.page, filePath, options);
|
|
529
|
-
context.injectJQuery = async () => {
|
|
530
|
-
if (context.request.state === RequestState.BEFORE_NAV) {
|
|
531
|
-
log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
|
|
532
|
-
await injectJQuery(context.page);
|
|
513
|
+
const logLevel = options.verbose ? 'info' : 'debug';
|
|
514
|
+
getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
|
|
515
|
+
const bb = await page
|
|
516
|
+
.evaluate(() => {
|
|
517
|
+
const div = document.querySelector('.main-content div');
|
|
518
|
+
return div?.getBoundingClientRect();
|
|
519
|
+
})
|
|
520
|
+
.catch(() => undefined);
|
|
521
|
+
if (!bb) {
|
|
533
522
|
return;
|
|
534
523
|
}
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
await
|
|
524
|
+
const randomOffset = (range) => {
|
|
525
|
+
return Math.round(100 * range * Math.random()) / 100;
|
|
526
|
+
};
|
|
527
|
+
let x = bb.x + 30;
|
|
528
|
+
let y = bb.y + 25;
|
|
529
|
+
// try to click the checkbox every second
|
|
530
|
+
for (let i = 0; i < 10; i++) {
|
|
531
|
+
await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
|
|
532
|
+
// break early if we are no longer on the CF challenge page
|
|
533
|
+
if (!(await isChallenge())) {
|
|
534
|
+
break;
|
|
535
|
+
}
|
|
536
|
+
if (options.clickPositionCallback) {
|
|
537
|
+
const pos = await options.clickPositionCallback(page);
|
|
538
|
+
if (pos) {
|
|
539
|
+
x = pos.x;
|
|
540
|
+
y = pos.y;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
if (options.clickCallback) {
|
|
544
|
+
await options.clickCallback(page, { x, y });
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
// we can click on the text too, so X can be a bit larger
|
|
548
|
+
const xRandomized = x + randomOffset(10);
|
|
549
|
+
const yRandomized = y + randomOffset(10);
|
|
550
|
+
getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
|
|
551
|
+
x: xRandomized,
|
|
552
|
+
y: yRandomized,
|
|
553
|
+
});
|
|
554
|
+
await page.mouse.click(xRandomized, yRandomized);
|
|
555
|
+
// sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
|
|
556
|
+
await page.mouse.click(xRandomized, yRandomized + 35);
|
|
545
557
|
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
context.handleCloudflareChallenge = async (options) => {
|
|
558
|
-
return handleCloudflareChallenge(context.page, context.request.url, context.session, options);
|
|
559
|
-
};
|
|
558
|
+
await sleep((options.sleepSecs ?? 10) * 1000);
|
|
559
|
+
if (await isChallenge()) {
|
|
560
|
+
throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
|
|
561
|
+
}
|
|
562
|
+
await retryBlocked();
|
|
563
|
+
}
|
|
564
|
+
finally {
|
|
565
|
+
if (had403) {
|
|
566
|
+
blockedStatusCodes?.add(403);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
560
569
|
}
|
|
561
570
|
export { enqueueLinksByClickingElements };
|
|
562
571
|
/** @internal */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-utils.js","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EACH,aAAa,EACb,aAAa,EAEb,YAAY,EAEZ,YAAY,EACZ,UAAU,GACb,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAqC,iBAAiB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,mBAAmB,IAAI,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AACjF,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,IAAI,MAAM,YAAY,CAAC;AAG9B,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AAEpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAEvD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE7C,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AA0BtH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,QAAgB,EAAE,UAA6B,EAAE;IAC1F,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CACL,CAAC;IAEF,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,KAAK,IAAI,EAAE,CACjC,IAAI;aACC,QAAQ,CAAC,QAAQ,CAAC;aAClB,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAClG,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAA0C;IACrF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,IAAI,IAAI,EAAE,CAAC,CAAC;AACrG,CAAC;AAyBD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,IAAU,EACV,OAAgB,EAChB,cAAuC,EAAE;IAEzC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;KACrD,CAAC,CACL,CAAC;IACF,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;IAElE,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,sDAAsD;QACtD,GAAG,CAAC,UAAU,CACV,+GAA+G;YAC3G,4DAA4D,CACnE,CAAC;QACF,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,KAAY,EAAE,EAAE;YACnD,IAAI,CAAC;gBACD,oGAAoG;gBACpG,uDAAuD;gBACvD,IAAI,SAAS,EAAE,CAAC;oBACZ,OAAO,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClC,CAAC;gBAED,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM,SAAS,GAAe,EAAE,CAAC;gBAEjC,IAAI,MAAM,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChD,IAAI,OAAO;oBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;gBACnD,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;YAED,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,UAAgC,EAAE;IAC9E,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,WAAW,GAAG,kCAAkC,EAAE,gBAAgB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE5F,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAExD,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpC,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACL,GAAG,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IACtF,CAAC;AACL,CAAC;AASD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,aAAa,CAAC,YAAoB,EAAE,UAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzF,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACD,IAAI,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;IACtI,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,wBAAwB,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAEtH,OAAO,IAAI,CAAC;AAChB,CAAC;AAsCD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,UAAiC,EAAE;IAChF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACnC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CACL,CAAC;IAEF,MAAM,EACF,WAAW,GAAG,CAAC,EACf,eAAe,GAAG,CAAC,EACnB,WAAW,GAAG,CAAC,EACf,eAAe,GAAG,KAAK,EACvB,cAAc,EACd,kBAAkB,GACrB,GAAG,OAAO,CAAC;IAEZ,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;YAChE,cAAc,CAAC,YAAY,EAAE,CAAC;QAClC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;YAC9D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;gBAC5C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;YACX,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9D,CAAC;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;YACrE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,8CAA8C;QAC9C,IAAI,eAAe,GAAG,CAAC,IAAI,gBAAgB,IAAI,eAAe,EAAE,CAAC;YAC7D,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,gBAAgB,IAAI,KAAK,CAAC;IAC9B,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAe,CAAC,CAAC;QAC7C,sDAAsD;QACtD,IAAI,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,gBAAgB,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACrB,IAAI,MAAM,kBAAkB,EAAE,EAAE,CAAC;gBAC7B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAwCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,UAA+B,EAAE;IAC5E,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC7B,CAAC,CACL,CAAC;IAEF,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,EACjB,MAAM,GACT,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE;YACtD,MAAM,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;SACpD,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;gBAC3C,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,iBAAiB;gBAC1B,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,UAAU;aACzB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7F,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,IAAU,EACV,iBAAiB,GAAG,KAAK,EACzB,aAAa,GAAG,KAAK;IAErB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACvB,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;gBAE1C,IAAI,MAAM,EAAE,CAAC;oBACT,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;oBAExC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;wBAC1B,MAAM,eAAe,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;wBACtD,eAAe,CAAC,SAAS,GAAG,CAAC,CAAC;wBAC9B,eAAe,CAAC,SAAS,GAAG,4BAA4B,CAAC;wBAEzD,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;oBACnC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,GAAG,CAAC,OAAO,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED,MAAM,IAAI,GAAG,iBAAiB;QAC1B,CAAC,CAAC,IAAI;QACN,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAY,CAAC;IACvF,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnD,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAC9C,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAErD,MAAM,IAAI,CAAC,QAAQ,CAAC,sBAAsB,EAAE,CAAC,CAAC;AAClD,CAAC;AAeD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,KAAK,UAAU,yBAAyB,CACpC,IAAU,EACV,GAAW,EACX,OAAiB,EACjB,UAA4C,EAAE;IAE9C,wCAAwC;IACxC,MAAM,kBAAkB,GAAG,OAAO,EAAE,CAAC,aAAa,CAAC,CAAC,oBAAoB,CAAa,CAAC;IAEtF,yDAAyD;IACzD,IAAI,kBAAkB,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5C,kBAAkB,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,CAAC,iBAAiB,KAAK,KAAK,IAAI,EAAE;QACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACvC,OAAO,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC;QACtG,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,CAAC,SAAS,CAAC;IACvB,CAAC,CAAC;IAEF,OAAO,CAAC,mBAAmB,KAAK,KAAK,IAAI,EAAE;QACvC,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE;YAClC,OAAO,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,yDAAyD,CAAC,CAAC;QAC/F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,iBAAkB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAE5E,IAAI,SAAS,EAAE,CAAC;YACZ,MAAM,IAAI,YAAY,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC,CAAC;IAEF,gDAAgD;IAChD,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;QAC3B,OAAO,OAAO,CAAC,mBAAoB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC,CAAC;IAEF,IAAI,CAAC,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,CAAC;QACzB,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO;IACX,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IACpD,GAAG,CAAC,QAAQ,CAAC,CACT,oCAAoC,GAAG,6CAA6C,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,WAAW,CAChI,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,IAAI;SAChB,QAAQ,CAAC,GAAG,EAAE;QACX,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;QACxD,OAAO,GAAG,EAAE,qBAAqB,EAAE,CAAC;IACxC,CAAC,CAAC;SACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAE5B,IAAI,CAAC,EAAE,EAAE,CAAC;QACN,OAAO;IACX,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,KAAa,EAAE,EAAE;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC,CAAC;IAEF,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpB,yCAAyC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1B,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAElB,2DAA2D;QAC3D,IAAI,CAAC,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,CAAC;YACzB,MAAM;QACV,CAAC;QAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5C,SAAS;QACb,CAAC;QAED,yDAAyD;QACzD,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;QAEzC,GAAG,CAAC,QAAQ,CAAC,CAAC,iDAAiD,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1G,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAEjD,qFAAqF;QACrF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,WAAW,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IAE9C,IAAI,MAAM,WAAW,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,YAAY,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,YAAY,EAAE,CAAC;AACzB,CAAC;AA4ND,MAAM,UAAU,sBAAsB,CAClC,OAAkC,EAClC,cAAwC;IAExC,OAAO,CAAC,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAE,OAA2B,EAAE,EAAE,CACzE,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChD,OAAO,CAAC,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,YAAY,CAAC,UAAU,EAAE,CAAC;YACpD,GAAG,CAAC,OAAO,CACP,mIAAmI,CACtI,CAAC;YACF,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO;QACX,CAAC;QACD,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC;IACF,OAAO,CAAC,aAAa,GAAG,KAAK,EAAE,OAA8B,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvG,OAAO,CAAC,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;QACpE,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;QACvD,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IACrE,CAAC,CAAC;IACF,OAAO,CAAC,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;QACtE,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACvD,CAAC;QAED,OAAO,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,cAAc,CAAC,iBAAiB,EAAE,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1G,CAAC,CAAC;IACF,OAAO,CAAC,cAAc,GAAG,KAAK,EAAE,OAA+B,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1G,OAAO,CAAC,YAAY,GAAG,KAAK,EAAE,OAA6B,EAAE,EAAE,CAC3D,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,OAAO,CAAC,8BAA8B,GAAG,KAAK,EAC1C,OAA6E,EAC/E,EAAE,CACA,8BAA8B,CAAC;QAC3B,GAAG,OAAO;QACV,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,YAAa;KAC9C,CAAC,CAAC;IACP,OAAO,CAAC,aAAa,GAAG,CAAC,YAAoB,EAAE,GAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACrG,OAAO,CAAC,iBAAiB,GAAG,KAAK,IAAI,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,OAAO,CAAC,yBAAyB,GAAG,KAAK,EAAE,OAA0C,EAAE,EAAE;QACrF,OAAO,yBAAyB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClG,CAAC,CAAC;AACN,CAAC;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,MAAM,CAAC,MAAM,eAAe,GAAG;IAC3B,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,8BAA8B;IAC9B,gBAAgB;IAChB,cAAc;IACd,YAAY;IACZ,aAAa;IACb,iBAAiB;IACjB,sBAAsB;IACtB,yBAAyB;CAC5B,CAAC"}
|
|
1
|
+
{"version":3,"file":"playwright-utils.js","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,aAAa,EAAE,aAAa,EAAgB,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAExH,OAAO,EAAqC,iBAAiB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjD,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,cAAc,CAAC,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAEpE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE7C,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AA0BtH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,QAAgB,EAAE,UAA6B,EAAE;IAC1F,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IACxB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CACL,CAAC;IAEF,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,gBAAgB,EAAE,KAAK,IAAI,EAAE,CACjC,IAAI;aACC,QAAQ,CAAC,QAAQ,CAAC;aAClB,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CACvG,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAA0C;IACrF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,IAAI,IAAI,EAAE,CAAC,CAAC;AACrG,CAAC;AAyBD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,IAAU,EACV,OAAgB,EAChB,cAAuC,EAAE;IAEzC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;KACrD,CAAC,CACL,CAAC;IACF,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;IAElE,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,sDAAsD;QACtD,MAAM,EAAE,CAAC,UAAU,CACf,+GAA+G;YAC3G,4DAA4D,CACnE,CAAC;QACF,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,KAAY,EAAE,EAAE;YACnD,IAAI,CAAC;gBACD,oGAAoG;gBACpG,uDAAuD;gBACvD,IAAI,SAAS,EAAE,CAAC;oBACZ,OAAO,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClC,CAAC;gBAED,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM,SAAS,GAAe,EAAE,CAAC;gBAEjC,IAAI,MAAM,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChD,IAAI,OAAO;oBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;gBACnD,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAClE,CAAC;YAED,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,UAAgC,EAAE;IAC9E,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,WAAW,GAAG,kCAAkC,EAAE,gBAAgB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE5F,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAExD,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpC,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACL,MAAM,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IAC3F,CAAC;AACL,CAAC;AASD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,aAAa,CAAC,YAAoB,EAAE,UAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzF,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACD,IAAI,GAAG,EAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;IACtI,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,EAAE,CAAC,SAAS,CAAC,GAAY,EAAE,wBAAwB,CAAC,CAAC;QAC3D,MAAM,GAAG,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAEtH,OAAO,IAAI,CAAC;AAChB,CAAC;AAsCD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,UAAiC,EAAE;IAChF,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACnC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CACL,CAAC;IAEF,MAAM,EACF,WAAW,GAAG,CAAC,EACf,eAAe,GAAG,CAAC,EACnB,WAAW,GAAG,CAAC,EACf,eAAe,GAAG,KAAK,EACvB,cAAc,EACd,kBAAkB,GACrB,GAAG,OAAO,CAAC;IAEZ,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;YAChE,cAAc,CAAC,YAAY,EAAE,CAAC;QAClC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;YAC9D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;gBAC5C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;YACX,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9D,CAAC;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;YACrE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,8CAA8C;QAC9C,IAAI,eAAe,GAAG,CAAC,IAAI,gBAAgB,IAAI,eAAe,EAAE,CAAC;YAC7D,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,gBAAgB,IAAI,KAAK,CAAC;IAC9B,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAe,CAAC,CAAC;QAC7C,sDAAsD;QACtD,IAAI,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,gBAAgB,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACrB,IAAI,MAAM,kBAAkB,EAAE,EAAE,CAAC;gBAC7B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAwCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,UAA+B,EAAE;IAC5E,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC7B,CAAC,CACL,CAAC;IAEF,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,EACjB,MAAM,GACT,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE;YACtD,MAAM,EAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE;SACpD,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC;gBAC3C,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,iBAAiB;gBAC1B,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,UAAU;aACzB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7F,CAAC;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,IAAU,EACV,iBAAiB,GAAG,KAAK,EACzB,aAAa,GAAG,KAAK;IAErB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAErD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACvB,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;gBAE1C,IAAI,MAAM,EAAE,CAAC;oBACT,MAAM,aAAa,GAAG,KAAK,IAAqB,EAAE;wBAC9C,IAAI,CAAC;4BACD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,CAAC;wBACtD,CAAC;wBAAC,MAAM,CAAC;4BACL,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC5B,CAAC;oBACL,CAAC,CAAC;oBAEF,MAAM,QAAQ,GAAG,MAAM,aAAa,EAAE,CAAC;oBAEvC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;wBAC1B,MAAM,eAAe,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;wBACtD,eAAe,CAAC,SAAS,GAAG,CAAC,CAAC;wBAC9B,eAAe,CAAC,SAAS,GAAG,4BAA4B,CAAC;wBAEzD,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;oBACnC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC,OAAO,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAC;YACnE,CAAC;QACL,CAAC,CAAC,CACL,CAAC;IACN,CAAC;IAED,MAAM,IAAI,GAAG,iBAAiB;QAC1B,CAAC,CAAC,IAAI;QACN,CAAC,CAAE,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAY,CAAC;IACvF,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnD,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,IAAI,eAAe,GAAiD,IAAI,CAAC;AACzE,KAAK,UAAU,kBAAkB;IAC7B,IAAI,eAAe;QAAE,OAAO,eAAe,CAAC;IAE5C,IAAI,CAAC;QACD,eAAe,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,OAAO,CAAC;;;;;;;;;EASvB,KAAK,CAAC,OAAO;CACd,CAAC,CAAC;IACC,CAAC;IACD,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAC9C,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAEzC,IAAI,KAAK,EAAE,mBAAmB,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACrD,CAAC;AACL,CAAC;AAmBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,KAAK,UAAU,yBAAyB,CACpC,IAAU,EACV,GAAW,EACX,UAA4C,EAAE,EAC9C,kBAAgC;IAEhC,4GAA4G;IAC5G,MAAM,MAAM,GAAG,kBAAkB,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5C,kBAAkB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAEhC,OAAO,CAAC,iBAAiB,KAAK,KAAK,IAAI,EAAE;QACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACvC,OAAO,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC;QACtG,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,CAAC,SAAS,CAAC;IACvB,CAAC,CAAC;IAEF,OAAO,CAAC,mBAAmB,KAAK,KAAK,IAAI,EAAE;QACvC,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE;YAClC,OAAO,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,yDAAyD,CAAC,CAAC;QAC/F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC5B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,iBAAkB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAE5E,IAAI,SAAS,EAAE,CAAC;YACZ,MAAM,IAAI,YAAY,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;IACL,CAAC,CAAC;IAEF,gDAAgD;IAChD,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;QAC3B,OAAO,OAAO,CAAC,mBAAoB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC,CAAC;IAEF,IAAI,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,CAAC;YACzB,MAAM,YAAY,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QACpD,MAAM,EAAE,CAAC,QAAQ,CAAC,CACd,oCAAoC,GAAG,6CAA6C,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,WAAW,CAChI,CAAC;QAEF,MAAM,EAAE,GAAG,MAAM,IAAI;aAChB,QAAQ,CAAC,GAAG,EAAE;YACX,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;YACxD,OAAO,GAAG,EAAE,qBAAqB,EAAE,CAAC;QACxC,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAE5B,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,CAAC,KAAa,EAAE,EAAE;YACnC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC;QACzD,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QAElB,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1B,MAAM,KAAK,CAAC,CAAC,OAAO,CAAC,qBAAqB,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAEzD,2DAA2D;YAC3D,IAAI,CAAC,CAAC,MAAM,WAAW,EAAE,CAAC,EAAE,CAAC;gBACzB,MAAM;YACV,CAAC;YAED,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;gBAChC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;gBACtD,IAAI,GAAG,EAAE,CAAC;oBACN,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;oBACV,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;gBACd,CAAC;YACL,CAAC;YAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,MAAM,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACb,CAAC;YAED,yDAAyD;YACzD,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YACzC,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YAEzC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,iDAAiD,GAAG,EAAE,EAAE;gBACvE,CAAC,EAAE,WAAW;gBACd,CAAC,EAAE,WAAW;aACjB,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAEjD,qFAAqF;YACrF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,WAAW,GAAG,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAE9C,IAAI,MAAM,WAAW,EAAE,EAAE,CAAC;YACtB,MAAM,IAAI,YAAY,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,YAAY,EAAE,CAAC;IACzB,CAAC;YAAS,CAAC;QACP,IAAI,MAAM,EAAE,CAAC;YACT,kBAAkB,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;AACL,CAAC;AAqOD,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,MAAM,CAAC,MAAM,eAAe,GAAG;IAC3B,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,8BAA8B;IAC9B,gBAAgB;IAChB,cAAc;IACd,YAAY;IACZ,aAAa;IACb,iBAAiB;IACjB,sBAAsB;IACtB,yBAAyB;CAC5B,CAAC"}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { Request } from '@crawlee/core';
|
|
1
|
+
import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
|
|
2
2
|
export type RenderingType = 'clientOnly' | 'static';
|
|
3
3
|
type URLComponents = string[];
|
|
4
4
|
type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
|
|
5
5
|
export interface RenderingTypePredictorOptions {
|
|
6
6
|
/** A number between 0 and 1 that determines the desired ratio of rendering type detections */
|
|
7
7
|
detectionRatio: number;
|
|
8
|
+
persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
|
|
8
9
|
}
|
|
9
10
|
/**
|
|
10
11
|
* Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
|
|
@@ -12,10 +13,13 @@ export interface RenderingTypePredictorOptions {
|
|
|
12
13
|
* @experimental
|
|
13
14
|
*/
|
|
14
15
|
export declare class RenderingTypePredictor {
|
|
15
|
-
private renderingTypeDetectionResults;
|
|
16
16
|
private detectionRatio;
|
|
17
|
-
private
|
|
18
|
-
constructor({ detectionRatio }: RenderingTypePredictorOptions);
|
|
17
|
+
private state;
|
|
18
|
+
constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
|
|
19
|
+
/**
|
|
20
|
+
* Initialize the predictor by restoring persisted state.
|
|
21
|
+
*/
|
|
22
|
+
initialize(): Promise<void>;
|
|
19
23
|
/**
|
|
20
24
|
* Predict the rendering type for a given URL and request label.
|
|
21
25
|
*/
|
|
@@ -26,7 +30,7 @@ export declare class RenderingTypePredictor {
|
|
|
26
30
|
/**
|
|
27
31
|
* Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
|
|
28
32
|
*/
|
|
29
|
-
storeResult(
|
|
33
|
+
storeResult(requests: Request | Request[], renderingType: RenderingType): void;
|
|
30
34
|
private resultCount;
|
|
31
35
|
protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
|
|
32
36
|
protected retrain(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rendering-type-prediction.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"rendering-type-prediction.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kCAAkC,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAMjF,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,QAAQ,CAAC;AAEpD,KAAK,aAAa,GAAG,MAAM,EAAE,CAAC;AAuB9B,KAAK,aAAa,GAAG,CAAC,uBAAuB,EAAE,MAAM,EAAE,2BAA2B,EAAE,MAAM,CAAC,CAAC;AAE5F,MAAM,WAAW,6BAA6B;IAC1C,8FAA8F;IAC9F,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,CAAC,EAAE,OAAO,CAAC,kCAAkC,CAAC,CAAC;CACpE;AAED;;;;GAIG;AACH,qBAAa,sBAAsB;IAC/B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,KAAK,CAGV;gBAES,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,6BAA6B;IAuCjF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC;;OAEG;IACI,OAAO,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,OAAO,GAAG;QAChD,aAAa,EAAE,aAAa,CAAC;QAC7B,kCAAkC,EAAE,MAAM,CAAC;KAC9C;IAqBD;;OAEG;IACI,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,EAAE,EAAE,aAAa,EAAE,aAAa;IAoB9E,OAAO,CAAC,WAAW;IAMnB,SAAS,CAAC,sBAAsB,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa;IAe9F,SAAS,CAAC,OAAO,IAAI,IAAI;CAkB5B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { RecoverableState } from '@crawlee/core';
|
|
1
2
|
import LogisticRegression from 'ml-logistic-regression';
|
|
2
3
|
import { Matrix } from 'ml-matrix';
|
|
3
4
|
import stringComparison from 'string-comparison';
|
|
@@ -22,27 +23,58 @@ const mean = (values) => (values.length > 0 ? sum(values) / values.length : unde
|
|
|
22
23
|
* @experimental
|
|
23
24
|
*/
|
|
24
25
|
export class RenderingTypePredictor {
|
|
25
|
-
renderingTypeDetectionResults = new Map();
|
|
26
26
|
detectionRatio;
|
|
27
|
-
|
|
28
|
-
constructor({ detectionRatio }) {
|
|
27
|
+
state;
|
|
28
|
+
constructor({ detectionRatio, persistenceOptions }) {
|
|
29
29
|
this.detectionRatio = detectionRatio;
|
|
30
|
-
this.
|
|
30
|
+
this.state = new RecoverableState({
|
|
31
|
+
defaultState: {
|
|
32
|
+
logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
|
|
33
|
+
detectionResults: new Map(),
|
|
34
|
+
},
|
|
35
|
+
serialize: (state) => JSON.stringify({
|
|
36
|
+
logreg: state.logreg.toJSON(),
|
|
37
|
+
detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
|
|
38
|
+
renderingType,
|
|
39
|
+
urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
|
|
40
|
+
label,
|
|
41
|
+
urlParts,
|
|
42
|
+
})),
|
|
43
|
+
})),
|
|
44
|
+
}),
|
|
45
|
+
deserialize: (serializedState) => {
|
|
46
|
+
const { logreg, detectionResults = [] } = JSON.parse(serializedState);
|
|
47
|
+
return {
|
|
48
|
+
logreg: LogisticRegression.load(logreg),
|
|
49
|
+
detectionResults: new Map(detectionResults.map((serializedItem) => [
|
|
50
|
+
serializedItem.renderingType,
|
|
51
|
+
new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
|
|
52
|
+
])),
|
|
53
|
+
};
|
|
54
|
+
},
|
|
55
|
+
persistStateKey: 'rendering-type-predictor-state',
|
|
56
|
+
persistenceEnabled: true,
|
|
57
|
+
...persistenceOptions,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Initialize the predictor by restoring persisted state.
|
|
62
|
+
*/
|
|
63
|
+
async initialize() {
|
|
64
|
+
await this.state.initialize();
|
|
31
65
|
}
|
|
32
66
|
/**
|
|
33
67
|
* Predict the rendering type for a given URL and request label.
|
|
34
68
|
*/
|
|
35
69
|
predict({ url, loadedUrl, label }) {
|
|
36
|
-
|
|
70
|
+
const { logreg } = this.state.currentValue;
|
|
71
|
+
if (logreg.classifiers.length === 0) {
|
|
37
72
|
return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
|
|
38
73
|
}
|
|
39
74
|
const predictionUrl = new URL(loadedUrl ?? url);
|
|
40
75
|
const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
|
|
41
|
-
const [prediction] =
|
|
42
|
-
const scores = [
|
|
43
|
-
this.logreg.classifiers[0].testScores(urlFeature),
|
|
44
|
-
this.logreg.classifiers[1].testScores(urlFeature),
|
|
45
|
-
];
|
|
76
|
+
const [prediction] = logreg.predict(urlFeature);
|
|
77
|
+
const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
|
|
46
78
|
return {
|
|
47
79
|
renderingType: prediction === 1 ? 'static' : 'clientOnly',
|
|
48
80
|
detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
|
|
@@ -53,26 +85,29 @@ export class RenderingTypePredictor {
|
|
|
53
85
|
/**
|
|
54
86
|
* Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
|
|
55
87
|
*/
|
|
56
|
-
storeResult(
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
88
|
+
storeResult(requests, renderingType) {
|
|
89
|
+
const state = this.state.currentValue;
|
|
90
|
+
for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
|
|
91
|
+
const resultUrl = new URL(loadedUrl ?? url);
|
|
92
|
+
if (!state.detectionResults.has(renderingType)) {
|
|
93
|
+
state.detectionResults.set(renderingType, new Map());
|
|
94
|
+
}
|
|
95
|
+
if (!state.detectionResults.get(renderingType).has(label)) {
|
|
96
|
+
state.detectionResults.get(renderingType).set(label, []);
|
|
97
|
+
}
|
|
98
|
+
state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
|
|
63
99
|
}
|
|
64
|
-
this.renderingTypeDetectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
|
|
65
100
|
this.retrain();
|
|
66
101
|
}
|
|
67
102
|
resultCount(label) {
|
|
68
|
-
return Array.from(this.
|
|
103
|
+
return Array.from(this.state.currentValue.detectionResults.values())
|
|
69
104
|
.map((results) => results.get(label)?.length ?? 0)
|
|
70
105
|
.reduce((acc, value) => acc + value, 0);
|
|
71
106
|
}
|
|
72
107
|
calculateFeatureVector(url, label) {
|
|
73
108
|
return [
|
|
74
|
-
mean((this.
|
|
75
|
-
mean((this.
|
|
109
|
+
mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
|
|
110
|
+
mean((this.state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
|
|
76
111
|
];
|
|
77
112
|
}
|
|
78
113
|
retrain() {
|
|
@@ -81,7 +116,7 @@ export class RenderingTypePredictor {
|
|
|
81
116
|
[1, 0],
|
|
82
117
|
];
|
|
83
118
|
const Y = [0, 1];
|
|
84
|
-
for (const [renderingType, urlsByLabel] of this.
|
|
119
|
+
for (const [renderingType, urlsByLabel] of this.state.currentValue.detectionResults.entries()) {
|
|
85
120
|
for (const [label, urls] of urlsByLabel) {
|
|
86
121
|
for (const url of urls) {
|
|
87
122
|
X.push(this.calculateFeatureVector(url, label));
|
|
@@ -89,7 +124,7 @@ export class RenderingTypePredictor {
|
|
|
89
124
|
}
|
|
90
125
|
}
|
|
91
126
|
}
|
|
92
|
-
this.logreg.train(new Matrix(X), Matrix.columnVector(Y));
|
|
127
|
+
this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
|
|
93
128
|
}
|
|
94
129
|
}
|
|
95
130
|
//# sourceMappingURL=rendering-type-prediction.js.map
|