@crawlee/playwright 3.7.4-beta.8 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +2 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/index.mjs +3 -0
- package/internals/adaptive-playwright-crawler.d.ts +112 -0
- package/internals/adaptive-playwright-crawler.d.ts.map +1 -0
- package/internals/adaptive-playwright-crawler.js +269 -0
- package/internals/adaptive-playwright-crawler.js.map +1 -0
- package/internals/utils/playwright-utils.d.ts +2 -0
- package/internals/utils/playwright-utils.d.ts.map +1 -1
- package/internals/utils/playwright-utils.js +2 -0
- package/internals/utils/playwright-utils.js.map +1 -1
- package/internals/utils/rendering-type-prediction.d.ts +34 -0
- package/internals/utils/rendering-type-prediction.d.ts.map +1 -0
- package/internals/utils/rendering-type-prediction.js +108 -0
- package/internals/utils/rendering-type-prediction.js.map +1 -0
- package/package.json +11 -6
- package/tsconfig.build.tsbuildinfo +1 -1
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
14
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
15
15
|
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
|
|
16
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-
|
|
16
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
|
|
17
17
|
</p>
|
|
18
18
|
|
|
19
19
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
package/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
2
|
export * from './internals/playwright-crawler';
|
|
3
3
|
export * from './internals/playwright-launcher';
|
|
4
|
+
export * from './internals/adaptive-playwright-crawler';
|
|
4
5
|
export * as playwrightUtils from './internals/utils/playwright-utils';
|
|
5
6
|
export * as playwrightClickElements from './internals/enqueue-links/click-elements';
|
|
6
7
|
export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils';
|
|
8
|
+
export type { RenderingType } from './internals/utils/rendering-type-prediction';
|
|
7
9
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,iCAAiC,CAAC;AAChD,cAAc,yCAAyC,CAAC;AAExD,OAAO,KAAK,eAAe,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,uBAAuB,MAAM,0CAA0C,CAAC;AACpF,YAAY,EAAE,uBAAuB,IAAI,iCAAiC,EAAE,MAAM,oCAAoC,CAAC;AACvH,YAAY,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAC"}
|
package/index.js
CHANGED
|
@@ -5,6 +5,7 @@ const tslib_1 = require("tslib");
|
|
|
5
5
|
tslib_1.__exportStar(require("@crawlee/browser"), exports);
|
|
6
6
|
tslib_1.__exportStar(require("./internals/playwright-crawler"), exports);
|
|
7
7
|
tslib_1.__exportStar(require("./internals/playwright-launcher"), exports);
|
|
8
|
+
tslib_1.__exportStar(require("./internals/adaptive-playwright-crawler"), exports);
|
|
8
9
|
exports.playwrightUtils = tslib_1.__importStar(require("./internals/utils/playwright-utils"));
|
|
9
10
|
exports.playwrightClickElements = tslib_1.__importStar(require("./internals/enqueue-links/click-elements"));
|
|
10
11
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,yEAA+C;AAC/C,0EAAgD;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,2DAAiC;AACjC,yEAA+C;AAC/C,0EAAgD;AAChD,kFAAwD;AAExD,8FAAsE;AACtE,4GAAoF"}
|
package/index.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import mod from "./index.js";
|
|
|
2
2
|
|
|
3
3
|
export default mod;
|
|
4
4
|
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
|
|
5
|
+
export const AdaptivePlaywrightCrawler = mod.AdaptivePlaywrightCrawler;
|
|
5
6
|
export const AutoscaledPool = mod.AutoscaledPool;
|
|
6
7
|
export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
|
|
7
8
|
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
@@ -38,6 +39,7 @@ export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
|
38
39
|
export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
|
|
39
40
|
export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
40
41
|
export const Request = mod.Request;
|
|
42
|
+
export const RequestHandlerResult = mod.RequestHandlerResult;
|
|
41
43
|
export const RequestList = mod.RequestList;
|
|
42
44
|
export const RequestProvider = mod.RequestProvider;
|
|
43
45
|
export const RequestQueue = mod.RequestQueue;
|
|
@@ -69,6 +71,7 @@ export const createRequestOptions = mod.createRequestOptions;
|
|
|
69
71
|
export const createRequests = mod.createRequests;
|
|
70
72
|
export const deserializeArray = mod.deserializeArray;
|
|
71
73
|
export const enqueueLinks = mod.enqueueLinks;
|
|
74
|
+
export const extractUrlsFromPage = mod.extractUrlsFromPage;
|
|
72
75
|
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
73
76
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
74
77
|
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { RestrictedCrawlingContext, StatisticState } from '@crawlee/core';
|
|
2
|
+
import { Configuration, RequestHandlerResult, Statistics } from '@crawlee/core';
|
|
3
|
+
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
4
|
+
import { type Cheerio, type Element } from 'cheerio';
|
|
5
|
+
import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from './playwright-crawler';
|
|
6
|
+
import { PlaywrightCrawler } from './playwright-crawler';
|
|
7
|
+
import { RenderingTypePredictor } from './utils/rendering-type-prediction';
|
|
8
|
+
type Result<TResult> = {
|
|
9
|
+
result: TResult;
|
|
10
|
+
ok: true;
|
|
11
|
+
} | {
|
|
12
|
+
error: unknown;
|
|
13
|
+
ok: false;
|
|
14
|
+
};
|
|
15
|
+
interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
|
|
16
|
+
httpOnlyRequestHandlerRuns?: number;
|
|
17
|
+
browserRequestHandlerRuns?: number;
|
|
18
|
+
renderingTypeMispredictions?: number;
|
|
19
|
+
}
|
|
20
|
+
declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
|
|
21
|
+
state: AdaptivePlaywrightCrawlerStatisticState;
|
|
22
|
+
reset(): void;
|
|
23
|
+
protected _maybeLoadStatistics(): Promise<void>;
|
|
24
|
+
trackHttpOnlyRequestHandlerRun(): void;
|
|
25
|
+
trackBrowserRequestHandlerRun(): void;
|
|
26
|
+
trackRenderingTypeMisprediction(): void;
|
|
27
|
+
}
|
|
28
|
+
interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingContext {
|
|
29
|
+
/**
|
|
30
|
+
* Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
|
|
31
|
+
*/
|
|
32
|
+
querySelector: (selector: string, timeoutMs?: number) => Awaitable<Cheerio<Element>>;
|
|
33
|
+
}
|
|
34
|
+
export interface AdaptivePlaywrightCrawlerOptions extends Omit<PlaywrightCrawlerOptions, 'requestHandler' | 'handlePageFunction'> {
|
|
35
|
+
/**
|
|
36
|
+
* Function that is called to process each request.
|
|
37
|
+
*
|
|
38
|
+
* The function receives the {@apilink AdaptivePlaywrightCrawlingContext} as an argument, and it must refrain from calling code with side effects,
|
|
39
|
+
* other than the methods of the crawling context. Any other side effects may be invoked repeatedly by the crawler, which can lead to inconsistent results.
|
|
40
|
+
*
|
|
41
|
+
* The function must return a promise, which is then awaited by the crawler.
|
|
42
|
+
*
|
|
43
|
+
* If the function throws an exception, the crawler will try to re-crawl the
|
|
44
|
+
* request later, up to `option.maxRequestRetries` times.
|
|
45
|
+
*/
|
|
46
|
+
requestHandler: (crawlingContext: AdaptivePlaywrightCrawlerContext) => Awaitable<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
|
|
49
|
+
*/
|
|
50
|
+
renderingTypeDetectionRatio: number;
|
|
51
|
+
/**
|
|
52
|
+
* An optional callback that is called on dataset items found by the request handler in plain HTTP mode.
|
|
53
|
+
* If it returns false, the request is retried in a browser.
|
|
54
|
+
* If no callback is specified, every dataset item is considered valid.
|
|
55
|
+
*/
|
|
56
|
+
resultChecker?: (result: RequestHandlerResult) => boolean;
|
|
57
|
+
/**
|
|
58
|
+
* An optional callback used in rendering type detection. On each detection, the result of the plain HTTP run is compared to that of the browser one.
|
|
59
|
+
* If the callback returns true, the results are considered equal and the target site is considered static.
|
|
60
|
+
* If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
|
|
61
|
+
* If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
|
|
62
|
+
*/
|
|
63
|
+
resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean;
|
|
64
|
+
/**
|
|
65
|
+
* A custom rendering type predictor
|
|
66
|
+
*/
|
|
67
|
+
renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult'>;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* An extension of {@apilink PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
|
|
71
|
+
*
|
|
72
|
+
* **Example usage:**
|
|
73
|
+
*
|
|
74
|
+
* ```javascript
|
|
75
|
+
* const crawler = new AdaptivePlaywrightCrawler({
|
|
76
|
+
* renderingTypeDetectionRatio: 0.1,
|
|
77
|
+
* async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
|
|
78
|
+
* // This function is called to extract data from a single web page
|
|
79
|
+
* const $prices = await querySelector('span.price')
|
|
80
|
+
*
|
|
81
|
+
* await pushData({
|
|
82
|
+
* url: request.url,
|
|
83
|
+
* price: $prices.filter(':contains("$")').first().text(),
|
|
84
|
+
* })
|
|
85
|
+
*
|
|
86
|
+
* await enqueueLinks({ selector: '.pagination a' })
|
|
87
|
+
* },
|
|
88
|
+
* });
|
|
89
|
+
*
|
|
90
|
+
* await crawler.run([
|
|
91
|
+
* 'http://www.example.com/page-1',
|
|
92
|
+
* 'http://www.example.com/page-2',
|
|
93
|
+
* ]);
|
|
94
|
+
* ```
|
|
95
|
+
*
|
|
96
|
+
* @experimental
|
|
97
|
+
*/
|
|
98
|
+
export declare class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
99
|
+
readonly config: Configuration;
|
|
100
|
+
private adaptiveRequestHandler;
|
|
101
|
+
private renderingTypePredictor;
|
|
102
|
+
private resultChecker;
|
|
103
|
+
private resultComparator;
|
|
104
|
+
readonly stats: AdaptivePlaywrightCrawlerStatistics;
|
|
105
|
+
constructor({ requestHandler, renderingTypeDetectionRatio, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, ...options }: AdaptivePlaywrightCrawlerOptions, config?: Configuration);
|
|
106
|
+
protected _runRequestHandler(crawlingContext: PlaywrightCrawlingContext<Dictionary>): Promise<void>;
|
|
107
|
+
protected commitResult(crawlingContext: PlaywrightCrawlingContext<Dictionary>, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
|
|
108
|
+
protected runRequestHandlerInBrowser(crawlingContext: PlaywrightCrawlingContext<Dictionary>): Promise<Result<RequestHandlerResult>>;
|
|
109
|
+
protected runRequestHandlerWithPlainHTTP(crawlingContext: PlaywrightCrawlingContext<Dictionary>): Promise<Result<RequestHandlerResult>>;
|
|
110
|
+
}
|
|
111
|
+
export {};
|
|
112
|
+
//# sourceMappingURL=adaptive-playwright-crawler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,yBAAyB,EAAE,cAAc,EAA2B,MAAM,eAAe,CAAC;AACxG,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5D,OAAO,EAAQ,KAAK,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AAG3D,OAAO,KAAK,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAsB,MAAM,mCAAmC,CAAC;AAE/F,KAAK,MAAM,CAAC,OAAO,IAAI;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,KAAK,CAAA;CAAE,CAAA;AAEpF,UAAU,uCAAwC,SAAQ,cAAc;IACpE,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACxC;AAQD,cAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,EAAE,uCAAuC,CAAe;IAE7D,KAAK,IAAI,IAAI;cAOG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAa9D,8BAA8B,IAAI,IAAI;IAKtC,6BAA6B,IAAI,IAAI;IAKrC,+BAA+B,IAAI,IAAI;CAI1C;AAED,UAAU,gCAAiC,SAAQ,yBAAyB;IACxE;;OAEG;IACH,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;CACxF;AAED,MAAM,WAAW,gCAAiC,SAAQ,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,oBAAoB,CAAC;IAC7H;;;;;;;;;;OAUG;IACH,cAAc,EAAE,CAAC,eAAe,EAAE,gCAAgC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;IAEvF;;OAEG;IACH,2BAA2B,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE1D;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE7F;;OAEG;IACH,sBAAsB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;CACpF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAA0B,SAAQ,iBAAiB;aAiBtC,MAAM;IAhB5B,OAAO,CAAC,sBAAsB,CAAqD;IACnF,OAAO,CAAC,sBAAsB,CAA0E;IACxG,OAAO,CAAC,aAAa,CAAiE;IACtF,OAAO,CAAC,gBAAgB,CAAoE;IAC5F,SAAkB,KAAK,EAAE,mCAAmC,CAAC;gBAGzD,EACI,cAAc,EACd,2BAA2B,EAC3B,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,GAAG,OAAO,EACb,EAAE,gCAAgC,EACjB,MAAM,gBAAkC;cA4BrC,kBAAkB,CAAC,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;cA2DlG,YAAY,CACxB,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,EACtD,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GACtD,OAAO,CAAC,IAAI,CAAC;cAcA,0BAA0B,CAAC,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;cA6CzH,8BAA8B,CAAC,eAAe,EAAE,yBAAyB,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;CAgChJ"}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AdaptivePlaywrightCrawler = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const timeout_1 = require("@apify/timeout");
|
|
6
|
+
const browser_1 = require("@crawlee/browser");
|
|
7
|
+
const core_1 = require("@crawlee/core");
|
|
8
|
+
const utils_1 = require("@crawlee/utils");
|
|
9
|
+
const cheerio_1 = require("cheerio");
|
|
10
|
+
const lodash_isequal_1 = tslib_1.__importDefault(require("lodash.isequal"));
|
|
11
|
+
const playwright_crawler_1 = require("./playwright-crawler");
|
|
12
|
+
const rendering_type_prediction_1 = require("./utils/rendering-type-prediction");
|
|
13
|
+
class AdaptivePlaywrightCrawlerStatistics extends core_1.Statistics {
|
|
14
|
+
constructor() {
|
|
15
|
+
super(...arguments);
|
|
16
|
+
Object.defineProperty(this, "state", {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true,
|
|
20
|
+
value: null
|
|
21
|
+
}); // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
|
|
22
|
+
}
|
|
23
|
+
reset() {
|
|
24
|
+
super.reset();
|
|
25
|
+
this.state.httpOnlyRequestHandlerRuns = 0;
|
|
26
|
+
this.state.browserRequestHandlerRuns = 0;
|
|
27
|
+
this.state.renderingTypeMispredictions = 0;
|
|
28
|
+
}
|
|
29
|
+
async _maybeLoadStatistics() {
|
|
30
|
+
await super._maybeLoadStatistics();
|
|
31
|
+
const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
|
|
32
|
+
if (!savedState) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
this.state.httpOnlyRequestHandlerRuns = savedState.httpOnlyRequestHandlerRuns;
|
|
36
|
+
this.state.browserRequestHandlerRuns = savedState.browserRequestHandlerRuns;
|
|
37
|
+
this.state.renderingTypeMispredictions = savedState.renderingTypeMispredictions;
|
|
38
|
+
}
|
|
39
|
+
trackHttpOnlyRequestHandlerRun() {
|
|
40
|
+
var _a;
|
|
41
|
+
(_a = this.state).httpOnlyRequestHandlerRuns ?? (_a.httpOnlyRequestHandlerRuns = 0);
|
|
42
|
+
this.state.httpOnlyRequestHandlerRuns += 1;
|
|
43
|
+
}
|
|
44
|
+
trackBrowserRequestHandlerRun() {
|
|
45
|
+
var _a;
|
|
46
|
+
(_a = this.state).browserRequestHandlerRuns ?? (_a.browserRequestHandlerRuns = 0);
|
|
47
|
+
this.state.browserRequestHandlerRuns += 1;
|
|
48
|
+
}
|
|
49
|
+
trackRenderingTypeMisprediction() {
|
|
50
|
+
var _a;
|
|
51
|
+
(_a = this.state).renderingTypeMispredictions ?? (_a.renderingTypeMispredictions = 0);
|
|
52
|
+
this.state.renderingTypeMispredictions += 1;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* An extension of {@apilink PlaywrightCrawler} that uses a more limited request handler interface so that it is able to switch to HTTP-only crawling when it detects it may be possible.
|
|
57
|
+
*
|
|
58
|
+
* **Example usage:**
|
|
59
|
+
*
|
|
60
|
+
* ```javascript
|
|
61
|
+
* const crawler = new AdaptivePlaywrightCrawler({
|
|
62
|
+
* renderingTypeDetectionRatio: 0.1,
|
|
63
|
+
* async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
|
|
64
|
+
* // This function is called to extract data from a single web page
|
|
65
|
+
* const $prices = await querySelector('span.price')
|
|
66
|
+
*
|
|
67
|
+
* await pushData({
|
|
68
|
+
* url: request.url,
|
|
69
|
+
* price: $prices.filter(':contains("$")').first().text(),
|
|
70
|
+
* })
|
|
71
|
+
*
|
|
72
|
+
* await enqueueLinks({ selector: '.pagination a' })
|
|
73
|
+
* },
|
|
74
|
+
* });
|
|
75
|
+
*
|
|
76
|
+
* await crawler.run([
|
|
77
|
+
* 'http://www.example.com/page-1',
|
|
78
|
+
* 'http://www.example.com/page-2',
|
|
79
|
+
* ]);
|
|
80
|
+
* ```
|
|
81
|
+
*
|
|
82
|
+
* @experimental
|
|
83
|
+
*/
|
|
84
|
+
class AdaptivePlaywrightCrawler extends playwright_crawler_1.PlaywrightCrawler {
|
|
85
|
+
constructor({ requestHandler, renderingTypeDetectionRatio, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, ...options }, config = core_1.Configuration.getGlobalConfig()) {
|
|
86
|
+
super(options, config);
|
|
87
|
+
Object.defineProperty(this, "config", {
|
|
88
|
+
enumerable: true,
|
|
89
|
+
configurable: true,
|
|
90
|
+
writable: true,
|
|
91
|
+
value: config
|
|
92
|
+
});
|
|
93
|
+
Object.defineProperty(this, "adaptiveRequestHandler", {
|
|
94
|
+
enumerable: true,
|
|
95
|
+
configurable: true,
|
|
96
|
+
writable: true,
|
|
97
|
+
value: void 0
|
|
98
|
+
});
|
|
99
|
+
Object.defineProperty(this, "renderingTypePredictor", {
|
|
100
|
+
enumerable: true,
|
|
101
|
+
configurable: true,
|
|
102
|
+
writable: true,
|
|
103
|
+
value: void 0
|
|
104
|
+
});
|
|
105
|
+
Object.defineProperty(this, "resultChecker", {
|
|
106
|
+
enumerable: true,
|
|
107
|
+
configurable: true,
|
|
108
|
+
writable: true,
|
|
109
|
+
value: void 0
|
|
110
|
+
});
|
|
111
|
+
Object.defineProperty(this, "resultComparator", {
|
|
112
|
+
enumerable: true,
|
|
113
|
+
configurable: true,
|
|
114
|
+
writable: true,
|
|
115
|
+
value: void 0
|
|
116
|
+
});
|
|
117
|
+
Object.defineProperty(this, "stats", {
|
|
118
|
+
enumerable: true,
|
|
119
|
+
configurable: true,
|
|
120
|
+
writable: true,
|
|
121
|
+
value: void 0
|
|
122
|
+
});
|
|
123
|
+
this.adaptiveRequestHandler = requestHandler;
|
|
124
|
+
this.renderingTypePredictor = renderingTypePredictor ?? new rendering_type_prediction_1.RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
|
|
125
|
+
this.resultChecker = resultChecker ?? (() => true);
|
|
126
|
+
if (resultComparator !== undefined) {
|
|
127
|
+
this.resultComparator = resultComparator;
|
|
128
|
+
}
|
|
129
|
+
else if (resultChecker !== undefined) {
|
|
130
|
+
this.resultComparator = (resultA, resultB) => this.resultChecker(resultA) && this.resultChecker(resultB);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
this.resultComparator = (resultA, resultB) => {
|
|
134
|
+
return resultA.datasetItems.length === resultB.datasetItems.length
|
|
135
|
+
&& resultA.datasetItems.every((itemA, i) => {
|
|
136
|
+
const itemB = resultB.datasetItems[i];
|
|
137
|
+
return (0, lodash_isequal_1.default)(itemA, itemB);
|
|
138
|
+
});
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
this.stats = new AdaptivePlaywrightCrawlerStatistics({
|
|
142
|
+
logMessage: `${this.log.getOptions().prefix} request statistics:`,
|
|
143
|
+
config,
|
|
144
|
+
...statisticsOptions,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
async _runRequestHandler(crawlingContext) {
|
|
148
|
+
const url = new URL(crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
|
|
149
|
+
const renderingTypePrediction = this.renderingTypePredictor.predict(url, crawlingContext.request.label);
|
|
150
|
+
const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
|
|
151
|
+
if (!shouldDetectRenderingType) {
|
|
152
|
+
crawlingContext.log.info(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
|
|
153
|
+
}
|
|
154
|
+
if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
|
|
155
|
+
crawlingContext.log.info(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
|
|
156
|
+
this.stats.trackHttpOnlyRequestHandlerRun();
|
|
157
|
+
const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
|
|
158
|
+
if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
|
|
159
|
+
crawlingContext.log.info(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
|
|
160
|
+
await this.commitResult(crawlingContext, plainHTTPRun.result);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
if (!plainHTTPRun.ok) {
|
|
164
|
+
crawlingContext.log.exception(plainHTTPRun.error, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
|
|
168
|
+
this.stats.trackRenderingTypeMisprediction();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
crawlingContext.log.info(`Running browser request handler for ${crawlingContext.request.url}`);
|
|
172
|
+
this.stats.trackBrowserRequestHandlerRun();
|
|
173
|
+
const browserRun = await this.runRequestHandlerInBrowser(crawlingContext);
|
|
174
|
+
if (!browserRun.ok) {
|
|
175
|
+
throw browserRun.error;
|
|
176
|
+
}
|
|
177
|
+
await this.commitResult(crawlingContext, browserRun.result);
|
|
178
|
+
if (shouldDetectRenderingType) {
|
|
179
|
+
crawlingContext.log.info(`Detecting rendering type for ${crawlingContext.request.url}`);
|
|
180
|
+
const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
|
|
181
|
+
const detectionResult = (() => {
|
|
182
|
+
if (!plainHTTPRun.ok) {
|
|
183
|
+
return 'clientOnly';
|
|
184
|
+
}
|
|
185
|
+
if (this.resultComparator(plainHTTPRun.result, browserRun.result)) {
|
|
186
|
+
return 'static';
|
|
187
|
+
}
|
|
188
|
+
return 'clientOnly';
|
|
189
|
+
})();
|
|
190
|
+
crawlingContext.log.info(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
|
|
191
|
+
this.renderingTypePredictor.storeResult(url, crawlingContext.request.label, detectionResult);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
|
|
195
|
+
await Promise.all([
|
|
196
|
+
...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
|
|
197
|
+
...calls.enqueueLinks.map(async (params) => await crawlingContext.enqueueLinks(...params)),
|
|
198
|
+
...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
|
|
199
|
+
...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
|
|
200
|
+
const store = await crawlingContext.getKeyValueStore(storeIdOrName);
|
|
201
|
+
await Promise.all(Object.entries(changes).map(async ([key, { changedValue, options }]) => store.setValue(key, changedValue, options)));
|
|
202
|
+
}),
|
|
203
|
+
]);
|
|
204
|
+
}
|
|
205
|
+
async runRequestHandlerInBrowser(crawlingContext) {
|
|
206
|
+
const result = new core_1.RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
|
|
207
|
+
try {
|
|
208
|
+
await super._runRequestHandler.call(new Proxy(this, {
|
|
209
|
+
get: (target, propertyName, receiver) => {
|
|
210
|
+
if (propertyName === 'userProvidedRequestHandler') {
|
|
211
|
+
return (playwrightContext) => this.adaptiveRequestHandler({
|
|
212
|
+
request: crawlingContext.request,
|
|
213
|
+
log: crawlingContext.log,
|
|
214
|
+
querySelector: async (selector, timeoutMs) => {
|
|
215
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
216
|
+
await locator.waitFor({ timeout: timeoutMs });
|
|
217
|
+
return (await playwrightContext.parseWithCheerio())(selector);
|
|
218
|
+
},
|
|
219
|
+
enqueueLinks: async (options = {}) => {
|
|
220
|
+
const selector = options.selector ?? 'a';
|
|
221
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
222
|
+
await locator.waitFor();
|
|
223
|
+
const urls = await (0, browser_1.extractUrlsFromPage)(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl ?? playwrightContext.request.url);
|
|
224
|
+
await result.enqueueLinks({ ...options, urls });
|
|
225
|
+
},
|
|
226
|
+
addRequests: result.addRequests,
|
|
227
|
+
pushData: result.pushData,
|
|
228
|
+
useState: result.useState,
|
|
229
|
+
getKeyValueStore: result.getKeyValueStore,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
return Reflect.get(target, propertyName, receiver);
|
|
233
|
+
},
|
|
234
|
+
}), crawlingContext);
|
|
235
|
+
return { result, ok: true };
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
return { error, ok: false };
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
async runRequestHandlerWithPlainHTTP(crawlingContext) {
|
|
242
|
+
const result = new core_1.RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
|
|
243
|
+
const response = await crawlingContext.sendRequest({});
|
|
244
|
+
const loadedUrl = response.url;
|
|
245
|
+
crawlingContext.request.loadedUrl = loadedUrl;
|
|
246
|
+
const $ = (0, cheerio_1.load)(response.body);
|
|
247
|
+
try {
|
|
248
|
+
await (0, timeout_1.addTimeoutToPromise)(async () => this.adaptiveRequestHandler({
|
|
249
|
+
request: crawlingContext.request,
|
|
250
|
+
log: crawlingContext.log,
|
|
251
|
+
querySelector: (selector) => $(selector),
|
|
252
|
+
enqueueLinks: async (options = {}) => {
|
|
253
|
+
const urls = (0, utils_1.extractUrlsFromCheerio)($, options.selector, options.baseUrl ?? loadedUrl);
|
|
254
|
+
await result.enqueueLinks({ ...options, urls });
|
|
255
|
+
},
|
|
256
|
+
addRequests: result.addRequests,
|
|
257
|
+
pushData: result.pushData,
|
|
258
|
+
useState: result.useState,
|
|
259
|
+
getKeyValueStore: result.getKeyValueStore,
|
|
260
|
+
}), this.requestHandlerTimeoutInnerMillis, 'Request handler timed out');
|
|
261
|
+
return { result, ok: true };
|
|
262
|
+
}
|
|
263
|
+
catch (error) {
|
|
264
|
+
return { error, ok: false };
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
exports.AdaptivePlaywrightCrawler = AdaptivePlaywrightCrawler;
|
|
269
|
+
//# sourceMappingURL=adaptive-playwright-crawler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adaptive-playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":";;;;AAAA,4CAAqD;AACrD,8CAAuD;AAEvD,wCAAgF;AAEhF,0CAAwD;AACxD,qCAA2D;AAC3D,4EAAqC;AAGrC,6DAAyD;AACzD,iFAA+F;AAgB/F,MAAM,mCAAoC,SAAQ,iBAAU;IAA5D;;QACa;;;;mBAAiD,IAAW;WAAC,CAAC,+HAA+H;IAoC1M,CAAC;IAlCY,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC/C,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAmD,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9H,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,UAAU,CAAC,0BAA0B,CAAC;QAC9E,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;IACpF,CAAC;IAED,8BAA8B;;QAC1B,MAAA,IAAI,CAAC,KAAK,EAAC,0BAA0B,QAA1B,0BAA0B,GAAK,CAAC,EAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,6BAA6B;;QACzB,MAAA,IAAI,CAAC,KAAK,EAAC,yBAAyB,QAAzB,yBAAyB,GAAK,CAAC,EAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;;QAC3B,MAAA,IAAI,CAAC,KAAK,EAAC,2BAA2B,QAA3B,2BAA2B,GAAK,CAAC,EAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAiDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAa,yBAA0B,SAAQ,sCAAiB;IAO5D,YACI,EACI,cAAc,EACd,2BAA2B,EAC3B,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,GAAG,OAAO,EACqB,EACjB,SAAS,oBAAa,CAAC,eAAe,EAAE;QAE1D,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAFvB;;;;mBAAkB,MAAM;WAAkC;QAhBtD;;;;;WAA2E;QAC3E;;;;;WAAgG;QAChG;;;;;WAA8E;QAC9E;;;;;WAAoF;QAC1E;;;;;WAA2C;QAezD,IAAI,CAAC,sBAAsB,GAAG,cAAc,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,IAAI,IAAI,kDAAsB,CAAC,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;QACpI,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEnD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC7C,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,OAAO,OAAO,CAAC,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,MAAM;uBAC3D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;wBACvC,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;wBACtC,OAAO,IAAA,wBAAO,EAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBACjC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,mCAAmC,CAAC;YACjD,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB;YACjE,MAAM;YACN,GAAG,iBAAiB;SACvB,CAAC,CAAC;IACP,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAAsD;QAC9F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAEtF,MAAM,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxG,MAAM,yBAAyB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,uBAAuB,CAAC,kCAAkC,CAAC;QAE7G,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAC7B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,4BAA4B,uBAAuB,CAAC,aAAa,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACrI,CAAC;QAED,IAAI,uBAAuB,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnF,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,yCAAyC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACjG,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC;YAE5C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,8BAA8B,CAAC,eAAe,CAAC,CAAC;YAEhF,IAAI,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACrB,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,KAAc,EAAE,wCAAwC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACtI,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,8DAA8D,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACzH,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,uCAAuC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC;QAE3C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,eAAe,CAAC,CAAC;QAC1E,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,CAAC,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,yBAAyB,EAAE,CAAC;YAC5B,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACxF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,8BAA8B,CAAC,eAAe,CAAC,CAAC;YAEhF,MAAM,eAAe,GAAkB,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChE,OAAO,QAAQ,CAAC;gBACpB,CAAC;gBAED,OAAO,YAAY,CAAC;YACxB,CAAC,CAAC,EAAE,CAAC;YAEL,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,eAAe,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1G,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QACjG,CAAC;IACL,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,eAAsD,EACtD,EAAE,KAAK,EAAE,oBAAoB,EAAwB;QAErD,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5E,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1F,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC;YAClF,GAAG,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE;gBAC3E,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CACtH,CAAC;YACN,CAAC,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,eAAsD;QAC7F,MAAM,MAAM,GAAG,IAAI,2BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAElG,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAC/B,IAAI,KAAK,CAAC,IAAI,EAAE;gBACZ,GAAG,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE;oBACpC,IAAI,YAAY,KAAK,4BAA4B,EAAE,CAAC;wBAChD,OAAO,CAAC,iBAA4C,EAAE,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC;4BACjF,OAAO,EAAE,eAAe,CAAC,OAAO;4BAChC,GAAG,EAAE,eAAe,CAAC,GAAG;4BACxB,aAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;gCACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gCACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gCAC9C,OAAO,CAAC,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAqB,CAAC;4BACtF,CAAC;4BACD,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;gCACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;gCACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gCACjE,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gCAExB,MAAM,IAAI,GAAG,MAAM,IAAA,6BAAmB,EAClC,iBAAiB,CAAC,IAAI,EACtB,QAAQ,EACR,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,IAAI,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAC1F,CAAC;gCACF,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;4BACpD,CAAC;4BACD,WAAW,EAAE,MAAM,CAAC,WAAW;4BAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;4BACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;4BACzB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;yBAC5C,CAAC,CAAC;oBACP,CAAC;oBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;gBACvD,CAAC;aACJ,CAAC,EACF,eAAe,CAClB,CAAC;YACF,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,8BAA8B,CAAC,eAAsD;QACjG,MAAM,MAAM,GAAG,IAAI,2BAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAElG,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;QAC/B,eAAe,CAAC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAC9C,MAAM,CAAC,GAAG,IAAA,cAAI,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE9B,IAAI,CAAC;YACD,MAAM,IAAA,6BAAmB,EACrB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC;gBACpC,OAAO,EAAE,eAAe,CAAC,OAAO;gBAChC,GAAG,EAAE,eAAe,CAAC,GAAG;gBACxB,aAAa,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAqB;gBAC5D,YAAY,EAAE,KAAK,EAAE,UAAoE,EAAE,EAAE,EAAE;oBAC3F,MAAM,IAAI,GAAG,IAAA,8BAAsB,EAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC;oBACvF,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACpD,CAAC;gBACD,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;aAC5C,CAAC,EACF,IAAI,CAAC,gCAAgC,EACrC,2BAA2B,CAC9B,CAAC;YAEF,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;IACL,CAAC;CACJ;AAtMD,8DAsMC"}
|
|
@@ -23,6 +23,7 @@ import type { BatchAddRequestsResult } from '@crawlee/types';
|
|
|
23
23
|
import type { CheerioRoot, Dictionary } from '@crawlee/utils';
|
|
24
24
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
25
25
|
import type { Page, Response } from 'playwright';
|
|
26
|
+
import { RenderingTypePredictor } from './rendering-type-prediction';
|
|
26
27
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements';
|
|
27
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements';
|
|
28
29
|
import type { PlaywrightCrawlingContext } from '../playwright-crawler';
|
|
@@ -472,5 +473,6 @@ export declare const playwrightUtils: {
|
|
|
472
473
|
saveSnapshot: typeof saveSnapshot;
|
|
473
474
|
compileScript: typeof compileScript;
|
|
474
475
|
closeCookieModals: typeof closeCookieModals;
|
|
476
|
+
RenderingTypePredictor: typeof RenderingTypePredictor;
|
|
475
477
|
};
|
|
476
478
|
//# sourceMappingURL=playwright-utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAA2C,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAI9D,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAExD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iCAAiC,CAAC;AAC7F,OAAO,EAAE,8BAA8B,EAAE,MAAM,iCAAiC,CAAC;AACjF,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AASvE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAqBhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAE,uBAA4B,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CA0CpI;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAwFnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqC/F;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAIvE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAIjE;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;OAUG;IACH,gBAAgB,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzC;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAE/I;;;;;;;;;;;;;;;;;;;;;;;;;MAyBE;IACF,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,yBAAyB,GAAG,IAAI,CAsB/E;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe
|
|
1
|
+
{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAA2C,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC1F,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAI9D,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iCAAiC,CAAC;AAC7F,OAAO,EAAE,8BAA8B,EAAE,MAAM,iCAAiC,CAAC;AACjF,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AASvE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAqBhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAE,uBAA4B,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CA0CpI;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAwFnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqC/F;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAIvE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAIjE;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;OAUG;IACH,gBAAgB,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;IAEzC;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAE/I;;;;;;;;;;;;;;;;;;;;;;;;;MAyBE;IACF,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,yBAAyB,GAAG,IAAI,CAsB/E;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe;;;;;;;;;;;;CAY3B,CAAC"}
|
|
@@ -29,6 +29,7 @@ const browser_1 = require("@crawlee/browser");
|
|
|
29
29
|
const cheerio = tslib_1.__importStar(require("cheerio"));
|
|
30
30
|
const idcac_playwright_1 = require("idcac-playwright");
|
|
31
31
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
32
|
+
const rendering_type_prediction_1 = require("./rendering-type-prediction");
|
|
32
33
|
const click_elements_1 = require("../enqueue-links/click-elements");
|
|
33
34
|
Object.defineProperty(exports, "enqueueLinksByClickingElements", { enumerable: true, get: function () { return click_elements_1.enqueueLinksByClickingElements; } });
|
|
34
35
|
const log = log_1.default.child({ prefix: 'Playwright Utils' });
|
|
@@ -440,5 +441,6 @@ exports.playwrightUtils = {
|
|
|
440
441
|
saveSnapshot,
|
|
441
442
|
compileScript,
|
|
442
443
|
closeCookieModals,
|
|
444
|
+
RenderingTypePredictor: rendering_type_prediction_1.RenderingTypePredictor,
|
|
443
445
|
};
|
|
444
446
|
//# sourceMappingURL=playwright-utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright-utils.js","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;GAkBG;;;;AAEH,+CAA4C;AAC5C,oDAAoB;AAEpB,0DAAiD;AACjD,6DAA8B;AAE9B,8CAA0F;AAG1F,yDAAmC;AACnC,uDAAiF;AACjF,oDAAoB;AAIpB,oEAAiF;AAsuBxE,+GAtuBA,+CAA8B,OAsuBA;AAnuBvC,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE7C,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AA0BtH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;GAUG;AACI,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,QAAgB,EAAE,UAA6B,EAAE;IAC1F,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CAAC,CAAC;IAEJ,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,gBAAgB,EACpB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;aAC9B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AArBD,gCAqBC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAA0C;IACrF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,IAAI,IAAI,EAAE,CAAC,CAAC;AACrG,CAAC;AAHD,oCAGC;AAyBD;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAAgB,EAAE,cAAuC,EAAE;IACtG,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC/B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;KACjD,CAAC,CAAC,CAAC;IACJ,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;IAElE,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,sDAAsD;QACtD,GAAG,CAAC,UAAU,CAAC,+GAA+G;cACxH,4DAA4D,CAAC,CAAC;QACpE,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,KAAY,EAAE,EAAE;YACnD,IAAI,CAAC;gBACD,oGAAoG;gBACpG,uDAAuD;gBACvD,IAAI,SAAS,EAAE,CAAC;oBACZ,OAAO,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClC,CAAC;gBAED,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM,SAAS,GAAe,EAAE,CAAC;gBAEjC,IAAI,MAAM,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChD,IAAI,OAAO;oBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;gBACnD,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC;AA1CD,oCA0CC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACI,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,UAAgC,EAAE;IAC9E,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,WAAW,GAAG,kCAAkC,EAChD,gBAAgB,GAAG,EAAE,GACxB,GAAG,OAAO,CAAC;IAEZ,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAExD,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpC,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IACtF,CAAC;AACL,CAAC;AAtBD,sCAsBC;AASD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,aAAa,CAAC,YAAoB,EAAE,UAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzF,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACD,IAAI,GAAG,YAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;IACtI,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,wBAAwB,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAEtH,OAAO,IAAI,CAAC;AAChB,CAAC;AAdD,sCAcC;AAsCD;;;;;GAKG;AACI,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,UAAiC,EAAE;IAChF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACnC,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAEvI,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;YAChE,cAAc,CAAC,YAAY,EAAE,CAAC;QAClC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;YAC9D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;gBAC5C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;YACX,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9D,CAAC;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;YACrE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,8CAA8C;QAC9C,IAAI,eAAe,GAAG,CAAC,IAAI,gBAAgB,IAAI,eAAe,EAAE,CAAC;YAC7D,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,gBAAgB,IAAI,KAAK,CAAC;IAC9B,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAe,CAAC,CAAC;QAC7C,sDAAsD;QACtD,IAAI,MAAM,IAAI,MAAM,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YACvC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,gBAAgB,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACrB,IAAI,MAAM,kBAAkB,EAAE,EAAE,CAAC;gBAC7B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAxFD,wCAwFC;AAwCD;;;;GAIG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,UAA+B,EAAE;IAC5E,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7B,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,EACjB,MAAM,GACT,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,uBAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,uBAAa,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QAEjH,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;YACrI,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7F,CAAC;AACL,CAAC;AArCD,oCAqCC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,gBAAgB,CAAC,IAAU;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACzC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAJD,4CAIC;AAEM,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAC9C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAErD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAA,sCAAsB,GAAE,CAAC,CAAC;AAClD,CAAC;AAJD,8CAIC;AAoLD,SAAgB,sBAAsB,CAAC,OAAkC;IACrE,OAAO,CAAC,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAE,OAA2B,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1H,OAAO,CAAC,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;QAC/B,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,sBAAY,CAAC,UAAU,EAAE,CAAC;YACpD,GAAG,CAAC,OAAO,CAAC,mIAAmI,CAAC,CAAC;YACjJ,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO;QACX,CAAC;QACD,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,aAAa,GAAG,KAAK,EAAE,OAA8B,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvG,OAAO,CAAC,gBAAgB,GAAG,KAAK,IAAI,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,OAAO,CAAC,cAAc,GAAG,KAAK,EAAE,OAA+B,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1G,OAAO,CAAC,YAAY,GAAG,KAAK,EAAE,OAA6B,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3I,mCAAmC;IACnC,OAAO,CAAC,8BAA8B,GAAG,KAAK,EAAE,OAA6E,EAAE,EAAE,CAAC,IAAA,+CAA8B,EAAC;QAC7J,GAAG,OAAO;QACV,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,YAAa;KAC9C,CAAC,CAAC;IACH,OAAO,CAAC,aAAa,GAAG,CAAC,YAAoB,EAAE,GAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACrG,OAAO,CAAC,iBAAiB,GAAG,KAAK,IAAI,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC5E,CAAC;AAtBD,wDAsBC;AAID,gBAAgB;AACH,QAAA,eAAe,GAAG;IAC3B,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,8BAA8B,EAA9B,+CAA8B;IAC9B,gBAAgB;IAChB,cAAc;IACd,YAAY;IACZ,aAAa;IACb,iBAAiB;CACpB,CAAC"}
|
|
1
|
+
{"version":3,"file":"playwright-utils.js","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;GAkBG;;;;AAEH,+CAA4C;AAC5C,oDAAoB;AAEpB,0DAAiD;AACjD,6DAA8B;AAE9B,8CAA0F;AAG1F,yDAAmC;AACnC,uDAAiF;AACjF,oDAAoB;AAGpB,2EAAqE;AAErE,oEAAiF;AAsuBxE,+GAtuBA,+CAA8B,OAsuBA;AAnuBvC,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE7C,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,MAAM,kCAAkC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AA0BtH;;GAEG;AACH,MAAM,kBAAkB,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAEnF;;;;;;;;;;GAUG;AACI,KAAK,UAAU,UAAU,CAAC,IAAU,EAAE,QAAgB,EAAE,UAA6B,EAAE;IAC1F,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IACxB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KAC1C,CAAC,CAAC,CAAC;IAEJ,IAAI,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,kBAAkB,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC7B,IAAI,CAAC,EAAE,CAAC,gBAAgB,EACpB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;aAC9B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AArBD,gCAqBC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAA0C;IACrF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,kBAAkB,EAAE,OAAO,EAAE,kBAAkB,IAAI,IAAI,EAAE,CAAC,CAAC;AACrG,CAAC;AAHD,oCAGC;AAyBD;;;;;;;;;;;GAWG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,OAAgB,EAAE,cAAuC,EAAE;IACtG,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC/B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;KACjD,CAAC,CAAC,CAAC;IACJ,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAE3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;IAElE,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,sDAAsD;QACtD,GAAG,CAAC,UAAU,CAAC,+GAA+G;cACxH,4DAA4D,CAAC,CAAC;QACpE,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,uBAAuB,GAAG,KAAK,EAAE,KAAY,EAAE,EAAE;YACnD,IAAI,CAAC;gBACD,oGAAoG;gBACpG,uDAAuD;gBACvD,IAAI,SAAS,EAAE,CAAC;oBACZ,OAAO,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClC,CAAC;gBAED,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM,SAAS,GAAe,EAAE,CAAC;gBAEjC,IAAI,MAAM,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;gBAChD,IAAI,OAAO;oBAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;gBACnD,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,GAAG,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7D,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AACvC,CAAC;AA1CD,oCA0CC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACI,KAAK,UAAU,aAAa,CAAC,IAAU,EAAE,UAAgC,EAAE;IAC9E,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChD,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;KACxD,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,WAAW,GAAG,kCAAkC,EAChD,gBAAgB,GAAG,EAAE,GACxB,GAAG,OAAO,CAAC;IAEZ,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,gBAAgB,CAAC,CAAC;IAE9D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAExD,MAAM,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpC,MAAM,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IACtF,CAAC;AACL,CAAC;AAtBD,sCAsBC;AASD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,aAAa,CAAC,YAAoB,EAAE,UAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACzF,MAAM,UAAU,GAAG,iCAAiC,YAAY,GAAG,CAAC;IAEpE,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACD,IAAI,GAAG,YAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,kFAAkF;IACtI,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,wBAAwB,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACd,CAAC;IAED,IAAI,OAAO,IAAI,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC,4BAA4B;IAEtH,OAAO,IAAI,CAAC;AAChB,CAAC;AAdD,sCAcC;AAsCD;;;;;GAKG;AACI,KAAK,UAAU,cAAc,CAAC,IAAU,EAAE,UAAiC,EAAE;IAChF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACnC,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;KAC3C,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,eAAe,GAAG,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC;IAEvI,IAAI,QAAQ,CAAC;IACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,qBAAqB,GAAG,IAAI,CAAC;IACnC,MAAM,qBAAqB,GAAG,KAAK,CAAC;IACpC,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,gCAAgC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG;QACnB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;KACjB,CAAC;IAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,IAAI,gCAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC;YAChE,cAAc,CAAC,YAAY,EAAE,CAAC;QAClC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,IAAI,cAAc,CAAC,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;YAC9D,cAAc,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,cAAc,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;gBAC5C,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,OAAO;YACX,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,cAAc,CAAC,WAAW,GAAG,CAAC,CAAC;YAC/B,cAAc,CAAC,YAAY,GAAG,cAAc,CAAC,YAAY,CAAC;QAC9D,CAAC;QACD,oCAAoC;QACpC,IAAI,WAAW,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;YACrE,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,8CAA8C;QAC9C,IAAI,eAAe,GAAG,CAAC,IAAI,gBAAgB,IAAI,eAAe,EAAE,CAAC;YAC7D,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;IACL,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAE1B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QACxB,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEhF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,gBAAgB,IAAI,KAAK,CAAC;IAC9B,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAe,CAAC,CAAC;QAC7C,sDAAsD;QACtD,IAAI,MAAM,IAAI,MAAM,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YACvC,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;IAEF,OAAO,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,eAAe,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,gBAAgB,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,kBAAkB,EAAE,CAAC;YACrB,IAAI,MAAM,kBAAkB,EAAE,EAAE,CAAC;gBAC7B,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC7B,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAxFD,wCAwFC;AAwCD;;;;GAIG;AACI,KAAK,UAAU,YAAY,CAAC,IAAU,EAAE,UAA+B,EAAE;IAC5E,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;QAChC,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC7B,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,EAAE,EACtB,cAAc,GAAG,IAAI,EACrB,QAAQ,GAAG,IAAI,EACf,iBAAiB,EACjB,MAAM,GACT,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,uBAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,uBAAa,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QAEjH,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,cAAc,GAAG,GAAG,GAAG,MAAM,CAAC;YACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;YACrI,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,GAAG,GAAG,OAAO,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,mBAAoB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7F,CAAC;AACL,CAAC;AArCD,oCAqCC;AAED;;;;;;;;;;GAUG;AACI,KAAK,UAAU,gBAAgB,CAAC,IAAU;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACzC,OAAO,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAJD,4CAIC;AAEM,KAAK,UAAU,iBAAiB,CAAC,IAAU;IAC9C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAU,CAAC,WAAW,CAAC,CAAC,CAAC;IAErD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAA,sCAAsB,GAAE,CAAC,CAAC;AAClD,CAAC;AAJD,8CAIC;AAoLD,SAAgB,sBAAsB,CAAC,OAAkC;IACrE,OAAO,CAAC,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAE,OAA2B,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1H,OAAO,CAAC,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;QAC/B,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,sBAAY,CAAC,UAAU,EAAE,CAAC;YACpD,GAAG,CAAC,OAAO,CAAC,mIAAmI,CAAC,CAAC;YACjJ,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO;QACX,CAAC;QACD,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,aAAa,GAAG,KAAK,EAAE,OAA8B,EAAE,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvG,OAAO,CAAC,gBAAgB,GAAG,KAAK,IAAI,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtE,OAAO,CAAC,cAAc,GAAG,KAAK,EAAE,OAA+B,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC1G,OAAO,CAAC,YAAY,GAAG,KAAK,EAAE,OAA6B,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3I,mCAAmC;IACnC,OAAO,CAAC,8BAA8B,GAAG,KAAK,EAAE,OAA6E,EAAE,EAAE,CAAC,IAAA,+CAA8B,EAAC;QAC7J,GAAG,OAAO;QACV,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,YAAa;KAC9C,CAAC,CAAC;IACH,OAAO,CAAC,aAAa,GAAG,CAAC,YAAoB,EAAE,GAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACrG,OAAO,CAAC,iBAAiB,GAAG,KAAK,IAAI,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAC5E,CAAC;AAtBD,wDAsBC;AAID,gBAAgB;AACH,QAAA,eAAe,GAAG;IAC3B,UAAU;IACV,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,8BAA8B,EAA9B,+CAA8B;IAC9B,gBAAgB;IAChB,cAAc;IACd,YAAY;IACZ,aAAa;IACb,iBAAiB;IACjB,sBAAsB,EAAtB,kDAAsB;CACzB,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export type RenderingType = 'clientOnly' | 'static';
|
|
2
|
+
type URLComponents = string[];
|
|
3
|
+
type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
|
|
4
|
+
export interface RenderingTypePredictorOptions {
|
|
5
|
+
/** A number between 0 and 1 that determines the desired ratio of rendering type detections */
|
|
6
|
+
detectionRatio: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
|
|
10
|
+
*
|
|
11
|
+
* @experimental
|
|
12
|
+
*/
|
|
13
|
+
export declare class RenderingTypePredictor {
|
|
14
|
+
private renderingTypeDetectionResults;
|
|
15
|
+
private detectionRatio;
|
|
16
|
+
private logreg;
|
|
17
|
+
constructor({ detectionRatio }: RenderingTypePredictorOptions);
|
|
18
|
+
/**
|
|
19
|
+
* Predict the rendering type for a given URL and request label.
|
|
20
|
+
*/
|
|
21
|
+
predict(url: URL, label: string | undefined): {
|
|
22
|
+
renderingType: RenderingType;
|
|
23
|
+
detectionProbabilityRecommendation: number;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
|
|
27
|
+
*/
|
|
28
|
+
storeResult(url: URL, label: string | undefined, renderingType: RenderingType): void;
|
|
29
|
+
private resultCount;
|
|
30
|
+
protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
|
|
31
|
+
protected retrain(): void;
|
|
32
|
+
}
|
|
33
|
+
export {};
|
|
34
|
+
//# sourceMappingURL=rendering-type-prediction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rendering-type-prediction.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,QAAQ,CAAA;AAEnD,KAAK,aAAa,GAAG,MAAM,EAAE,CAAC;AAuB9B,KAAK,aAAa,GAAG,CAAC,uBAAuB,EAAE,MAAM,EAAE,2BAA2B,EAAE,MAAM,CAAC,CAAC;AAE5F,MAAM,WAAW,6BAA6B;IAC1C,8FAA8F;IAC9F,cAAc,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,sBAAsB;IAC/B,OAAO,CAAC,6BAA6B,CAAsE;IAC3G,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,MAAM,CAAqB;gBAEvB,EAAE,cAAc,EAAE,EAAE,6BAA6B;IAK7D;;OAEG;IACI,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG;QAAE,aAAa,EAAE,aAAa,CAAC;QAAC,kCAAkC,EAAE,MAAM,CAAA;KAAE;IAejI;;OAEG;IACI,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,aAAa,EAAE,aAAa;IAapF,OAAO,CAAC,WAAW;IAMnB,SAAS,CAAC,sBAAsB,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa;IAO9F,SAAS,CAAC,OAAO,IAAI,IAAI;CAkB5B"}
|