@crawlee/playwright 4.0.0-beta.2 → 4.0.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/internals/adaptive-playwright-crawler.d.ts +38 -54
- package/internals/adaptive-playwright-crawler.d.ts.map +1 -1
- package/internals/adaptive-playwright-crawler.js +244 -169
- package/internals/adaptive-playwright-crawler.js.map +1 -1
- package/internals/enqueue-links/click-elements.d.ts +10 -0
- package/internals/enqueue-links/click-elements.d.ts.map +1 -1
- package/internals/enqueue-links/click-elements.js +14 -2
- package/internals/enqueue-links/click-elements.js.map +1 -1
- package/internals/playwright-crawler.d.ts +17 -42
- package/internals/playwright-crawler.d.ts.map +1 -1
- package/internals/playwright-crawler.js +47 -9
- package/internals/playwright-crawler.js.map +1 -1
- package/internals/utils/playwright-utils.d.ts +10 -3
- package/internals/utils/playwright-utils.d.ts.map +1 -1
- package/internals/utils/playwright-utils.js +36 -38
- package/internals/utils/playwright-utils.js.map +1 -1
- package/internals/utils/rendering-type-prediction.d.ts +8 -3
- package/internals/utils/rendering-type-prediction.d.ts.map +1 -1
- package/internals/utils/rendering-type-prediction.js +22 -10
- package/internals/utils/rendering-type-prediction.js.map +1 -1
- package/package.json +12 -8
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
import { BasicCrawler } from '@crawlee/basic';
|
|
1
3
|
import { extractUrlsFromPage } from '@crawlee/browser';
|
|
2
|
-
import {
|
|
4
|
+
import { CheerioCrawler } from '@crawlee/cheerio';
|
|
5
|
+
import { Configuration, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
|
|
3
6
|
import { extractUrlsFromCheerio } from '@crawlee/utils';
|
|
4
|
-
import { load } from 'cheerio';
|
|
5
|
-
import isEqual from 'lodash.isequal';
|
|
6
7
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
7
8
|
import { PlaywrightCrawler } from './playwright-crawler.js';
|
|
8
9
|
import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
|
|
@@ -80,24 +81,32 @@ const proxyLogMethods = [
|
|
|
80
81
|
*
|
|
81
82
|
* @experimental
|
|
82
83
|
*/
|
|
83
|
-
export class AdaptivePlaywrightCrawler extends
|
|
84
|
+
export class AdaptivePlaywrightCrawler extends BasicCrawler {
|
|
84
85
|
config;
|
|
85
|
-
adaptiveRequestHandler;
|
|
86
86
|
renderingTypePredictor;
|
|
87
87
|
resultChecker;
|
|
88
88
|
resultComparator;
|
|
89
89
|
preventDirectStorageAccess;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
router = Router.create();
|
|
90
|
+
staticContextPipeline;
|
|
91
|
+
browserContextPipeline;
|
|
92
|
+
individualRequestHandlerTimeoutMillis;
|
|
93
|
+
resultObjects = new WeakMap();
|
|
94
|
+
teardownHooks = [];
|
|
96
95
|
constructor(options = {}, config = Configuration.getGlobalConfig()) {
|
|
97
|
-
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, ...rest } = options;
|
|
98
|
-
super(
|
|
96
|
+
const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks, postNavigationHooks, extendContext, contextPipelineBuilder, ...rest } = options;
|
|
97
|
+
super({
|
|
98
|
+
...rest,
|
|
99
|
+
// Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers
|
|
100
|
+
errorHandler,
|
|
101
|
+
failedRequestHandler,
|
|
102
|
+
// Same for request handler
|
|
103
|
+
requestHandler,
|
|
104
|
+
// The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines
|
|
105
|
+
// (that would be a logical error in this class)
|
|
106
|
+
contextPipelineBuilder: () => null,
|
|
107
|
+
}, config);
|
|
99
108
|
this.config = config;
|
|
100
|
-
this.
|
|
109
|
+
this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
|
|
101
110
|
this.renderingTypePredictor =
|
|
102
111
|
renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
|
|
103
112
|
this.resultChecker = resultChecker ?? (() => true);
|
|
@@ -112,10 +121,67 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
112
121
|
return (resultA.datasetItems.length === resultB.datasetItems.length &&
|
|
113
122
|
resultA.datasetItems.every((itemA, i) => {
|
|
114
123
|
const itemB = resultB.datasetItems[i];
|
|
115
|
-
return
|
|
124
|
+
return isDeepStrictEqual(itemA, itemB);
|
|
116
125
|
}));
|
|
117
126
|
};
|
|
118
127
|
}
|
|
128
|
+
const staticCrawler = new CheerioCrawler({
|
|
129
|
+
...rest,
|
|
130
|
+
useSessionPool: false,
|
|
131
|
+
statisticsOptions: {
|
|
132
|
+
persistenceOptions: { enable: false },
|
|
133
|
+
},
|
|
134
|
+
preNavigationHooks: [
|
|
135
|
+
async (context) => {
|
|
136
|
+
for (const hook of preNavigationHooks ?? []) {
|
|
137
|
+
await hook(context, undefined);
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
],
|
|
141
|
+
postNavigationHooks: [
|
|
142
|
+
async (context) => {
|
|
143
|
+
for (const hook of postNavigationHooks ?? []) {
|
|
144
|
+
await hook(context, undefined);
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
],
|
|
148
|
+
}, config);
|
|
149
|
+
const browserCrawler = new PlaywrightCrawler({
|
|
150
|
+
...rest,
|
|
151
|
+
useSessionPool: false,
|
|
152
|
+
statisticsOptions: {
|
|
153
|
+
persistenceOptions: { enable: false },
|
|
154
|
+
},
|
|
155
|
+
preNavigationHooks: [
|
|
156
|
+
async (context, gotoOptions) => {
|
|
157
|
+
for (const hook of preNavigationHooks ?? []) {
|
|
158
|
+
await hook(context, gotoOptions);
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
],
|
|
162
|
+
postNavigationHooks: [
|
|
163
|
+
async (context, gotoOptions) => {
|
|
164
|
+
for (const hook of postNavigationHooks ?? []) {
|
|
165
|
+
await hook(context, gotoOptions);
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
],
|
|
169
|
+
}, config);
|
|
170
|
+
this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
|
|
171
|
+
this.staticContextPipeline = staticCrawler.contextPipeline
|
|
172
|
+
.compose({
|
|
173
|
+
action: this.adaptCheerioContext.bind(this),
|
|
174
|
+
})
|
|
175
|
+
.compose({
|
|
176
|
+
action: async (context) => extendContext ? await extendContext(context) : context,
|
|
177
|
+
});
|
|
178
|
+
this.browserContextPipeline = browserCrawler.contextPipeline
|
|
179
|
+
.compose({
|
|
180
|
+
action: this.adaptPlaywrightContext.bind(this),
|
|
181
|
+
})
|
|
182
|
+
.compose({
|
|
183
|
+
action: async (context) => extendContext ? await extendContext(context) : context,
|
|
184
|
+
});
|
|
119
185
|
this.stats = new AdaptivePlaywrightCrawlerStatistics({
|
|
120
186
|
logMessage: `${this.log.getOptions().prefix} request statistics:`,
|
|
121
187
|
config,
|
|
@@ -123,7 +189,108 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
123
189
|
});
|
|
124
190
|
this.preventDirectStorageAccess = preventDirectStorageAccess;
|
|
125
191
|
}
|
|
126
|
-
async
|
|
192
|
+
async _init() {
|
|
193
|
+
await this.renderingTypePredictor.initialize();
|
|
194
|
+
return await super._init();
|
|
195
|
+
}
|
|
196
|
+
async adaptCheerioContext(cheerioContext) {
|
|
197
|
+
// Capture the original response to avoid infinite recursion when the getter is copied to the context
|
|
198
|
+
const result = this.resultObjects.get(cheerioContext);
|
|
199
|
+
if (result === undefined) {
|
|
200
|
+
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
201
|
+
}
|
|
202
|
+
return {
|
|
203
|
+
get page() {
|
|
204
|
+
throw new Error('Page object was used in HTTP-only request handler');
|
|
205
|
+
},
|
|
206
|
+
async querySelector(selector) {
|
|
207
|
+
return cheerioContext.$(selector);
|
|
208
|
+
},
|
|
209
|
+
enqueueLinks: async (options = {}) => {
|
|
210
|
+
const urls = options.urls ??
|
|
211
|
+
extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
|
|
212
|
+
return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result));
|
|
213
|
+
},
|
|
214
|
+
response: cheerioContext.response,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
async adaptPlaywrightContext(playwrightContext) {
|
|
218
|
+
const originalResponse = playwrightContext.response;
|
|
219
|
+
const result = this.resultObjects.get(playwrightContext);
|
|
220
|
+
if (result === undefined) {
|
|
221
|
+
throw new Error('Logical error - `this.resultObjects` does not contain the result object');
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
response: new Response(Uint8Array.from(await originalResponse.body()), {
|
|
225
|
+
headers: originalResponse.headers(),
|
|
226
|
+
status: originalResponse.status(),
|
|
227
|
+
statusText: originalResponse.statusText(),
|
|
228
|
+
}),
|
|
229
|
+
async querySelector(selector, timeoutMs = 5000) {
|
|
230
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
231
|
+
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
232
|
+
const $ = await playwrightContext.parseWithCheerio();
|
|
233
|
+
return $(selector);
|
|
234
|
+
},
|
|
235
|
+
enqueueLinks: async (options = {}, timeoutMs = 5000) => {
|
|
236
|
+
// TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
|
|
237
|
+
let urls;
|
|
238
|
+
if (options.urls === undefined) {
|
|
239
|
+
const selector = options.selector ?? 'a';
|
|
240
|
+
const locator = playwrightContext.page.locator(selector).first();
|
|
241
|
+
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
242
|
+
urls =
|
|
243
|
+
options.urls ??
|
|
244
|
+
(await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
urls = options.urls;
|
|
248
|
+
}
|
|
249
|
+
return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result));
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
async crawlOne(renderingType, context, useStateFunction) {
|
|
254
|
+
const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
|
|
255
|
+
const logs = [];
|
|
256
|
+
const deferredCleanup = [];
|
|
257
|
+
const resultBoundContextHelpers = {
|
|
258
|
+
addRequests: result.addRequests,
|
|
259
|
+
pushData: result.pushData,
|
|
260
|
+
useState: this.allowStorageAccess(useStateFunction),
|
|
261
|
+
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
|
|
262
|
+
enqueueLinks: async (options) => {
|
|
263
|
+
return await this.enqueueLinks(options, context.request, result);
|
|
264
|
+
},
|
|
265
|
+
log: this.createLogProxy(context.log, logs),
|
|
266
|
+
registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
|
|
267
|
+
};
|
|
268
|
+
const subCrawlerContext = { ...context, ...resultBoundContextHelpers };
|
|
269
|
+
this.resultObjects.set(subCrawlerContext, result);
|
|
270
|
+
try {
|
|
271
|
+
const callAdaptiveRequestHandler = async () => {
|
|
272
|
+
if (renderingType === 'static') {
|
|
273
|
+
await this.staticContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
|
|
274
|
+
}
|
|
275
|
+
else if (renderingType === 'clientOnly') {
|
|
276
|
+
await this.browserContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
|
|
280
|
+
if (this.preventDirectStorageAccess) {
|
|
281
|
+
throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
|
|
282
|
+
}
|
|
283
|
+
}, callAdaptiveRequestHandler), this.individualRequestHandlerTimeoutMillis, 'Request handler timed out');
|
|
284
|
+
return { result, ok: true, logs };
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
return { error, ok: false, logs };
|
|
288
|
+
}
|
|
289
|
+
finally {
|
|
290
|
+
await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
async runRequestHandler(crawlingContext) {
|
|
127
294
|
const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
|
|
128
295
|
const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
|
|
129
296
|
if (!shouldDetectRenderingType) {
|
|
@@ -132,15 +299,19 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
132
299
|
if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
|
|
133
300
|
crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
|
|
134
301
|
this.stats.trackHttpOnlyRequestHandlerRun();
|
|
135
|
-
const plainHTTPRun = await this.
|
|
302
|
+
const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
|
|
136
303
|
if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
|
|
137
304
|
crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
|
|
138
305
|
plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
|
|
139
306
|
await this.commitResult(crawlingContext, plainHTTPRun.result);
|
|
140
307
|
return;
|
|
141
308
|
}
|
|
309
|
+
// Execution will "fall through" and try running the request handler in a browser
|
|
142
310
|
if (!plainHTTPRun.ok) {
|
|
143
|
-
|
|
311
|
+
const actualError = plainHTTPRun.error instanceof RequestHandlerError
|
|
312
|
+
? plainHTTPRun.error.cause
|
|
313
|
+
: plainHTTPRun.error;
|
|
314
|
+
crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
|
|
144
315
|
}
|
|
145
316
|
else {
|
|
146
317
|
crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
|
|
@@ -153,31 +324,52 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
153
324
|
// a rendering type detection if necessary. Without this measure, the HTTP request handler would run
|
|
154
325
|
// under different conditions, which could change its behavior. Changes done to the crawler state by
|
|
155
326
|
// the HTTP request handler will not be committed to the actual storage.
|
|
156
|
-
const
|
|
327
|
+
const stateTracker = {
|
|
328
|
+
stateCopy: null,
|
|
329
|
+
async getLiveState(defaultValue = {}) {
|
|
330
|
+
const state = await crawlingContext.useState(defaultValue);
|
|
331
|
+
if (this.stateCopy === null) {
|
|
332
|
+
this.stateCopy = JSON.parse(JSON.stringify(state));
|
|
333
|
+
}
|
|
334
|
+
return state;
|
|
335
|
+
},
|
|
336
|
+
async getStateCopy(defaultValue = {}) {
|
|
337
|
+
if (this.stateCopy === null) {
|
|
338
|
+
return defaultValue;
|
|
339
|
+
}
|
|
340
|
+
return this.stateCopy;
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
|
|
157
344
|
if (!browserRun.ok) {
|
|
158
345
|
throw browserRun.error;
|
|
159
346
|
}
|
|
160
347
|
await this.commitResult(crawlingContext, browserRun.result);
|
|
161
348
|
if (shouldDetectRenderingType) {
|
|
162
349
|
crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
|
|
163
|
-
const plainHTTPRun = await this.
|
|
350
|
+
const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker));
|
|
164
351
|
const detectionResult = (() => {
|
|
165
352
|
if (!plainHTTPRun.ok) {
|
|
166
353
|
return 'clientOnly';
|
|
167
354
|
}
|
|
168
|
-
|
|
355
|
+
const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
|
|
356
|
+
if (comparisonResult === true || comparisonResult === 'equal') {
|
|
169
357
|
return 'static';
|
|
170
358
|
}
|
|
171
|
-
|
|
359
|
+
if (comparisonResult === false || comparisonResult === 'different') {
|
|
360
|
+
return 'clientOnly';
|
|
361
|
+
}
|
|
362
|
+
return undefined;
|
|
172
363
|
})();
|
|
173
364
|
crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
|
|
174
|
-
|
|
365
|
+
if (detectionResult !== undefined) {
|
|
366
|
+
this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
|
|
367
|
+
}
|
|
175
368
|
}
|
|
176
369
|
}
|
|
177
370
|
async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
|
|
178
371
|
await Promise.all([
|
|
179
372
|
...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
|
|
180
|
-
...calls.enqueueLinks.map(async (params) => await crawlingContext.enqueueLinks(...params)),
|
|
181
373
|
...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
|
|
182
374
|
...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
|
|
183
375
|
const store = await crawlingContext.getKeyValueStore(storeIdOrName);
|
|
@@ -188,151 +380,28 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
188
380
|
allowStorageAccess(func) {
|
|
189
381
|
return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
|
|
190
382
|
}
|
|
191
|
-
async
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
redirectUrls: [],
|
|
214
|
-
},
|
|
215
|
-
log: crawlingContext.log,
|
|
216
|
-
page: crawlingContext.page,
|
|
217
|
-
querySelector: async (selector, timeoutMs = 5_000) => {
|
|
218
|
-
const locator = playwrightContext.page.locator(selector).first();
|
|
219
|
-
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
220
|
-
const $ = await playwrightContext.parseWithCheerio();
|
|
221
|
-
return $(selector);
|
|
222
|
-
},
|
|
223
|
-
async waitForSelector(selector, timeoutMs = 5_000) {
|
|
224
|
-
const locator = playwrightContext.page.locator(selector).first();
|
|
225
|
-
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
226
|
-
},
|
|
227
|
-
async parseWithCheerio(selector, timeoutMs = 5_000) {
|
|
228
|
-
if (selector) {
|
|
229
|
-
const locator = playwrightContext.page.locator(selector).first();
|
|
230
|
-
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
231
|
-
}
|
|
232
|
-
return playwrightContext.parseWithCheerio();
|
|
233
|
-
},
|
|
234
|
-
async enqueueLinks(options = {}, timeoutMs = 5_000) {
|
|
235
|
-
const selector = options.selector ?? 'a';
|
|
236
|
-
const locator = playwrightContext.page.locator(selector).first();
|
|
237
|
-
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
|
|
238
|
-
const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ??
|
|
239
|
-
playwrightContext.request.loadedUrl ??
|
|
240
|
-
playwrightContext.request.url);
|
|
241
|
-
await result.enqueueLinks({ ...options, urls });
|
|
242
|
-
},
|
|
243
|
-
addRequests: result.addRequests,
|
|
244
|
-
pushData: result.pushData,
|
|
245
|
-
useState: this.allowStorageAccess(async (defaultValue) => {
|
|
246
|
-
const state = await result.useState(defaultValue);
|
|
247
|
-
if (initialStateCopy === undefined) {
|
|
248
|
-
initialStateCopy = JSON.parse(JSON.stringify(state));
|
|
249
|
-
}
|
|
250
|
-
return state;
|
|
251
|
-
}),
|
|
252
|
-
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
|
|
253
|
-
}));
|
|
254
|
-
}
|
|
255
|
-
return Reflect.get(target, propertyName, receiver);
|
|
256
|
-
},
|
|
257
|
-
}), crawlingContext);
|
|
258
|
-
return { result: { result, ok: true }, initialStateCopy };
|
|
259
|
-
}
|
|
260
|
-
catch (error) {
|
|
261
|
-
return { result: { error, ok: false }, initialStateCopy };
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
async runRequestHandlerWithPlainHTTP(crawlingContext, oldStateCopy) {
|
|
265
|
-
const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
|
|
266
|
-
const logs = [];
|
|
267
|
-
const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler
|
|
268
|
-
try {
|
|
269
|
-
await withCheckedStorageAccess(() => {
|
|
270
|
-
if (this.preventDirectStorageAccess) {
|
|
271
|
-
throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
|
|
272
|
-
}
|
|
273
|
-
}, async () => addTimeoutToPromise(async () => {
|
|
274
|
-
const hookContext = {
|
|
275
|
-
id: crawlingContext.id,
|
|
276
|
-
session: crawlingContext.session,
|
|
277
|
-
proxyInfo: crawlingContext.proxyInfo,
|
|
278
|
-
request: crawlingContext.request,
|
|
279
|
-
log: this.createLogProxy(crawlingContext.log, logs),
|
|
280
|
-
};
|
|
281
|
-
await this._executeHooks(this.preNavigationHooks, {
|
|
282
|
-
...hookContext,
|
|
283
|
-
get page() {
|
|
284
|
-
throw new Error('Page object was used in HTTP-only pre-navigation hook');
|
|
285
|
-
},
|
|
286
|
-
}, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context
|
|
287
|
-
pageGotoOptions);
|
|
288
|
-
const response = await crawlingContext.sendRequest({});
|
|
289
|
-
const loadedUrl = response.url;
|
|
290
|
-
crawlingContext.request.loadedUrl = loadedUrl;
|
|
291
|
-
const $ = load(response.body);
|
|
292
|
-
await this.adaptiveRequestHandler({
|
|
293
|
-
...hookContext,
|
|
294
|
-
request: crawlingContext.request,
|
|
295
|
-
response,
|
|
296
|
-
get page() {
|
|
297
|
-
throw new Error('Page object was used in HTTP-only request handler');
|
|
298
|
-
},
|
|
299
|
-
async querySelector(selector, _timeoutMs) {
|
|
300
|
-
return $(selector);
|
|
301
|
-
},
|
|
302
|
-
async waitForSelector(selector, _timeoutMs) {
|
|
303
|
-
if ($(selector).get().length === 0) {
|
|
304
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
305
|
-
}
|
|
306
|
-
},
|
|
307
|
-
async parseWithCheerio(selector, _timeoutMs) {
|
|
308
|
-
if (selector && $(selector).get().length === 0) {
|
|
309
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
310
|
-
}
|
|
311
|
-
return $;
|
|
312
|
-
},
|
|
313
|
-
async enqueueLinks(options = {}) {
|
|
314
|
-
const urls = extractUrlsFromCheerio($, options.selector, options.baseUrl ?? loadedUrl);
|
|
315
|
-
await result.enqueueLinks({ ...options, urls });
|
|
316
|
-
},
|
|
317
|
-
addRequests: result.addRequests,
|
|
318
|
-
pushData: result.pushData,
|
|
319
|
-
useState: async (defaultValue) => {
|
|
320
|
-
// return the old state before the browser handler was executed
|
|
321
|
-
// when rerunning the handler via HTTP for detection
|
|
322
|
-
if (oldStateCopy !== undefined) {
|
|
323
|
-
return oldStateCopy ?? defaultValue; // fallback to the default for `null`
|
|
324
|
-
}
|
|
325
|
-
return this.allowStorageAccess(result.useState)(defaultValue);
|
|
326
|
-
},
|
|
327
|
-
getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
|
|
328
|
-
});
|
|
329
|
-
await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions);
|
|
330
|
-
}, this.requestHandlerTimeoutInnerMillis, 'Request handler timed out'));
|
|
331
|
-
return { result, logs, ok: true };
|
|
332
|
-
}
|
|
333
|
-
catch (error) {
|
|
334
|
-
return { error, logs, ok: false };
|
|
335
|
-
}
|
|
383
|
+
async enqueueLinks(options, request, result) {
|
|
384
|
+
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
385
|
+
enqueueStrategy: options?.strategy,
|
|
386
|
+
finalRequestUrl: request.loadedUrl,
|
|
387
|
+
originalRequestUrl: request.url,
|
|
388
|
+
userProvidedBaseUrl: options?.baseUrl,
|
|
389
|
+
});
|
|
390
|
+
const addRequestsBatched = async (requests) => {
|
|
391
|
+
await result.addRequests(requests);
|
|
392
|
+
return {
|
|
393
|
+
addedRequests: requests.map(({ uniqueKey, id }) => ({
|
|
394
|
+
uniqueKey,
|
|
395
|
+
requestId: id ?? '',
|
|
396
|
+
wasAlreadyPresent: false,
|
|
397
|
+
wasAlreadyHandled: false,
|
|
398
|
+
})),
|
|
399
|
+
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
400
|
+
};
|
|
401
|
+
};
|
|
402
|
+
// We need to use a mock request queue implementation, in order to add the requests into our result object
|
|
403
|
+
const mockRequestQueue = { addRequestsBatched };
|
|
404
|
+
return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
|
|
336
405
|
}
|
|
337
406
|
createLogProxy(log, logs) {
|
|
338
407
|
return new Proxy(log, {
|
|
@@ -346,6 +415,12 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
|
|
|
346
415
|
},
|
|
347
416
|
});
|
|
348
417
|
}
|
|
418
|
+
async teardown() {
|
|
419
|
+
await super.teardown();
|
|
420
|
+
for (const hook of this.teardownHooks) {
|
|
421
|
+
await hook();
|
|
422
|
+
}
|
|
423
|
+
}
|
|
349
424
|
}
|
|
350
425
|
export function createAdaptivePlaywrightRouter(routes) {
|
|
351
426
|
return Router.create(routes);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adaptive-playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAUvD,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAElH,OAAO,EAAoB,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAgB,IAAI,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,OAAO,MAAM,gBAAgB,CAAC;AAIrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAkBlG,MAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,GAA4C,IAAW,CAAC,CAAC,+HAA+H;IAEtM,YAAY,UAA6B,EAAE;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAEQ,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC/C,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CACjD,IAAI,CAAC,eAAe,CACvB,CAAC;QAEF,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,UAAU,CAAC,0BAA0B,CAAC;QAC9E,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;IACpF,CAAC;IAED,8BAA8B;QAC1B,IAAI,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,6BAA6B;QACzB,IAAI,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;QAC3B,IAAI,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAsHD,MAAM,eAAe,GAAG;IACpB,OAAO;IACP,WAAW;IACX,UAAU;IACV,MAAM;IACN,OAAO;IACP,MAAM;IACN,aAAa;IACb,YAAY;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,yBAA0B,SAAQ,iBAAiB;IAkBtC;IAjBd,sBAAsB,CAA0D;IAChF,sBAAsB,CAA0E;IAChG,aAAa,CAAiE;IAC9E,gBAAgB,CAAoE;IACpF,0BAA0B,CAAU;IAG5C;;;OAGG;IACH,aAAa;IACK,MAAM,GACpB,MAAM,CAAC,MAAM,EAAoC,CAAC;IAEtD,YACI,UAA4C,EAAE,EAC5B,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,MAAM,EACF,cAAc,EACd,2BAA2B,GAAG,GAAG,EACjC,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,GAAG,IAAI,EACjC,GAAG,IAAI,EACV,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAbF,WAAM,GAAN,MAAM,CAAkC;QAc1D,IAAI,CAAC,sBAAsB,GAAG,cAAc,IAAI,IAAI,CAAC,MAAM,CAAC;QAC5D,IAAI,CAAC,sBAAsB;YACvB,sBAAsB,IAAI,IAAI,sBAAsB,CAAC,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEnD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC7C,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,OAAO,CACH,OAAO,CAAC,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,MAAM;oBAC3D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;wBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;wBACtC,OAAO,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBACjC,CAAC,CAAC,CACL,CAAC;YACN,CAAC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,IAAI,mCAAmC,CAAC;YACjD,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB;YACjE,MAAM;YACN,GAAG,iBAAiB;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;IACjE,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,eAA0C;QAClF,MAAM,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,uBAAuB,CAAC,kCAAkC,CAAC;QAE7G,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAC7B,eAAe,CAAC,GAAG,CAAC,KAAK,CACrB,4BAA4B,uBAAuB,CAAC,aAAa,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACzG,CAAC;QACN,CAAC;QAED,IAAI,uBAAuB,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnF,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,yCAAyC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClG,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC;YAE5C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,8BAA8B,CAAC,eAAe,CAAC,CAAC;YAEhF,IAAI,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAI,IAAmB,CAAC,CAAC,CAAC;gBAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACnB,eAAe,CAAC,GAAG,CAAC,SAAS,CACzB,YAAY,CAAC,KAAc,EAC3B,wCAAwC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACxE,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,GAAG,CAAC,OAAO,CACvB,8DAA8D,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAC9F,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC;QAE3C,qGAAqG;QACrG,oGAAoG;QACpG,oGAAoG;QACpG,wEAAwE;QACxE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,eAAe,CAAC,CAAC;QAExG,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,CAAC,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,yBAAyB,EAAE,CAAC;YAC5B,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACzF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,8BAA8B,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC;YAElG,MAAM,eAAe,GAAkB,CAAC,GAAG,EAAE;gBACzC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChE,OAAO,QAAQ,CAAC;gBACpB,CAAC;gBAED,OAAO,YAAY,CAAC;YACxB,CAAC,CAAC,EAAE,CAAC;YAEL,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,eAAe,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC3G,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACtF,CAAC;IACL,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,eAA0C,EAC1C,EAAE,KAAK,EAAE,oBAAoB,EAAwB;QAErD,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5E,GAAG,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,eAAe,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1F,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC;YAClF,GAAG,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE;gBAC3E,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CACnE,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAC7C,CACJ,CAAC;YACN,CAAC,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IAES,kBAAkB,CACxB,IAAoC;QAEpC,OAAO,KAAK,EAAE,GAAG,IAAW,EAAE,EAAE,CAC5B,wBAAwB,CACpB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAC5B,CAAC;IACV,CAAC;IAES,KAAK,CAAC,0BAA0B,CACtC,eAA0C;QAE1C,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAClG,IAAI,gBAAqD,CAAC;QAE1D,IAAI,CAAC;YACD,MAAM,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAC/B,IAAI,KAAK,CAAC,IAAI,EAAE;gBACZ,GAAG,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE;oBACpC,IAAI,YAAY,KAAK,4BAA4B,EAAE,CAAC;wBAChD,OAAO,KAAK,EAAE,iBAA4C,EAAE,EAAE,CAC1D,wBAAwB,CACpB,GAAG,EAAE;4BACD,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;gCAClC,MAAM,IAAI,KAAK,CACX,6FAA6F,CAChG,CAAC;4BACN,CAAC;wBACL,CAAC,EACD,GAAG,EAAE,CACD,IAAI,CAAC,sBAAsB,CAAC;4BACxB,EAAE,EAAE,eAAe,CAAC,EAAE;4BACtB,OAAO,EAAE,eAAe,CAAC,OAAO;4BAChC,SAAS,EAAE,eAAe,CAAC,SAAS;4BACpC,OAAO,EAAE,eAAe,CAAC,OAAiC;4BAC1D,QAAQ,EAAE;gCACN,GAAG,EAAE,eAAe,CAAC,QAAS,CAAC,GAAG,EAAE;gCACpC,UAAU,EAAE,eAAe,CAAC,QAAS,CAAC,MAAM,EAAE;gCAC9C,OAAO,EAAE,eAAe,CAAC,QAAS,CAAC,OAAO,EAAE;gCAC5C,QAAQ,EAAE,EAAE;gCACZ,QAAQ,EAAE,IAAI;gCACd,YAAY,EAAE,EAAE;6BACnB;4BACD,GAAG,EAAE,eAAe,CAAC,GAAG;4BACxB,IAAI,EAAE,eAAe,CAAC,IAAI;4BAC1B,aAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;gCACjD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gCACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gCACjE,MAAM,CAAC,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;gCAErD,OAAO,CAAC,CAAC,QAAQ,CAAiB,CAAC;4BACvC,CAAC;4BACD,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,GAAG,KAAK;gCAC7C,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gCACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;4BACrE,CAAC;4BACD,KAAK,CAAC,gBAAgB,CAClB,QAAiB,EACjB,SAAS,GAAG,KAAK;gCAEjB,IAAI,QAAQ,EAAE,CAAC;oCACX,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;oCACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gCACrE,CAAC;gCAED,OAAO,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;4BAChD,CAAC;4BACD,KAAK,CAAC,YAAY,CAAC,OAAO,GAAG,EAAE,EAAE,SAAS,GAAG,KAAK;gCAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;gCACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gCACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gCAEjE,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,iBAAiB,CAAC,IAAI,EACtB,QAAQ,EACR,OAAO,CAAC,OAAO;oCACX,iBAAiB,CAAC,OAAO,CAAC,SAAS;oCACnC,iBAAiB,CAAC,OAAO,CAAC,GAAG,CACpC,CAAC;gCACF,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;4BACpD,CAAC;4BACD,WAAW,EAAE,MAAM,CAAC,WAAW;4BAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;4BACzB,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;gCACrD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gCAClD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oCACjC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gCACzD,CAAC;gCACD,OAAO,KAAK,CAAC;4BACjB,CAAC,CAAC;4BACF,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC;yBACrE,CAAC,CACT,CAAC;oBACV,CAAC;oBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;gBACvD,CAAC;aACJ,CAAC,EACF,eAAe,CAClB,CAAC;YACF,OAAO,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB,EAAE,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,gBAAgB,EAAE,CAAC;QAC9D,CAAC;IACL,CAAC;IAES,KAAK,CAAC,8BAA8B,CAC1C,eAA0C,EAC1C,YAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAClG,MAAM,IAAI,GAAmB,EAAE,CAAC;QAEhC,MAAM,eAAe,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,6CAA6C;QAEhH,IAAI,CAAC;YACD,MAAM,wBAAwB,CAC1B,GAAG,EAAE;gBACD,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CACX,6FAA6F,CAChG,CAAC;gBACN,CAAC;YACL,CAAC,EACD,KAAK,IAAI,EAAE,CACP,mBAAmB,CACf,KAAK,IAAI,EAAE;gBACP,MAAM,WAAW,GAAgC;oBAC7C,EAAE,EAAE,eAAe,CAAC,EAAE;oBACtB,OAAO,EAAE,eAAe,CAAC,OAAO;oBAChC,SAAS,EAAE,eAAe,CAAC,SAAS;oBACpC,OAAO,EAAE,eAAe,CAAC,OAAO;oBAChC,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC;iBACtD,CAAC;gBAEF,MAAM,IAAI,CAAC,aAAa,CACpB,IAAI,CAAC,kBAAkB,EACvB;oBACI,GAAG,WAAW;oBACd,IAAI,IAAI;wBACJ,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;oBAC7E,CAAC;iBACyB,EAAE,4GAA4G;gBAC5I,eAAe,CAClB,CAAC;gBAEF,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;gBAC/B,eAAe,CAAC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;gBAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAE9B,MAAM,IAAI,CAAC,sBAAsB,CAAC;oBAC9B,GAAG,WAAW;oBACd,OAAO,EAAE,eAAe,CAAC,OAAiC;oBAC1D,QAAQ;oBACR,IAAI,IAAI;wBACJ,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;oBACzE,CAAC;oBACD,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAmB;wBAC7C,OAAO,CAAC,CAAC,QAAQ,CAAiB,CAAC;oBACvC,CAAC;oBACD,KAAK,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAmB;wBAC/C,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;wBACzD,CAAC;oBACL,CAAC;oBACD,KAAK,CAAC,gBAAgB,CAAC,QAAiB,EAAE,UAAmB;wBACzD,IAAI,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC7C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;wBACzD,CAAC;wBAED,OAAO,CAAC,CAAC;oBACb,CAAC;oBACD,KAAK,CAAC,YAAY,CACd,UAAoE,EAAE;wBAEtE,MAAM,IAAI,GAAG,sBAAsB,CAC/B,CAAC,EACD,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAAO,IAAI,SAAS,CAC/B,CAAC;wBACF,MAAM,MAAM,CAAC,YAAY,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBACpD,CAAC;oBACD,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE;wBAC7B,+DAA+D;wBAC/D,oDAAoD;wBACpD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;4BAC7B,OAAO,YAAY,IAAI,YAAY,CAAC,CAAC,qCAAqC;wBAC9E,CAAC;wBAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;oBAClE,CAAC;oBACD,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC;iBACrE,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;YACzF,CAAC,EACD,IAAI,CAAC,gCAAgC,EACrC,2BAA2B,CAC9B,CACR,CAAC;YAEF,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC;IACL,CAAC;IAEO,cAAc,CAAC,GAAQ,EAAE,IAAoB;QACjD,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE;YAClB,GAAG,CAAC,MAAW,EAAE,YAA8C,EAAE,QAAa;gBAC1E,IAAI,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;wBAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;oBAC/C,CAAC,CAAC;gBACN,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;YACvD,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;CACJ;AAED,MAAM,UAAU,8BAA8B,CAG5C,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"adaptive-playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAalD,OAAO,EACH,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,sCAAsC,EACtC,MAAM,EACN,UAAU,EACV,wBAAwB,GAC3B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAoB,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAO1E,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAkBlG,MAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,GAA4C,IAAW,CAAC,CAAC,+HAA+H;IAEtM,YAAY,UAA6B,EAAE;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAEQ,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,CAAC;IAC/C,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,MAAM,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CACjD,IAAI,CAAC,eAAe,CACvB,CAAC;QAEF,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,0BAA0B,GAAG,UAAU,CAAC,0BAA0B,CAAC;QAC9E,IAAI,CAAC,KAAK,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;IACpF,CAAC;IAED,8BAA8B;QAC1B,IAAI,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,6BAA6B;QACzB,IAAI,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,yBAAyB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;QAC3B,IAAI,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAyHD,MAAM,eAAe,GAAG;IACpB,OAAO;IACP,WAAW;IACX,UAAU;IACV,MAAM;IACN,OAAO;IACP,MAAM;IACN,aAAa;IACb,YAAY;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,yBAEX,SAAQ,YAA+D;IAe/C;IAdd,sBAAsB,CAA0E;IAChG,aAAa,CAAiE;IAC9E,gBAAgB,CAAoE;IACpF,0BAA0B,CAAU;IACpC,qBAAqB,CAAoD;IACzE,sBAAsB,CAAoD;IAC1E,qCAAqC,CAAS;IAE9C,aAAa,GAAG,IAAI,OAAO,EAAyC,CAAC;IAErE,aAAa,GAA+B,EAAE,CAAC;IAEvD,YACI,UAA6D,EAAE,EAC7C,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,MAAM,EACF,cAAc,EACd,2BAA2B,GAAG,GAAG,EACjC,sBAAsB,EACtB,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,GAAG,IAAI,EACjC,yBAAyB,GAAG,EAAE,EAC9B,YAAY,EACZ,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,EACnB,aAAa,EACb,sBAAsB,EACtB,GAAG,IAAI,EACV,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,IAAI;YACP,uHAAuH;YACvH,YAAY;YACZ,oBAAoB;YACpB,2BAA2B;YAC3B,cAAc;YACd,0JAA0J;YAC1J,gDAAgD;YAChD,sBAAsB,EAAE,GAAG,EAAE,CACzB,IAAqF;SAC5F,EACD,MAAM,CACT,CAAC;QAlCgB,WAAM,GAAN,MAAM,CAAkC;QAmC1D,IAAI,CAAC,qCAAqC,GAAG,yBAAyB,GAAG,IAAI,CAAC;QAE9E,IAAI,CAAC,sBAAsB;YACvB,sBAAsB,IAAI,IAAI,sBAAsB,CAAC,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAC1G,IAAI,CAAC,aAAa,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAEnD,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC7C,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,gBAAgB,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;gBACzC,OAAO,CACH,OAAO,CAAC,YAAY,CAAC,MAAM,KAAK,OAAO,CAAC,YAAY,CAAC,MAAM;oBAC3D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;wBACpC,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;wBACtC,OAAO,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBAC3C,CAAC,CAAC,CACL,CAAC;YACN,CAAC,CAAC;QACN,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,cAAc,CACpC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,EAAE;oBACd,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,iBAAiB,CACxC;YACI,GAAG,IAAI;YACP,cAAc,EAAE,KAAK;YACrB,iBAAiB,EAAE;gBACf,kBAAkB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;aACxC;YACD,kBAAkB,EAAE;gBAChB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,kBAAkB,IAAI,EAAE,EAAE,CAAC;wBAC1C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;YACD,mBAAmB,EAAE;gBACjB,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;oBAC3B,KAAK,MAAM,IAAI,IAAI,mBAAmB,IAAI,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;aACJ;SACJ,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QAEtE,IAAI,CAAC,qBAAqB,GAAG,aAAa,CAAC,eAAe;aACrD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9C,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,sBAAsB,GAAG,cAAc,CAAC,eAAe;aACvD,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC;aACD,OAAO,CAAC;YACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CACtB,aAAa,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,OAAsC;SAC7F,CAAC,CAAC;QAEP,IAAI,CAAC,KAAK,GAAG,IAAI,mCAAmC,CAAC;YACjD,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB;YACjE,MAAM;YACN,GAAG,iBAAiB;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,0BAA0B,GAAG,0BAA0B,CAAC;IACjE,CAAC;IAEkB,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,CAAC;QAC/C,OAAO,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,cAAsC;QACpE,qGAAqG;QACrG,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACzE,CAAC;YACD,KAAK,CAAC,aAAa,CAAC,QAAgB;gBAChC,OAAO,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,UAA+B,EAAE,EAAE,EAAE;gBACtD,MAAM,IAAI,GACN,OAAO,CAAC,IAAI;oBACZ,sBAAsB,CAClB,cAAc,CAAC,CAAC,EAChB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,SAAS,CACtD,CAAC;gBACN,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAC3B,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,EACpB,cAAc,CAAC,OAAO,EACtB,MAAM,CACT,CAAoB,CAAC;YAC1B,CAAC;YACD,QAAQ,EAAE,cAAc,CAAC,QAAQ;SACpC,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,iBAA4C;QAC7E,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,QAAQ,CAAC;QAEpD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACzD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO;YACH,QAAQ,EAAE,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE;gBACnE,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE;gBACnC,MAAM,EAAE,gBAAgB,CAAC,MAAM,EAAE;gBACjC,UAAU,EAAE,gBAAgB,CAAC,UAAU,EAAE;aAC5C,CAAC;YACF,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,SAAS,GAAG,IAAI;gBAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBACjE,MAAM,CAAC,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,CAAC;gBAErD,OAAO,CAAC,CAAC,QAAQ,CAAiB,CAAC;YACvC,CAAC;YACD,YAAY,EAAE,KAAK,EAAE,UAA+B,EAAE,EAAE,SAAS,GAAG,IAAI,EAAE,EAAE;gBACxE,mGAAmG;gBACnG,IAAI,IAAuB,CAAC;gBAE5B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;oBACzC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;oBACjE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;oBACjE,IAAI;wBACA,OAAO,CAAC,IAAI;4BACZ,CAAC,MAAM,mBAAmB,CACtB,iBAAiB,CAAC,IAAI,EACtB,QAAQ,EACR,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,SAAS,CACzD,CAAC,CAAC;gBACX,CAAC;qBAAM,CAAC;oBACJ,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;gBACxB,CAAC;gBAED,OAAO,CAAC,MAAM,IAAI,CAAC,YAAY,CAC3B,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,EACpB,iBAAiB,CAAC,OAAO,EACzB,MAAM,CACT,CAAoB,CAAC;YAC1B,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,QAAQ,CAClB,aAA4B,EAC5B,OAAwB,EACxB,gBAAoE;QAEpE,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,iBAAiB,CAAC,CAAC;QAClG,MAAM,IAAI,GAAmB,EAAE,CAAC;QAEhC,MAAM,eAAe,GAA+B,EAAE,CAAC;QAEvD,MAAM,yBAAyB,GAAG;YAC9B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC;YACnD,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC;YAClE,YAAY,EAAE,KAAK,EAAE,OAAiD,EAAE,EAAE;gBACtE,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACrE,CAAC;YACD,GAAG,EAAE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC;YAC3C,uBAAuB,EAAE,CAAC,OAA+B,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;SAC9F,CAAC;QAEF,MAAM,iBAAiB,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,yBAAyB,EAAE,CAAC;QACvE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC;QAElD,IAAI,CAAC;YACD,MAAM,0BAA0B,GAAG,KAAK,IAAI,EAAE;gBAC1C,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CACjC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;qBAAM,IAAI,aAAa,KAAK,YAAY,EAAE,CAAC;oBACxC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAClC,iBAAiB,EACjB,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAClE,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CACP,wBAAwB,CAAC,GAAG,EAAE;gBAC1B,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CACX,6FAA6F,CAChG,CAAC;gBACN,CAAC;YACL,CAAC,EAAE,0BAA0B,CAAC,EAClC,IAAI,CAAC,qCAAqC,EAC1C,2BAA2B,CAC9B,CAAC;YAEF,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;gBAAS,CAAC;YACP,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,iBAAiB,CAAC,eAAgC;QACvE,MAAM,uBAAuB,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,uBAAuB,CAAC,kCAAkC,CAAC;QAE7G,IAAI,CAAC,yBAAyB,EAAE,CAAC;YAC7B,eAAe,CAAC,GAAG,CAAC,KAAK,CACrB,4BAA4B,uBAAuB,CAAC,aAAa,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACzG,CAAC;QACN,CAAC;QAED,IAAI,uBAAuB,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnF,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,yCAAyC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAClG,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC;YAE5C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;YAE9F,IAAI,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACpG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAI,IAAmB,CAAC,CAAC,CAAC;gBAC7F,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC9D,OAAO;YACX,CAAC;YAED,iFAAiF;YACjF,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;gBACnB,MAAM,WAAW,GACb,YAAY,CAAC,KAAK,YAAY,mBAAmB;oBAC7C,CAAC,CAAE,YAAY,CAAC,KAAK,CAAC,KAAe;oBACrC,CAAC,CAAE,YAAY,CAAC,KAAe,CAAC;gBAExC,eAAe,CAAC,GAAG,CAAC,SAAS,CACzB,WAAW,EACX,wCAAwC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CACxE,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,eAAe,CAAC,GAAG,CAAC,OAAO,CACvB,8DAA8D,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAC9F,CAAC;gBACF,IAAI,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC;YACjD,CAAC;QACL,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QAChG,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC;QAE3C,qGAAqG;QACrG,oGAAoG;QACpG,oGAAoG;QACpG,wEAAwE;QACxE,MAAM,YAAY,GAAG;YACjB,SAAS,EAAE,IAAI;YACf,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;gBAE3D,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvD,CAAC;gBAED,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,KAAK,CAAC,YAAY,CAAC,eAA2B,EAAE;gBAC5C,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC1B,OAAO,YAAY,CAAC;gBACxB,CAAC;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC;YAC1B,CAAC;SACJ,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAClC,YAAY,EACZ,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,UAAU,CAAC,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,yBAAyB,EAAE,CAAC;YAC5B,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YACzF,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpC,QAAQ,EACR,eAAe,EACf,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAC/C,CAAC;YAEF,MAAM,eAAe,GAA8B,CAAC,GAAG,EAAE;gBACrD,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;oBACnB,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;gBACvF,IAAI,gBAAgB,KAAK,IAAI,IAAI,gBAAgB,KAAK,OAAO,EAAE,CAAC;oBAC5D,OAAO,QAAQ,CAAC;gBACpB,CAAC;gBAED,IAAI,gBAAgB,KAAK,KAAK,IAAI,gBAAgB,KAAK,WAAW,EAAE,CAAC;oBACjE,OAAO,YAAY,CAAC;gBACxB,CAAC;gBAED,OAAO,SAAS,CAAC;YACrB,CAAC,CAAC,EAAE,CAAC;YAEL,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,eAAe,QAAQ,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAE3G,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;gBAChC,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;IACL,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,eAAgC,EAChC,EAAE,KAAK,EAAE,oBAAoB,EAAwB;QAErD,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5E,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,CAAC;YAClF,GAAG,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE;gBAC3E,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACpE,MAAM,OAAO,CAAC,GAAG,CACb,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CACnE,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAC7C,CACJ,CAAC;YACN,CAAC,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IAES,kBAAkB,CACxB,IAAoC;QAEpC,OAAO,KAAK,EAAE,GAAG,IAAW,EAAE,EAAE,CAC5B,wBAAwB,CACpB,GAAG,EAAE,GAAE,CAAC,EACR,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAC5B,CAAC;IACV,CAAC;IAES,KAAK,CAAC,YAAY,CACxB,OAAiD,EACjD,OAA6C,EAC7C,MAA4B;QAE5B,MAAM,OAAO,GAAG,sCAAsC,CAAC;YACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;YAClC,eAAe,EAAE,OAAO,CAAC,SAAS;YAClC,kBAAkB,EAAE,OAAO,CAAC,GAAG;YAC/B,mBAAmB,EAAE,OAAO,EAAE,OAAO;SACxC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAuC,KAAK,EAAE,QAA+B,EAAE,EAAE;YACrG,MAAM,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAEnC,OAAO;gBACH,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;oBAChD,SAAS;oBACT,SAAS,EAAE,EAAE,IAAI,EAAE;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,iBAAiB,EAAE,KAAK;iBAC3B,CAAC,CAAC;gBACH,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC,CAAC;QACF,0GAA0G;QAC1G,MAAM,gBAAgB,GAAG,EAAE,kBAAkB,EAAkB,CAAC;QAEhE,OAAO,MAAM,IAAI,CAAC,0BAA0B,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACrG,CAAC;IAEO,cAAc,CAAC,GAAQ,EAAE,IAAoB;QACjD,OAAO,IAAI,KAAK,CAAC,GAAG,EAAE;YAClB,GAAG,CAAC,MAAW,EAAE,YAA8C,EAAE,QAAa;gBAC1E,IAAI,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzC,OAAO,CAAC,GAAG,IAAe,EAAE,EAAE;wBAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;oBAC/C,CAAC,CAAC;gBACN,CAAC;gBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;YACvD,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEQ,KAAK,CAAC,QAAQ;QACnB,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACpC,MAAM,IAAI,EAAE,CAAC;QACjB,CAAC;IACL,CAAC;CACJ;AAED,MAAM,UAAU,8BAA8B,CAG5C,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -40,6 +40,16 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
40
40
|
* after clicking on elements matching the provided CSS selector.
|
|
41
41
|
*/
|
|
42
42
|
globs?: GlobInput[];
|
|
43
|
+
/**
|
|
44
|
+
* An array of glob pattern strings, regexp patterns or plain objects
|
|
45
|
+
* containing patterns matching URLs that will **never** be enqueued.
|
|
46
|
+
*
|
|
47
|
+
* The plain objects must include either the `glob` property or the `regexp` property.
|
|
48
|
+
*
|
|
49
|
+
* Glob matching is always case-insensitive.
|
|
50
|
+
* If you need case-sensitive matching, provide a regexp.
|
|
51
|
+
*/
|
|
52
|
+
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
43
53
|
/**
|
|
44
54
|
* An array of regular expressions or plain objects
|
|
45
55
|
* containing regular expressions matching the URLs to be enqueued.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EAEX,eAAe,EACf,gBAAgB,EAEnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAS,IAAI,EAAkB,MAAM,YAAY,CAAC;AAO9D,KAAK,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjD,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,eAAe,CAAC;IAE9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAChD,OAAO,EAAE,qCAAqC,GAC/C,OAAO,CAAC,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"click-elements.d.ts","sourceRoot":"","sources":["../../../src/internals/enqueue-links/click-elements.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EAEX,eAAe,EACf,gBAAgB,EAEnB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAS,IAAI,EAAkB,MAAM,YAAY,CAAC;AAO9D,KAAK,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjD,MAAM,WAAW,qCAAqC;IAClD;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,eAAe,CAAC;IAE9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;;OAkBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAsB,8BAA8B,CAChD,OAAO,EAAE,qCAAqC,GAC/C,OAAO,CAAC,sBAAsB,CAAC,CAmFjC;AAED,UAAU,sBAAsB;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,UAAU,kDAAmD,SAAQ,sBAAsB;IACvF,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAsB,2CAA2C,CAC7D,OAAO,EAAE,kDAAkD,GAC5D,OAAO,CAAC,UAAU,EAAE,CAAC,CA4BvB;AAoHD;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA2B5G"}
|
|
@@ -55,6 +55,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
55
55
|
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
56
56
|
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
57
57
|
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
58
|
+
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
58
59
|
transformRequestFunction: ow.optional.function,
|
|
59
60
|
waitForPageIdleSecs: ow.optional.number,
|
|
60
61
|
maxWaitForPageIdleSecs: ow.optional.number,
|
|
@@ -62,10 +63,21 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
62
63
|
forefront: ow.optional.boolean,
|
|
63
64
|
skipNavigation: ow.optional.boolean,
|
|
64
65
|
}));
|
|
65
|
-
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
|
|
66
|
+
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, } = options;
|
|
66
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
67
68
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
69
|
+
const urlExcludePatternObjects = [];
|
|
68
70
|
const urlPatternObjects = [];
|
|
71
|
+
if (exclude?.length) {
|
|
72
|
+
for (const excl of exclude) {
|
|
73
|
+
if (typeof excl === 'string' || 'glob' in excl) {
|
|
74
|
+
urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
|
|
75
|
+
}
|
|
76
|
+
else if (excl instanceof RegExp || 'regexp' in excl) {
|
|
77
|
+
urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
69
81
|
if (pseudoUrls?.length) {
|
|
70
82
|
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
71
83
|
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
@@ -87,7 +99,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
87
99
|
if (transformRequestFunction) {
|
|
88
100
|
requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
|
|
89
101
|
}
|
|
90
|
-
const requests = createRequests(requestOptions, urlPatternObjects);
|
|
102
|
+
const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);
|
|
91
103
|
const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
|
|
92
104
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
93
105
|
}
|