@crawlee/playwright 4.0.0-beta.21 → 4.0.0-beta.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +14 -14
  2. package/index.d.ts +4 -3
  3. package/index.js +2 -2
  4. package/internals/adaptive-playwright-crawler.d.ts +136 -65
  5. package/internals/adaptive-playwright-crawler.js +417 -274
  6. package/internals/enqueue-links/click-elements.d.ts +36 -64
  7. package/internals/enqueue-links/click-elements.js +65 -67
  8. package/internals/playwright-browser-pool.d.ts +71 -0
  9. package/internals/playwright-browser-pool.js +61 -0
  10. package/internals/playwright-crawler.d.ts +176 -148
  11. package/internals/playwright-crawler.js +77 -73
  12. package/internals/playwright-launcher.d.ts +30 -20
  13. package/internals/playwright-launcher.js +22 -17
  14. package/internals/utils/playwright-utils.d.ts +54 -56
  15. package/internals/utils/playwright-utils.js +117 -138
  16. package/internals/utils/rendering-type-prediction.d.ts +37 -11
  17. package/internals/utils/rendering-type-prediction.js +81 -27
  18. package/package.json +15 -19
  19. package/index.d.ts.map +0 -1
  20. package/index.js.map +0 -1
  21. package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
  22. package/internals/adaptive-playwright-crawler.js.map +0 -1
  23. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  24. package/internals/enqueue-links/click-elements.js.map +0 -1
  25. package/internals/playwright-crawler.d.ts.map +0 -1
  26. package/internals/playwright-crawler.js.map +0 -1
  27. package/internals/playwright-launcher.d.ts.map +0 -1
  28. package/internals/playwright-launcher.js.map +0 -1
  29. package/internals/utils/playwright-utils.d.ts.map +0 -1
  30. package/internals/utils/playwright-utils.js.map +0 -1
  31. package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
  32. package/internals/utils/rendering-type-prediction.js.map +0 -1
@@ -1,70 +1,46 @@
1
- import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterRoutes } from '@crawlee/browser';
2
- import { BrowserCrawler, Configuration } from '@crawlee/browser';
3
- import type { PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
1
+ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
2
+ import { BrowserCrawler } from '@crawlee/browser';
4
3
  import type { Dictionary } from '@crawlee/types';
5
4
  // @ts-ignore optional peer dependency or compatibility with es2022
6
- import type { LaunchOptions, Page, Response } from 'playwright';
5
+ import type { Page, Response } from 'playwright';
6
+ import { z } from 'zod';
7
7
  import type { PlaywrightLaunchContext } from './playwright-launcher.js';
8
- import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
9
- export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {
8
+ import type { DirectNavigationOptions, HandleCloudflareChallengeOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
9
+ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
10
+ export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
10
11
  }
11
- // @ts-ignore optional peer dependency or compatibility with es2022
12
- export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {
13
- }
14
- export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
15
- export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response, PlaywrightController, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
16
- browserPlugins: [PlaywrightPlugin];
17
- }> {
12
+ export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
13
+ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
18
14
  /**
19
15
  * The same options as used by {@link launchPlaywright}.
20
16
  */
21
17
  launchContext?: PlaywrightLaunchContext;
22
18
  /**
23
- * Function that is called to process each request.
24
- *
25
- * The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
26
- * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
27
- * - `page` is an instance of the `Playwright`
28
- * [`Page`](https://playwright.dev/docs/api/class-page)
29
- * - `browserController` is an instance of the
30
- * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
31
- * - `response` is an instance of the `Playwright`
32
- * [`Response`](https://playwright.dev/docs/api/class-response),
33
- * which is the main resource response as returned by `page.goto(request.url)`.
34
- *
35
- * The function must return a promise, which is then awaited by the crawler.
36
- *
37
- * If the function throws an exception, the crawler will try to re-crawl the
38
- * request later, up to `option.maxRequestRetries` times.
39
- * If all the retries fail, the crawler calls the function
40
- * provided to the `failedRequestHandler` parameter.
41
- * To make this work, you should **always**
42
- * let your function throw exceptions rather than catch them.
43
- * The exceptions are logged to the request using the
44
- * {@link Request.pushErrorMessage} function.
19
+ * Whether to run browser in headless mode. Defaults to `true`.
20
+ * Can be also set via {@link Configuration}.
45
21
  */
46
- requestHandler?: RequestHandler<ExtendedContext>;
22
+ headless?: boolean;
47
23
  /**
48
24
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
49
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
50
- * which are passed to the `page.goto()` function the crawler calls to navigate.
25
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
26
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
27
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
28
+ * (e.g. to override context members for subsequent hooks and pipeline stages).
51
29
  * Example:
52
30
  * ```
53
31
  * preNavigationHooks: [
54
- * async (crawlingContext, gotoOptions) => {
55
- * const { page } = crawlingContext;
32
+ * async ({ page, gotoOptions }) => {
56
33
  * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
34
+ * gotoOptions.timeout = 60_000;
57
35
  * },
58
36
  * ]
59
37
  * ```
60
- *
61
- * Modyfing `pageOptions` is supported only in Playwright incognito.
62
- * See {@link PrePageCreateHook}
63
38
  */
64
- preNavigationHooks?: PlaywrightHook[];
39
+ preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
65
40
  /**
66
41
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
67
- * The function accepts `crawlingContext` as the only parameter.
42
+ * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
43
+ * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
68
44
  * Example:
69
45
  * ```
70
46
  * postNavigationHooks: [
@@ -77,7 +53,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
77
53
  * ]
78
54
  * ```
79
55
  */
80
- postNavigationHooks?: PlaywrightHook[];
56
+ postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
81
57
  }
82
58
  /**
83
59
  * Provides a simple framework for parallel crawling of web pages
@@ -90,24 +66,26 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
90
66
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
91
67
  * which downloads the pages using raw HTTP requests and is about 10x faster.
92
68
  *
93
- * The source URLs are represented using {@link Request} objects that are fed from
94
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
95
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
69
+ * The source URLs are represented using {@link Request} objects that are fed from the
70
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
71
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
72
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
73
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
74
+ * result as `requestManager`.
96
75
  *
97
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
98
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
99
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
76
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
77
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
100
78
  *
101
79
  * The crawler finishes when there are no more {@link Request} objects to crawl.
102
80
  *
103
81
  * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
104
82
  * and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
105
83
  *
106
- * New pages are only opened when there is enough free CPU and memory available,
107
- * using the functionality provided by the {@link AutoscaledPool} class.
108
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
109
- * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
110
- * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
84
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
85
+ * {@link ConcurrencySystem}.
86
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
87
+ * `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
88
+ * {@link ConcurrencySystem|`concurrencySystem`}.
111
89
  *
112
90
  * Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
113
91
  *
@@ -142,101 +120,151 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
142
120
  * ```
143
121
  * @category Crawlers
144
122
  */
145
- export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response, PlaywrightController, {
146
- browserPlugins: [PlaywrightPlugin];
147
- }, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
148
- readonly config: Configuration;
123
+ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
124
+ #private;
125
+ /**
126
+ * @internal
127
+ */
149
128
  protected static optionsShape: {
150
- // @ts-ignore optional peer dependency or compatibility with es2022
151
- browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
152
- // @ts-ignore optional peer dependency or compatibility with es2022
153
- launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
154
- // @ts-ignore optional peer dependency or compatibility with es2022
155
- ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
156
- // @ts-ignore optional peer dependency or compatibility with es2022
157
- ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
158
- // @ts-ignore optional peer dependency or compatibility with es2022
159
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
160
- // @ts-ignore optional peer dependency or compatibility with es2022
161
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
162
- // @ts-ignore optional peer dependency or compatibility with es2022
163
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
164
- // @ts-ignore optional peer dependency or compatibility with es2022
165
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
166
- // @ts-ignore optional peer dependency or compatibility with es2022
167
- headless: import("ow").AnyPredicate<string | boolean>;
168
- // @ts-ignore optional peer dependency or compatibility with es2022
169
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
170
- // @ts-ignore optional peer dependency or compatibility with es2022
171
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
172
- // @ts-ignore optional peer dependency or compatibility with es2022
173
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
174
- // @ts-ignore optional peer dependency or compatibility with es2022
175
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
176
- // @ts-ignore optional peer dependency or compatibility with es2022
177
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
178
- // @ts-ignore optional peer dependency or compatibility with es2022
179
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
180
- // @ts-ignore optional peer dependency or compatibility with es2022
181
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
182
- // @ts-ignore optional peer dependency or compatibility with es2022
183
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
184
- // @ts-ignore optional peer dependency or compatibility with es2022
185
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
186
- // @ts-ignore optional peer dependency or compatibility with es2022
187
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
188
- // @ts-ignore optional peer dependency or compatibility with es2022
189
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
190
- // @ts-ignore optional peer dependency or compatibility with es2022
191
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
192
- // @ts-ignore optional peer dependency or compatibility with es2022
193
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
194
- // @ts-ignore optional peer dependency or compatibility with es2022
195
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
196
- // @ts-ignore optional peer dependency or compatibility with es2022
197
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
198
- // @ts-ignore optional peer dependency or compatibility with es2022
199
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
200
- // @ts-ignore optional peer dependency or compatibility with es2022
201
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
202
- // @ts-ignore optional peer dependency or compatibility with es2022
203
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
204
- // @ts-ignore optional peer dependency or compatibility with es2022
205
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
206
- // @ts-ignore optional peer dependency or compatibility with es2022
207
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
208
- // @ts-ignore optional peer dependency or compatibility with es2022
209
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
210
- // @ts-ignore optional peer dependency or compatibility with es2022
211
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
212
- // @ts-ignore optional peer dependency or compatibility with es2022
213
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
214
- // @ts-ignore optional peer dependency or compatibility with es2022
215
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
216
- // @ts-ignore optional peer dependency or compatibility with es2022
217
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
218
- // @ts-ignore optional peer dependency or compatibility with es2022
219
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
220
- // @ts-ignore optional peer dependency or compatibility with es2022
221
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
222
- // @ts-ignore optional peer dependency or compatibility with es2022
223
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
224
- // @ts-ignore optional peer dependency or compatibility with es2022
225
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
226
- // @ts-ignore optional peer dependency or compatibility with es2022
227
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
228
- // @ts-ignore optional peer dependency or compatibility with es2022
229
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
230
- // @ts-ignore optional peer dependency or compatibility with es2022
231
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
129
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
130
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
131
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
132
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
133
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
134
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
135
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
136
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
137
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
138
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
139
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
140
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
141
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
142
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
143
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
144
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
145
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
146
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
147
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
148
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
149
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
150
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
151
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
152
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
153
+ requestQueue: z.ZodOptional<z.ZodEnum<{
154
+ deferred: "deferred";
155
+ writeThrough: "writeThrough";
156
+ }>>;
157
+ }, z.core.$strict>]>>;
158
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
159
+ // @ts-ignore optional peer dependency or compatibility with es2022
160
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
161
+ // @ts-ignore optional peer dependency or compatibility with es2022
162
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
163
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
164
+ // @ts-ignore optional peer dependency or compatibility with es2022
165
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
166
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
167
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
168
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
169
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
170
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
171
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
172
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
173
+ id: z.ZodOptional<z.ZodString>;
174
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
175
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
176
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
177
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
178
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
179
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
180
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
181
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
182
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
183
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
184
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
185
+ headless: z.ZodOptional<z.ZodBoolean>;
232
186
  };
187
+ /** @internal */
188
+ protected static optionsSchema: z.ZodObject<{
189
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
190
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
191
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
192
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
193
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
194
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
195
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
196
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
197
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
198
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
199
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
200
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
201
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
202
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
203
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
204
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
205
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
206
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
207
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
208
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
209
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
210
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
211
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
212
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
213
+ requestQueue: z.ZodOptional<z.ZodEnum<{
214
+ deferred: "deferred";
215
+ writeThrough: "writeThrough";
216
+ }>>;
217
+ }, z.core.$strict>]>>;
218
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
219
+ // @ts-ignore optional peer dependency or compatibility with es2022
220
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
221
+ // @ts-ignore optional peer dependency or compatibility with es2022
222
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
223
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
224
+ // @ts-ignore optional peer dependency or compatibility with es2022
225
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
226
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
227
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
228
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
229
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
230
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
231
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
232
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
233
+ id: z.ZodOptional<z.ZodString>;
234
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
235
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
236
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
237
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
238
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
239
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
240
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
241
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
242
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
243
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
244
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
245
+ headless: z.ZodOptional<z.ZodBoolean>;
246
+ }, z.core.$strict>;
233
247
  /**
234
248
  * All `PlaywrightCrawler` parameters are passed via an options object.
235
249
  */
236
- constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>, config?: Configuration);
237
- protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
250
+ constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
251
+ protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
238
252
  private enhanceContext;
239
253
  }
254
+ /**
255
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
256
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
257
+ *
258
+ * **Example usage**
259
+ * ```ts
260
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
261
+ *
262
+ * const crawler = new PlaywrightCrawler({
263
+ * postNavigationHooks: [handleCloudflareChallengeHook()],
264
+ * });
265
+ * ```
266
+ */
267
+ export declare function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
240
268
  /**
241
269
  * Creates new {@link Router} instance that works based on request labels.
242
270
  * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
@@ -261,6 +289,6 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
261
289
  * await crawler.run();
262
290
  * ```
263
291
  */
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
266
- //# sourceMappingURL=playwright-crawler.d.ts.map
292
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
293
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
294
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;