@crawlee/playwright 4.0.0-beta.23 → 4.0.0-beta.231

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +4 -3
  3. package/index.js +2 -2
  4. package/internals/adaptive-playwright-crawler.d.ts +129 -69
  5. package/internals/adaptive-playwright-crawler.js +422 -278
  6. package/internals/enqueue-links/click-elements.d.ts +36 -64
  7. package/internals/enqueue-links/click-elements.js +65 -67
  8. package/internals/playwright-browser-pool.d.ts +55 -0
  9. package/internals/playwright-browser-pool.js +34 -0
  10. package/internals/playwright-crawler.d.ts +179 -149
  11. package/internals/playwright-crawler.js +76 -74
  12. package/internals/playwright-launcher.d.ts +30 -20
  13. package/internals/playwright-launcher.js +24 -18
  14. package/internals/utils/playwright-utils.d.ts +54 -56
  15. package/internals/utils/playwright-utils.js +117 -138
  16. package/internals/utils/rendering-type-prediction.d.ts +35 -12
  17. package/internals/utils/rendering-type-prediction.js +87 -33
  18. package/package.json +15 -20
  19. package/index.d.ts.map +0 -1
  20. package/index.js.map +0 -1
  21. package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
  22. package/internals/adaptive-playwright-crawler.js.map +0 -1
  23. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  24. package/internals/enqueue-links/click-elements.js.map +0 -1
  25. package/internals/playwright-crawler.d.ts.map +0 -1
  26. package/internals/playwright-crawler.js.map +0 -1
  27. package/internals/playwright-launcher.d.ts.map +0 -1
  28. package/internals/playwright-launcher.js.map +0 -1
  29. package/internals/utils/playwright-utils.d.ts.map +0 -1
  30. package/internals/utils/playwright-utils.js.map +0 -1
  31. package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
  32. package/internals/utils/rendering-type-prediction.js.map +0 -1
@@ -1,70 +1,49 @@
1
- import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterRoutes } from '@crawlee/browser';
2
- import { BrowserCrawler, Configuration } from '@crawlee/browser';
3
- import type { PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
1
+ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
2
+ import { BrowserCrawler } from '@crawlee/browser';
4
3
  import type { Dictionary } from '@crawlee/types';
5
4
  // @ts-ignore optional peer dependency or compatibility with es2022
6
- import type { LaunchOptions, Page, Response } from 'playwright';
5
+ import type { Page, Response } from 'playwright';
6
+ import { z } from 'zod';
7
+ import { type PlaywrightRemoteBrowserOptions } from './playwright-browser-pool.js';
7
8
  import type { PlaywrightLaunchContext } from './playwright-launcher.js';
8
- import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
9
- export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {
9
+ import type { DirectNavigationOptions, HandleCloudflareChallengeOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
10
+ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
11
+ export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
10
12
  }
11
- // @ts-ignore optional peer dependency or compatibility with es2022
12
- export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {
13
- }
14
- export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
15
- export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response, PlaywrightController, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
16
- browserPlugins: [PlaywrightPlugin];
17
- }> {
13
+ export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
14
+ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
18
15
  /**
19
16
  * The same options as used by {@link launchPlaywright}.
20
17
  */
21
18
  launchContext?: PlaywrightLaunchContext;
22
19
  /**
23
- * Function that is called to process each request.
24
- *
25
- * The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
26
- * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
27
- * - `page` is an instance of the `Playwright`
28
- * [`Page`](https://playwright.dev/docs/api/class-page)
29
- * - `browserController` is an instance of the
30
- * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
31
- * - `response` is an instance of the `Playwright`
32
- * [`Response`](https://playwright.dev/docs/api/class-response),
33
- * which is the main resource response as returned by `page.goto(request.url)`.
34
- *
35
- * The function must return a promise, which is then awaited by the crawler.
36
- *
37
- * If the function throws an exception, the crawler will try to re-crawl the
38
- * request later, up to `option.maxRequestRetries` times.
39
- * If all the retries fail, the crawler calls the function
40
- * provided to the `failedRequestHandler` parameter.
41
- * To make this work, you should **always**
42
- * let your function throw exceptions rather than catch them.
43
- * The exceptions are logged to the request using the
44
- * {@link Request.pushErrorMessage} function.
20
+ * Whether to run browser in headless mode. Defaults to `true`.
21
+ * Can be also set via {@link Configuration}.
45
22
  */
46
- requestHandler?: RequestHandler<ExtendedContext>;
23
+ headless?: boolean;
24
+ /** See {@link BrowserCrawlerOptions.remoteBrowser}; Playwright additionally lets you pick the `protocol`. */
25
+ remoteBrowser?: PlaywrightRemoteBrowserOptions;
47
26
  /**
48
27
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
49
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
50
- * which are passed to the `page.goto()` function the crawler calls to navigate.
28
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
29
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
30
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
31
+ * (e.g. to override context members for subsequent hooks and pipeline stages).
51
32
  * Example:
52
33
  * ```
53
34
  * preNavigationHooks: [
54
- * async (crawlingContext, gotoOptions) => {
55
- * const { page } = crawlingContext;
35
+ * async ({ page, gotoOptions }) => {
56
36
  * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
37
+ * gotoOptions.timeout = 60_000;
57
38
  * },
58
39
  * ]
59
40
  * ```
60
- *
61
- * Modyfing `pageOptions` is supported only in Playwright incognito.
62
- * See {@link PrePageCreateHook}
63
41
  */
64
- preNavigationHooks?: PlaywrightHook[];
42
+ preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
65
43
  /**
66
44
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
67
- * The function accepts `crawlingContext` as the only parameter.
45
+ * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
46
+ * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
68
47
  * Example:
69
48
  * ```
70
49
  * postNavigationHooks: [
@@ -77,7 +56,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
77
56
  * ]
78
57
  * ```
79
58
  */
80
- postNavigationHooks?: PlaywrightHook[];
59
+ postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
81
60
  }
82
61
  /**
83
62
  * Provides a simple framework for parallel crawling of web pages
@@ -90,24 +69,26 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
90
69
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
91
70
  * which downloads the pages using raw HTTP requests and is about 10x faster.
92
71
  *
93
- * The source URLs are represented using {@link Request} objects that are fed from
94
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
95
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
72
+ * The source URLs are represented using {@link Request} objects that are fed from the
73
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
74
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
75
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
76
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
77
+ * result as `requestManager`.
96
78
  *
97
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
98
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
99
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
79
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
80
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
100
81
  *
101
82
  * The crawler finishes when there are no more {@link Request} objects to crawl.
102
83
  *
103
84
  * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
104
85
  * and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
105
86
  *
106
- * New pages are only opened when there is enough free CPU and memory available,
107
- * using the functionality provided by the {@link AutoscaledPool} class.
108
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
109
- * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
110
- * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
87
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
88
+ * {@link ConcurrencySystem}.
89
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
90
+ * `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
91
+ * {@link ConcurrencySystem|`concurrencySystem`}.
111
92
  *
112
93
  * Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
113
94
  *
@@ -142,101 +123,150 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
142
123
  * ```
143
124
  * @category Crawlers
144
125
  */
145
- export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response, PlaywrightController, {
146
- browserPlugins: [PlaywrightPlugin];
147
- }, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
148
- readonly config: Configuration;
126
+ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
127
+ #private;
128
+ /**
129
+ * @internal
130
+ */
149
131
  protected static optionsShape: {
150
- // @ts-ignore optional peer dependency or compatibility with es2022
151
- browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
152
- // @ts-ignore optional peer dependency or compatibility with es2022
153
- launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
154
- // @ts-ignore optional peer dependency or compatibility with es2022
155
- ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
156
- // @ts-ignore optional peer dependency or compatibility with es2022
157
- ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
158
- // @ts-ignore optional peer dependency or compatibility with es2022
159
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
160
- // @ts-ignore optional peer dependency or compatibility with es2022
161
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
162
- // @ts-ignore optional peer dependency or compatibility with es2022
163
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
164
- // @ts-ignore optional peer dependency or compatibility with es2022
165
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
166
- // @ts-ignore optional peer dependency or compatibility with es2022
167
- headless: import("ow").AnyPredicate<string | boolean>;
168
- // @ts-ignore optional peer dependency or compatibility with es2022
169
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
170
- // @ts-ignore optional peer dependency or compatibility with es2022
171
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
172
- // @ts-ignore optional peer dependency or compatibility with es2022
173
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
174
- // @ts-ignore optional peer dependency or compatibility with es2022
175
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
176
- // @ts-ignore optional peer dependency or compatibility with es2022
177
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
178
- // @ts-ignore optional peer dependency or compatibility with es2022
179
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
180
- // @ts-ignore optional peer dependency or compatibility with es2022
181
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
182
- // @ts-ignore optional peer dependency or compatibility with es2022
183
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
184
- // @ts-ignore optional peer dependency or compatibility with es2022
185
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
186
- // @ts-ignore optional peer dependency or compatibility with es2022
187
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
188
- // @ts-ignore optional peer dependency or compatibility with es2022
189
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
190
- // @ts-ignore optional peer dependency or compatibility with es2022
191
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
192
- // @ts-ignore optional peer dependency or compatibility with es2022
193
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
194
- // @ts-ignore optional peer dependency or compatibility with es2022
195
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
196
- // @ts-ignore optional peer dependency or compatibility with es2022
197
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
198
- // @ts-ignore optional peer dependency or compatibility with es2022
199
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
200
- // @ts-ignore optional peer dependency or compatibility with es2022
201
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
202
- // @ts-ignore optional peer dependency or compatibility with es2022
203
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
204
- // @ts-ignore optional peer dependency or compatibility with es2022
205
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
206
- // @ts-ignore optional peer dependency or compatibility with es2022
207
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
208
- // @ts-ignore optional peer dependency or compatibility with es2022
209
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
210
- // @ts-ignore optional peer dependency or compatibility with es2022
211
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
212
- // @ts-ignore optional peer dependency or compatibility with es2022
213
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
214
- // @ts-ignore optional peer dependency or compatibility with es2022
215
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
216
- // @ts-ignore optional peer dependency or compatibility with es2022
217
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
218
- // @ts-ignore optional peer dependency or compatibility with es2022
219
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
220
- // @ts-ignore optional peer dependency or compatibility with es2022
221
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
222
- // @ts-ignore optional peer dependency or compatibility with es2022
223
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
224
- // @ts-ignore optional peer dependency or compatibility with es2022
225
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
226
- // @ts-ignore optional peer dependency or compatibility with es2022
227
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
228
- // @ts-ignore optional peer dependency or compatibility with es2022
229
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
230
- // @ts-ignore optional peer dependency or compatibility with es2022
231
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
132
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
133
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
134
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
135
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
136
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
137
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
138
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
139
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
140
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
141
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
142
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
143
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
144
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
145
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
146
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
147
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
148
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
149
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
150
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
151
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
152
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
153
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
154
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
155
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
156
+ requestQueue: z.ZodOptional<z.ZodEnum<{
157
+ deferred: "deferred";
158
+ writeThrough: "writeThrough";
159
+ }>>;
160
+ }, z.core.$strict>]>>;
161
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
162
+ // @ts-ignore optional peer dependency or compatibility with es2022
163
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
164
+ // @ts-ignore optional peer dependency or compatibility with es2022
165
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
166
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
167
+ // @ts-ignore optional peer dependency or compatibility with es2022
168
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
169
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
170
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
171
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
172
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
173
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
174
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
175
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
176
+ id: z.ZodOptional<z.ZodString>;
177
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
178
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
179
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
180
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
181
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
182
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
183
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
184
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
185
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
186
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
187
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
188
+ headless: z.ZodOptional<z.ZodBoolean>;
232
189
  };
190
+ /** @internal */
191
+ protected static optionsSchema: z.ZodObject<{
192
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
193
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
194
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
195
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
196
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
197
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
198
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
199
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
200
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
201
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
202
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
203
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
204
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
205
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
206
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
207
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
208
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
209
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
210
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
211
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
212
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
213
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
214
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
215
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
216
+ requestQueue: z.ZodOptional<z.ZodEnum<{
217
+ deferred: "deferred";
218
+ writeThrough: "writeThrough";
219
+ }>>;
220
+ }, z.core.$strict>]>>;
221
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
222
+ // @ts-ignore optional peer dependency or compatibility with es2022
223
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
224
+ // @ts-ignore optional peer dependency or compatibility with es2022
225
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
226
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
227
+ // @ts-ignore optional peer dependency or compatibility with es2022
228
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
229
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
230
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
231
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
232
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
233
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
234
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
235
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
236
+ id: z.ZodOptional<z.ZodString>;
237
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
238
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
239
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
240
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
241
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
242
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
243
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
244
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
245
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
246
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
247
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
248
+ headless: z.ZodOptional<z.ZodBoolean>;
249
+ }, z.core.$strict>;
233
250
  /**
234
251
  * All `PlaywrightCrawler` parameters are passed via an options object.
235
252
  */
236
- constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>, config?: Configuration);
237
- protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
238
- private enhanceContext;
253
+ constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
254
+ protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
239
255
  }
256
+ /**
257
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
258
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
259
+ *
260
+ * **Example usage**
261
+ * ```ts
262
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
263
+ *
264
+ * const crawler = new PlaywrightCrawler({
265
+ * postNavigationHooks: [handleCloudflareChallengeHook()],
266
+ * });
267
+ * ```
268
+ */
269
+ export declare function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
240
270
  /**
241
271
  * Creates new {@link Router} instance that works based on request labels.
242
272
  * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
@@ -261,6 +291,6 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
261
291
  * await crawler.run();
262
292
  * ```
263
293
  */
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
266
- //# sourceMappingURL=playwright-crawler.d.ts.map
294
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
295
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
296
+ export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;