@crawlee/playwright 4.0.0-beta.21 → 4.0.0-beta.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/index.d.ts +4 -3
- package/index.js +2 -2
- package/internals/adaptive-playwright-crawler.d.ts +136 -65
- package/internals/adaptive-playwright-crawler.js +417 -274
- package/internals/enqueue-links/click-elements.d.ts +36 -64
- package/internals/enqueue-links/click-elements.js +65 -67
- package/internals/playwright-browser-pool.d.ts +71 -0
- package/internals/playwright-browser-pool.js +61 -0
- package/internals/playwright-crawler.d.ts +176 -148
- package/internals/playwright-crawler.js +77 -73
- package/internals/playwright-launcher.d.ts +30 -20
- package/internals/playwright-launcher.js +22 -17
- package/internals/utils/playwright-utils.d.ts +54 -56
- package/internals/utils/playwright-utils.js +117 -138
- package/internals/utils/rendering-type-prediction.d.ts +37 -11
- package/internals/utils/rendering-type-prediction.js +81 -27
- package/package.json +15 -19
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
- package/internals/adaptive-playwright-crawler.js.map +0 -1
- package/internals/enqueue-links/click-elements.d.ts.map +0 -1
- package/internals/enqueue-links/click-elements.js.map +0 -1
- package/internals/playwright-crawler.d.ts.map +0 -1
- package/internals/playwright-crawler.js.map +0 -1
- package/internals/playwright-launcher.d.ts.map +0 -1
- package/internals/playwright-launcher.js.map +0 -1
- package/internals/utils/playwright-utils.d.ts.map +0 -1
- package/internals/utils/playwright-utils.js.map +0 -1
- package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
- package/internals/utils/rendering-type-prediction.js.map +0 -1
|
@@ -1,70 +1,46 @@
|
|
|
1
|
-
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest,
|
|
2
|
-
import { BrowserCrawler
|
|
3
|
-
import type { PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
1
|
+
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
2
|
+
import { BrowserCrawler } from '@crawlee/browser';
|
|
4
3
|
import type { Dictionary } from '@crawlee/types';
|
|
5
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
|
-
import type {
|
|
5
|
+
import type { Page, Response } from 'playwright';
|
|
6
|
+
import { z } from 'zod';
|
|
7
7
|
import type { PlaywrightLaunchContext } from './playwright-launcher.js';
|
|
8
|
-
import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
|
|
9
|
-
export
|
|
8
|
+
import type { DirectNavigationOptions, HandleCloudflareChallengeOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
|
|
9
|
+
export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
10
|
+
export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
|
|
10
11
|
}
|
|
11
|
-
|
|
12
|
-
export interface
|
|
13
|
-
}
|
|
14
|
-
export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
|
|
15
|
-
export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response, PlaywrightController, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
|
|
16
|
-
browserPlugins: [PlaywrightPlugin];
|
|
17
|
-
}> {
|
|
12
|
+
export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
|
|
13
|
+
export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
18
14
|
/**
|
|
19
15
|
* The same options as used by {@link launchPlaywright}.
|
|
20
16
|
*/
|
|
21
17
|
launchContext?: PlaywrightLaunchContext;
|
|
22
18
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
|
|
26
|
-
* - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
|
|
27
|
-
* - `page` is an instance of the `Playwright`
|
|
28
|
-
* [`Page`](https://playwright.dev/docs/api/class-page)
|
|
29
|
-
* - `browserController` is an instance of the
|
|
30
|
-
* [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
|
|
31
|
-
* - `response` is an instance of the `Playwright`
|
|
32
|
-
* [`Response`](https://playwright.dev/docs/api/class-response),
|
|
33
|
-
* which is the main resource response as returned by `page.goto(request.url)`.
|
|
34
|
-
*
|
|
35
|
-
* The function must return a promise, which is then awaited by the crawler.
|
|
36
|
-
*
|
|
37
|
-
* If the function throws an exception, the crawler will try to re-crawl the
|
|
38
|
-
* request later, up to `option.maxRequestRetries` times.
|
|
39
|
-
* If all the retries fail, the crawler calls the function
|
|
40
|
-
* provided to the `failedRequestHandler` parameter.
|
|
41
|
-
* To make this work, you should **always**
|
|
42
|
-
* let your function throw exceptions rather than catch them.
|
|
43
|
-
* The exceptions are logged to the request using the
|
|
44
|
-
* {@link Request.pushErrorMessage} function.
|
|
19
|
+
* Whether to run browser in headless mode. Defaults to `true`.
|
|
20
|
+
* Can be also set via {@link Configuration}.
|
|
45
21
|
*/
|
|
46
|
-
|
|
22
|
+
headless?: boolean;
|
|
47
23
|
/**
|
|
48
24
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
49
|
-
* or browser properties before navigation. The function
|
|
50
|
-
*
|
|
25
|
+
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
26
|
+
* forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
|
|
27
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context
|
|
28
|
+
* (e.g. to override context members for subsequent hooks and pipeline stages).
|
|
51
29
|
* Example:
|
|
52
30
|
* ```
|
|
53
31
|
* preNavigationHooks: [
|
|
54
|
-
* async (
|
|
55
|
-
* const { page } = crawlingContext;
|
|
32
|
+
* async ({ page, gotoOptions }) => {
|
|
56
33
|
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
34
|
+
* gotoOptions.timeout = 60_000;
|
|
57
35
|
* },
|
|
58
36
|
* ]
|
|
59
37
|
* ```
|
|
60
|
-
*
|
|
61
|
-
* Modyfing `pageOptions` is supported only in Playwright incognito.
|
|
62
|
-
* See {@link PrePageCreateHook}
|
|
63
38
|
*/
|
|
64
|
-
preNavigationHooks?:
|
|
39
|
+
preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
65
40
|
/**
|
|
66
41
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
67
|
-
* The function accepts `crawlingContext` as the only parameter.
|
|
42
|
+
* The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
|
|
43
|
+
* whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
|
|
68
44
|
* Example:
|
|
69
45
|
* ```
|
|
70
46
|
* postNavigationHooks: [
|
|
@@ -77,7 +53,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
|
|
|
77
53
|
* ]
|
|
78
54
|
* ```
|
|
79
55
|
*/
|
|
80
|
-
postNavigationHooks?:
|
|
56
|
+
postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
81
57
|
}
|
|
82
58
|
/**
|
|
83
59
|
* Provides a simple framework for parallel crawling of web pages
|
|
@@ -90,24 +66,26 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
|
|
|
90
66
|
* If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
|
|
91
67
|
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
92
68
|
*
|
|
93
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
94
|
-
* {@link
|
|
95
|
-
*
|
|
69
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
70
|
+
* {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
|
|
71
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
72
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
73
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
74
|
+
* result as `requestManager`.
|
|
96
75
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
76
|
+
* > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
|
|
77
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
100
78
|
*
|
|
101
79
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
102
80
|
*
|
|
103
81
|
* `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
|
|
104
82
|
* and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
|
|
105
83
|
*
|
|
106
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
* {@link
|
|
84
|
+
* New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
|
|
85
|
+
* {@link ConcurrencySystem}.
|
|
86
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
87
|
+
* `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
88
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
111
89
|
*
|
|
112
90
|
* Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
|
|
113
91
|
*
|
|
@@ -142,101 +120,151 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
|
|
|
142
120
|
* ```
|
|
143
121
|
* @category Crawlers
|
|
144
122
|
*/
|
|
145
|
-
export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response,
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
123
|
+
export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
124
|
+
#private;
|
|
125
|
+
/**
|
|
126
|
+
* @internal
|
|
127
|
+
*/
|
|
149
128
|
protected static optionsShape: {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
181
|
-
|
|
182
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
208
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
209
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
210
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
211
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
212
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
213
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
214
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
215
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
216
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
217
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
218
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
219
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
220
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
221
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
222
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
223
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
224
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
225
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
226
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
227
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
228
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
229
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
230
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
231
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
129
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
130
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
131
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
132
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
133
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
134
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
135
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
136
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
137
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
138
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
139
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
140
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
141
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
142
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
143
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
144
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
145
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
146
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
147
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
148
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
149
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
150
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
151
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
152
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
153
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
154
|
+
deferred: "deferred";
|
|
155
|
+
writeThrough: "writeThrough";
|
|
156
|
+
}>>;
|
|
157
|
+
}, z.core.$strict>]>>;
|
|
158
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
159
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
160
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
161
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
162
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
163
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
164
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
165
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
166
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
167
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
168
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
169
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
170
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
171
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
172
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
173
|
+
id: z.ZodOptional<z.ZodString>;
|
|
174
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
175
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
176
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
177
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
178
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
179
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
180
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
181
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
182
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
183
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
184
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
185
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
232
186
|
};
|
|
187
|
+
/** @internal */
|
|
188
|
+
protected static optionsSchema: z.ZodObject<{
|
|
189
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
190
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
191
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
192
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
193
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
194
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
195
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
196
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
197
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
198
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
199
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
200
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
201
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
202
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
203
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
204
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
205
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
206
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
207
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
208
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
209
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
210
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
211
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
212
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
213
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
214
|
+
deferred: "deferred";
|
|
215
|
+
writeThrough: "writeThrough";
|
|
216
|
+
}>>;
|
|
217
|
+
}, z.core.$strict>]>>;
|
|
218
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
219
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
220
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
221
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
222
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
223
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
224
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
225
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
226
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
227
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
228
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
229
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
230
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
231
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
232
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
233
|
+
id: z.ZodOptional<z.ZodString>;
|
|
234
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
235
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
236
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
237
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
238
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
239
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
240
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
241
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
242
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
243
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
244
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
245
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
246
|
+
}, z.core.$strict>;
|
|
233
247
|
/**
|
|
234
248
|
* All `PlaywrightCrawler` parameters are passed via an options object.
|
|
235
249
|
*/
|
|
236
|
-
constructor(options?: PlaywrightCrawlerOptions<ExtendedContext
|
|
237
|
-
protected
|
|
250
|
+
constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
251
|
+
protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
|
|
238
252
|
private enhanceContext;
|
|
239
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
|
|
256
|
+
* and propagates the post-challenge {@link Response} back into the crawling context via its return value.
|
|
257
|
+
*
|
|
258
|
+
* **Example usage**
|
|
259
|
+
* ```ts
|
|
260
|
+
* import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
|
|
261
|
+
*
|
|
262
|
+
* const crawler = new PlaywrightCrawler({
|
|
263
|
+
* postNavigationHooks: [handleCloudflareChallengeHook()],
|
|
264
|
+
* });
|
|
265
|
+
* ```
|
|
266
|
+
*/
|
|
267
|
+
export declare function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
|
|
240
268
|
/**
|
|
241
269
|
* Creates new {@link Router} instance that works based on request labels.
|
|
242
270
|
* This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
|
|
@@ -261,6 +289,6 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
|
|
|
261
289
|
* await crawler.run();
|
|
262
290
|
* ```
|
|
263
291
|
*/
|
|
264
|
-
|
|
265
|
-
export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData
|
|
266
|
-
|
|
292
|
+
export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
293
|
+
export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
294
|
+
export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|