@crawlee/browser 4.0.0-beta.29 → 4.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/internals/browser-crawler.d.ts +242 -187
- package/internals/browser-crawler.js +337 -270
- package/internals/browser-launcher.d.ts +71 -27
- package/internals/browser-launcher.js +78 -26
- package/package.json +11 -12
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/browser-crawler.d.ts.map +0 -1
- package/internals/browser-crawler.js.map +0 -1
- package/internals/browser-launcher.d.ts.map +0 -1
- package/internals/browser-launcher.js.map +0 -1
|
@@ -1,19 +1,26 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
|
-
import type {
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
import type {
|
|
7
|
-
import type { ReadonlyDeep } from 'type-fest';
|
|
8
|
-
import type { BrowserLaunchContext } from './browser-launcher.js';
|
|
3
|
+
import type { CommonPage } from '@crawlee/browser-pool';
|
|
4
|
+
import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
|
|
9
7
|
interface BaseResponse {
|
|
10
8
|
status(): number;
|
|
9
|
+
/** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
|
|
10
|
+
headers?(): Record<string, string>;
|
|
11
11
|
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
/**
|
|
13
|
+
* The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
|
|
14
|
+
* additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
|
|
15
|
+
* {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
|
|
16
|
+
* `destroy()` once the crawler itself is destroyed.
|
|
17
|
+
*/
|
|
18
|
+
export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
|
|
19
|
+
releaseAllBrowsers: () => Promise<void>;
|
|
20
|
+
destroy: () => Promise<void>;
|
|
21
|
+
};
|
|
22
|
+
export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
23
|
+
GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
17
24
|
/**
|
|
18
25
|
* The browser page object where the web page is loaded and rendered.
|
|
19
26
|
*/
|
|
@@ -21,19 +28,53 @@ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Re
|
|
|
21
28
|
/**
|
|
22
29
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
23
30
|
*/
|
|
24
|
-
request: LoadedRequest<
|
|
31
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
25
32
|
/**
|
|
26
33
|
* The HTTP response object returned by the browser's navigation.
|
|
27
34
|
*/
|
|
28
35
|
response: Response;
|
|
36
|
+
/**
|
|
37
|
+
* Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
|
|
38
|
+
* object (or return `{ gotoOptions: ... }`) to influence the navigation.
|
|
39
|
+
*/
|
|
40
|
+
gotoOptions: GoToOptions;
|
|
41
|
+
/**
|
|
42
|
+
* Extracts URLs from the current page, without adding them to the request queue.
|
|
43
|
+
*/
|
|
44
|
+
extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
|
|
29
45
|
/**
|
|
30
46
|
* Helper function for extracting URLs from the current page and adding them to the request queue.
|
|
31
47
|
*/
|
|
32
|
-
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<
|
|
48
|
+
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
|
|
33
49
|
}
|
|
34
|
-
export type BrowserHook<Context = BrowserCrawlingContext,
|
|
35
|
-
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse,
|
|
36
|
-
|
|
50
|
+
export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
|
|
51
|
+
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
|
|
52
|
+
/**
|
|
53
|
+
* The browser pool the crawler should serve its pages from. This is the single way to run a pool with
|
|
54
|
+
* non-default options: build one with the factory that matches your crawler
|
|
55
|
+
* ({@link playwrightBrowserPool}, {@link puppeteerBrowserPool}, {@link stagehandBrowserPool}) - it
|
|
56
|
+
* accepts every {@link BrowserPoolOptions|`BrowserPool` option} and supplies the correct browser plugin
|
|
57
|
+
* itself, so the pool can never mismatch the crawler.
|
|
58
|
+
*
|
|
59
|
+
* A pool passed in this way is borrowed, not owned: the crawler will not tear it down, which is what makes it
|
|
60
|
+
* shareable across crawlers. Since the crawler then builds nothing itself, the options that configure its own
|
|
61
|
+
* pool (`launchContext`, `headless`, `remoteBrowser`) are rejected rather than silently ignored.
|
|
62
|
+
*
|
|
63
|
+
* When omitted, the crawler builds - and tears down - a default pool for its own browser.
|
|
64
|
+
*/
|
|
65
|
+
browserPool?: IBrowserPool<Page>;
|
|
66
|
+
/**
|
|
67
|
+
* Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
|
|
68
|
+
*
|
|
69
|
+
* The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
|
|
70
|
+
* for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
|
|
71
|
+
* per launch, or a {@link RemoteBrowserProvider}.
|
|
72
|
+
*
|
|
73
|
+
* Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
|
|
74
|
+
* `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
|
|
75
|
+
* {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
|
|
76
|
+
*/
|
|
77
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
37
78
|
/**
|
|
38
79
|
* Function that is called to process each request.
|
|
39
80
|
*
|
|
@@ -44,7 +85,6 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
44
85
|
* - {@link BrowserCrawlingContext.page|`page`} is an instance of the
|
|
45
86
|
* Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
|
|
46
87
|
* Playwright [Page](https://playwright.dev/docs/api/class-page);
|
|
47
|
-
* - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
|
|
48
88
|
* - {@link BrowserCrawlingContext.response|`response`} is an instance of the
|
|
49
89
|
* Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
|
|
50
90
|
* Playwright [Response](https://playwright.dev/docs/api/class-response),
|
|
@@ -59,9 +99,9 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
59
99
|
* To make this work, we should **always**
|
|
60
100
|
* let our function throw exceptions rather than catch them.
|
|
61
101
|
* The exceptions are logged to the request using the
|
|
62
|
-
* {@link
|
|
102
|
+
* {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
|
|
63
103
|
*/
|
|
64
|
-
requestHandler?: RequestHandler<ExtendedContext>;
|
|
104
|
+
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
65
105
|
/**
|
|
66
106
|
* User-provided function that allows modifying the request object before it gets retried by the crawler.
|
|
67
107
|
* It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
|
|
@@ -83,22 +123,16 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
83
123
|
* represents the last error thrown during processing of the request.
|
|
84
124
|
*/
|
|
85
125
|
failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
|
86
|
-
/**
|
|
87
|
-
* Custom options passed to the underlying {@link BrowserPool} constructor.
|
|
88
|
-
* We can tweak those to fine-tune browser management.
|
|
89
|
-
*/
|
|
90
|
-
browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
|
|
91
126
|
/**
|
|
92
127
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
93
|
-
* or browser properties before navigation. The function
|
|
94
|
-
*
|
|
128
|
+
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
129
|
+
* forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
|
|
95
130
|
*
|
|
96
131
|
* **Example:**
|
|
97
132
|
*
|
|
98
133
|
* ```js
|
|
99
134
|
* preNavigationHooks: [
|
|
100
|
-
* async (
|
|
101
|
-
* const { page } = crawlingContext;
|
|
135
|
+
* async ({ page, gotoOptions }) => {
|
|
102
136
|
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
103
137
|
* gotoOptions.timeout = 60_000;
|
|
104
138
|
* gotoOptions.waitUntil = 'domcontentloaded';
|
|
@@ -106,14 +140,22 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
106
140
|
* ]
|
|
107
141
|
* ```
|
|
108
142
|
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
143
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
144
|
+
* allowing the hook to override context members for subsequent hooks and pipeline stages.
|
|
145
|
+
*
|
|
146
|
+
* The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
|
|
147
|
+
* `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
|
|
148
|
+
* This means the members added by `extendContext` are already available here, but navigation-dependent
|
|
149
|
+
* members (e.g. `page`, `response`) are not.
|
|
111
150
|
*/
|
|
112
|
-
preNavigationHooks?: BrowserHook<Context>[];
|
|
151
|
+
preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
|
|
113
152
|
/**
|
|
114
153
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
115
154
|
* The function accepts `crawlingContext` as the only parameter.
|
|
116
155
|
*
|
|
156
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context.
|
|
157
|
+
* This is useful for overriding context members (e.g. `response`) after solving a challenge.
|
|
158
|
+
*
|
|
117
159
|
* **Example:**
|
|
118
160
|
*
|
|
119
161
|
* ```js
|
|
@@ -124,24 +166,27 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
124
166
|
* await solveCaptcha(page);
|
|
125
167
|
* }
|
|
126
168
|
* },
|
|
169
|
+
* async (crawlingContext) => {
|
|
170
|
+
* if (await needsRevalidation(crawlingContext)) {
|
|
171
|
+
* return { response: await crawlingContext.page.reload() };
|
|
172
|
+
* }
|
|
173
|
+
* },
|
|
127
174
|
* ]
|
|
128
175
|
* ```
|
|
129
176
|
*/
|
|
130
|
-
postNavigationHooks?: BrowserHook<Context>[];
|
|
177
|
+
postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
|
|
131
178
|
/**
|
|
132
|
-
* Timeout
|
|
179
|
+
* Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,
|
|
180
|
+
* the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the
|
|
181
|
+
* navigation uses. Separate from the
|
|
182
|
+
* {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
|
|
183
|
+
* request handler.
|
|
133
184
|
*/
|
|
134
185
|
navigationTimeoutSecs?: number;
|
|
135
186
|
/**
|
|
136
|
-
* Defines whether the cookies should be persisted for sessions.
|
|
137
|
-
* This can only be used when `useSessionPool` is set to `true`.
|
|
187
|
+
* Defines whether the cookies should be persisted for sessions. Enabled by default.
|
|
138
188
|
*/
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Whether to run browser in headless mode. Defaults to `true`.
|
|
142
|
-
* Can be also set via {@link Configuration}.
|
|
143
|
-
*/
|
|
144
|
-
headless?: boolean | 'new' | 'old';
|
|
189
|
+
saveResponseCookies?: boolean;
|
|
145
190
|
/**
|
|
146
191
|
* Whether to ignore custom elements (and their #shadow-roots) when processing the page content via `parseWithCheerio` helper.
|
|
147
192
|
* By default, they are expanded automatically. Use this option to disable this behavior.
|
|
@@ -165,181 +210,192 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
165
210
|
* If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
|
|
166
211
|
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
167
212
|
*
|
|
168
|
-
* The source URLs are represented by the {@link Request} objects that are fed from the
|
|
169
|
-
*
|
|
170
|
-
* constructor
|
|
213
|
+
* The source URLs are represented by the {@link Request} objects that are fed from the
|
|
214
|
+
* {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
|
|
215
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
|
|
171
216
|
* the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
172
217
|
* or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
|
|
173
218
|
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
219
|
+
* To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
|
|
220
|
+
* combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
|
|
221
|
+
* and pass the result as `requestManager`.
|
|
222
|
+
*
|
|
223
|
+
* > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
|
|
224
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
177
225
|
*
|
|
178
226
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
179
227
|
*
|
|
180
228
|
* `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
|
|
181
229
|
* and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
|
|
182
230
|
*
|
|
183
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
*
|
|
188
|
-
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
189
|
-
* underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
|
|
231
|
+
* New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
|
|
232
|
+
* {@link ConcurrencySystem}.
|
|
233
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
234
|
+
* `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
235
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
190
236
|
*
|
|
191
237
|
* > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
|
|
192
238
|
*
|
|
193
239
|
* @category Crawlers
|
|
194
240
|
*/
|
|
195
|
-
export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse,
|
|
241
|
+
export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
242
|
+
#private;
|
|
196
243
|
/**
|
|
197
|
-
* A reference to the underlying
|
|
244
|
+
* A reference to the underlying browser pool that manages the crawler's browsers. Typed as
|
|
245
|
+
* {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
|
|
198
246
|
*/
|
|
199
|
-
browserPool:
|
|
200
|
-
launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
|
|
247
|
+
get browserPool(): IBrowserPool<Page>;
|
|
201
248
|
protected readonly ignoreShadowRoots: boolean;
|
|
202
249
|
protected readonly ignoreIframes: boolean;
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
protected persistCookiesPerSession: boolean;
|
|
250
|
+
/**
|
|
251
|
+
* @internal
|
|
252
|
+
*/
|
|
207
253
|
protected static optionsShape: {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
239
|
-
|
|
240
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
264
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
265
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
266
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
267
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
268
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
269
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
270
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
271
|
-
storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
272
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
273
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
274
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
275
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
276
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
277
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
278
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
279
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
280
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
281
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
282
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
283
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
284
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
285
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
286
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
287
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
288
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
289
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
254
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
255
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
256
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
257
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
258
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
259
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
260
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
261
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
262
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
263
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
264
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
265
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
266
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
267
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
268
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
269
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
270
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
271
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
272
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
273
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
274
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
275
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
276
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
277
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
278
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
279
|
+
deferred: "deferred";
|
|
280
|
+
writeThrough: "writeThrough";
|
|
281
|
+
}>>;
|
|
282
|
+
}, z.core.$strict>]>>;
|
|
283
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
284
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
285
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
286
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
287
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
288
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
289
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
290
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
291
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
292
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
293
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
294
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
295
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
296
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
297
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
298
|
+
id: z.ZodOptional<z.ZodString>;
|
|
299
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
300
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
301
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
302
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
303
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
304
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
305
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
306
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
307
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
308
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
290
309
|
};
|
|
310
|
+
/** @internal */
|
|
311
|
+
protected static optionsSchema: z.ZodObject<{
|
|
312
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
313
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
314
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
315
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
316
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
317
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
318
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
319
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
320
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
321
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
322
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
323
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
324
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
325
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
326
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
327
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
328
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
329
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
330
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
331
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
332
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
333
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
334
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
335
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
336
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
337
|
+
deferred: "deferred";
|
|
338
|
+
writeThrough: "writeThrough";
|
|
339
|
+
}>>;
|
|
340
|
+
}, z.core.$strict>]>>;
|
|
341
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
342
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
343
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
344
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
345
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
346
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
347
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
348
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
349
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
350
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
351
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
352
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
353
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
354
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
355
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
356
|
+
id: z.ZodOptional<z.ZodString>;
|
|
357
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
358
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
359
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
360
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
361
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
362
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
363
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
364
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
365
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
366
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
367
|
+
}, z.core.$strict>;
|
|
291
368
|
/**
|
|
292
369
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
370
|
+
* @internal
|
|
293
371
|
*/
|
|
294
|
-
protected constructor(options: BrowserCrawlerOptions<Page, Response,
|
|
372
|
+
protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
|
|
295
373
|
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
|
|
374
|
+
/**
|
|
375
|
+
* Builds the pool the crawler owns, used only when the user injected no `browserPool`. Supplied by the
|
|
376
|
+
* concrete crawler, which is the only place that knows which browser plugin to run - that is also why
|
|
377
|
+
* `remoteBrowser` is handed to it rather than acted upon here.
|
|
378
|
+
*/
|
|
379
|
+
browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
|
|
296
380
|
});
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
protected
|
|
300
|
-
private
|
|
301
|
-
private performNavigation;
|
|
302
|
-
private handleBlockedRequestByContent;
|
|
303
|
-
private restoreRequestState;
|
|
304
|
-
protected _applyCookies({ session, request, page, browserController }: BrowserCrawlingContext, preHooksCookies: string, postHooksCookies: string): Promise<void>;
|
|
381
|
+
/** @internal */
|
|
382
|
+
protected getNavigationTimeoutMillis(): number;
|
|
383
|
+
protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
|
|
384
|
+
private navigate;
|
|
305
385
|
/**
|
|
306
|
-
*
|
|
386
|
+
* Runs the user request handler, then re-reads browser cookies so login flows /
|
|
387
|
+
* `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
|
|
388
|
+
* @internal
|
|
307
389
|
*/
|
|
308
|
-
protected
|
|
390
|
+
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
|
391
|
+
protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
|
|
309
392
|
/**
|
|
310
|
-
*
|
|
311
|
-
|
|
312
|
-
protected _throwIfProxyError(error: Error): void;
|
|
313
|
-
protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
|
|
314
|
-
private processResponse;
|
|
315
|
-
protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
|
|
316
|
-
protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
|
|
317
|
-
/**
|
|
318
|
-
* Function for cleaning up after all requests are processed.
|
|
319
|
-
* @ignore
|
|
393
|
+
* Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
|
|
394
|
+
* crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
|
|
320
395
|
*/
|
|
321
396
|
teardown(): Promise<void>;
|
|
397
|
+
destroy(): Promise<void>;
|
|
322
398
|
}
|
|
323
|
-
/** @internal */
|
|
324
|
-
interface EnqueueLinksInternalOptions {
|
|
325
|
-
options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
|
|
326
|
-
page: CommonPage;
|
|
327
|
-
requestQueue: RequestProvider;
|
|
328
|
-
robotsTxtFile?: RobotsTxtFile;
|
|
329
|
-
onSkippedRequest?: SkippedRequestCallback;
|
|
330
|
-
originalRequestUrl: string;
|
|
331
|
-
finalRequestUrl?: string;
|
|
332
|
-
}
|
|
333
|
-
/** @internal */
|
|
334
|
-
interface BoundEnqueueLinksInternalOptions {
|
|
335
|
-
enqueueLinks: BasicCrawlingContext['enqueueLinks'];
|
|
336
|
-
options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
|
|
337
|
-
originalRequestUrl: string;
|
|
338
|
-
finalRequestUrl?: string;
|
|
339
|
-
page: CommonPage;
|
|
340
|
-
}
|
|
341
|
-
/** @internal */
|
|
342
|
-
export declare function browserCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
|
343
399
|
/**
|
|
344
400
|
* Extracts URLs from a given page.
|
|
345
401
|
* @ignore
|
|
@@ -348,4 +404,3 @@ export declare function extractUrlsFromPage(page: {
|
|
|
348
404
|
$$eval: Function;
|
|
349
405
|
}, selector: string, baseUrl: string): Promise<string[]>;
|
|
350
406
|
export {};
|
|
351
|
-
//# sourceMappingURL=browser-crawler.d.ts.map
|