@crawlee/browser 4.0.0-beta.2 → 4.0.0-beta.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,20 +1,80 @@
1
- import type { Awaitable, BasicCrawlerOptions, CrawlingContext, Dictionary, EnqueueLinksOptions, ErrorHandler, LoadedContext, ProxyConfiguration, RequestHandler, RequestProvider, SkippedRequestCallback } from '@crawlee/basic';
2
- import { BasicCrawler, Configuration } from '@crawlee/basic';
3
- import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
4
- import { BrowserPool } from '@crawlee/browser-pool';
5
- import type { RobotsTxtFile } from '@crawlee/utils';
6
- import type { ReadonlyDeep } from 'type-fest';
7
- import type { BrowserLaunchContext } from './browser-launcher.js';
8
- export interface BrowserCrawlingContext<Crawler = unknown, Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<Crawler, UserData> {
9
- browserController: ProvidedController;
1
+ import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
2
+ import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
+ import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
4
+ import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
+ import { z } from 'zod';
6
+ interface BaseResponse {
7
+ status(): number;
8
+ /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
9
+ headers?(): Record<string, string>;
10
+ }
11
+ /**
12
+ * The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
13
+ * additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
14
+ * {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
15
+ * `destroy()` once the crawler itself is destroyed.
16
+ */
17
+ export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
18
+ releaseAllBrowsers: () => Promise<void>;
19
+ destroy: () => Promise<void>;
20
+ };
21
+ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
22
+ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
23
+ /**
24
+ * The browser page object where the web page is loaded and rendered.
25
+ */
10
26
  page: Page;
11
- response?: Response;
27
+ /**
28
+ * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
29
+ */
30
+ request: LoadedRequest<CrawlingRequest<UserData>>;
31
+ /**
32
+ * The HTTP response object returned by the browser's navigation.
33
+ */
34
+ response: Response;
35
+ /**
36
+ * Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
37
+ * object (or return `{ gotoOptions: ... }`) to influence the navigation.
38
+ */
39
+ gotoOptions: GoToOptions;
40
+ /**
41
+ * Extracts URLs from the current page, without adding them to the request queue.
42
+ */
43
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
44
+ /**
45
+ * Helper function for extracting URLs from the current page and adding them to the request queue.
46
+ */
47
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
12
48
  }
13
- export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;
14
- export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = ErrorHandler<Context>;
15
- export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
16
- export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
17
- launchContext?: BrowserLaunchContext<any, any>;
49
+ export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
50
+ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
51
+ /**
52
+ * The browser pool the crawler should serve its pages from. This is the single way to run a pool with
53
+ * non-default options: build one with the factory that matches your crawler
54
+ * ({@link playwrightBrowserPool}, {@link puppeteerBrowserPool}, {@link stagehandBrowserPool}) - it
55
+ * accepts every {@link BrowserPoolOptions|`BrowserPool` option} and supplies the correct browser plugin
56
+ * itself, so the pool can never mismatch the crawler.
57
+ *
58
+ * A pool passed in this way is borrowed, not owned: the crawler will not tear it down, which is what makes it
59
+ * shareable across crawlers. Since the crawler then builds nothing itself, the options that configure its own
60
+ * pool (`launchContext`, `headless`, `remoteBrowser`) are rejected rather than silently ignored.
61
+ *
62
+ * When omitted, the crawler builds - and tears down - a default pool for its own browser.
63
+ */
64
+ browserPool?: IBrowserPool<Page>;
65
+ /**
66
+ * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
67
+ *
68
+ * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
69
+ * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
70
+ * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
71
+ * or a {@link RemoteBrowserProvider}.
72
+ *
73
+ * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it
74
+ * across crawlers, build it with the remote factory for your crawler ({@link remotePlaywrightBrowserPool},
75
+ * {@link remotePuppeteerBrowserPool}, {@link remoteStagehandBrowserPool}) and pass it as `browserPool`.
76
+ */
77
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
18
78
  /**
19
79
  * Function that is called to process each request.
20
80
  *
@@ -25,7 +85,6 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
25
85
  * - {@link BrowserCrawlingContext.page|`page`} is an instance of the
26
86
  * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
27
87
  * Playwright [Page](https://playwright.dev/docs/api/class-page);
28
- * - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
29
88
  * - {@link BrowserCrawlingContext.response|`response`} is an instance of the
30
89
  * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
31
90
  * Playwright [Response](https://playwright.dev/docs/api/class-response),
@@ -40,9 +99,9 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
40
99
  * To make this work, we should **always**
41
100
  * let our function throw exceptions rather than catch them.
42
101
  * The exceptions are logged to the request using the
43
- * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
102
+ * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
44
103
  */
45
- requestHandler?: BrowserRequestHandler<LoadedContext<Context>>;
104
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
46
105
  /**
47
106
  * User-provided function that allows modifying the request object before it gets retried by the crawler.
48
107
  * It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
@@ -53,7 +112,7 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
53
112
  * Second argument is the `Error` instance that
54
113
  * represents the last error thrown during processing of the request.
55
114
  */
56
- errorHandler?: BrowserErrorHandler<Context>;
115
+ errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
57
116
  /**
58
117
  * A function to handle requests that failed more than `option.maxRequestRetries` times.
59
118
  *
@@ -63,28 +122,17 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
63
122
  * Second argument is the `Error` instance that
64
123
  * represents the last error thrown during processing of the request.
65
124
  */
66
- failedRequestHandler?: BrowserErrorHandler<Context>;
67
- /**
68
- * Custom options passed to the underlying {@link BrowserPool} constructor.
69
- * We can tweak those to fine-tune browser management.
70
- */
71
- browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
72
- /**
73
- * If set, the crawler will be configured for all connections to use
74
- * the Proxy URLs provided and rotated according to the configuration.
75
- */
76
- proxyConfiguration?: ProxyConfiguration;
125
+ failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
77
126
  /**
78
127
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
79
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
80
- * which are passed to the `page.goto()` function the crawler calls to navigate.
128
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
129
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
81
130
  *
82
131
  * **Example:**
83
132
  *
84
133
  * ```js
85
134
  * preNavigationHooks: [
86
- * async (crawlingContext, gotoOptions) => {
87
- * const { page } = crawlingContext;
135
+ * async ({ page, gotoOptions }) => {
88
136
  * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
89
137
  * gotoOptions.timeout = 60_000;
90
138
  * gotoOptions.waitUntil = 'domcontentloaded';
@@ -92,14 +140,22 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
92
140
  * ]
93
141
  * ```
94
142
  *
95
- * Modyfing `pageOptions` is supported only in Playwright incognito.
96
- * See {@link PrePageCreateHook}
143
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
144
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
145
+ *
146
+ * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
147
+ * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
148
+ * This means the members added by `extendContext` are already available here, but navigation-dependent
149
+ * members (e.g. `page`, `response`) are not.
97
150
  */
98
- preNavigationHooks?: BrowserHook<Context>[];
151
+ preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
99
152
  /**
100
153
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
101
154
  * The function accepts `crawlingContext` as the only parameter.
102
155
  *
156
+ * A hook may optionally return a partial object whose properties are merged into the crawling context.
157
+ * This is useful for overriding context members (e.g. `response`) after solving a challenge.
158
+ *
103
159
  * **Example:**
104
160
  *
105
161
  * ```js
@@ -110,24 +166,27 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
110
166
  * await solveCaptcha(page);
111
167
  * }
112
168
  * },
169
+ * async (crawlingContext) => {
170
+ * if (await needsRevalidation(crawlingContext)) {
171
+ * return { response: await crawlingContext.page.reload() };
172
+ * }
173
+ * },
113
174
  * ]
114
175
  * ```
115
176
  */
116
- postNavigationHooks?: BrowserHook<Context>[];
177
+ postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
117
178
  /**
118
- * Timeout in which page navigation needs to finish, in seconds.
179
+ * Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,
180
+ * the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the
181
+ * navigation uses. Separate from the
182
+ * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
183
+ * request handler.
119
184
  */
120
185
  navigationTimeoutSecs?: number;
121
186
  /**
122
- * Defines whether the cookies should be persisted for sessions.
123
- * This can only be used when `useSessionPool` is set to `true`.
187
+ * Defines whether the cookies should be persisted for sessions. Enabled by default.
124
188
  */
125
- persistCookiesPerSession?: boolean;
126
- /**
127
- * Whether to run browser in headless mode. Defaults to `true`.
128
- * Can be also set via {@link Configuration}.
129
- */
130
- headless?: boolean | 'new' | 'old';
189
+ saveResponseCookies?: boolean;
131
190
  /**
132
191
  * Whether to ignore custom elements (and their #shadow-roots) when processing the page content via `parseWithCheerio` helper.
133
192
  * By default, they are expanded automatically. Use this option to disable this behavior.
@@ -151,173 +210,213 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
151
210
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
152
211
  * which downloads the pages using raw HTTP requests and is about 10x faster.
153
212
  *
154
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
155
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
156
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
213
+ * The source URLs are represented by the {@link Request} objects that are fed from the
214
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
215
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
157
216
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
158
217
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
159
218
  *
160
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
161
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
162
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
219
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
220
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
221
+ * and pass the result as `requestManager`.
222
+ *
223
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
224
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
163
225
  *
164
226
  * The crawler finishes when there are no more {@link Request} objects to crawl.
165
227
  *
166
228
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
167
229
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
168
230
  *
169
- * New pages are only opened when there is enough free CPU and memory available,
170
- * using the functionality provided by the {@link AutoscaledPool} class.
171
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
172
- * parameter of the `BrowserCrawler` constructor.
173
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
174
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
175
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
231
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
232
+ * {@link ConcurrencySystem}.
233
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
234
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
235
+ * {@link ConcurrencySystem|`concurrencySystem`}.
176
236
  *
177
237
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
178
238
  *
179
239
  * @category Crawlers
180
240
  */
181
- export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext = BrowserCrawlingContext, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context> {
182
- readonly config: Configuration;
241
+ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
242
+ #private;
183
243
  /**
184
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
185
- * Only available if used by the crawler.
244
+ * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
245
+ * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
186
246
  */
187
- proxyConfiguration?: ProxyConfiguration;
247
+ get browserPool(): IBrowserPool<Page>;
248
+ protected readonly ignoreShadowRoots: boolean;
249
+ protected readonly ignoreIframes: boolean;
188
250
  /**
189
- * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
251
+ * @internal
190
252
  */
191
- browserPool: BrowserPool<InternalBrowserPoolOptions>;
192
- launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
193
- protected userProvidedRequestHandler: BrowserRequestHandler<Context>;
194
- protected navigationTimeoutMillis: number;
195
- protected requestHandlerTimeoutInnerMillis: number;
196
- protected preNavigationHooks: BrowserHook<Context>[];
197
- protected postNavigationHooks: BrowserHook<Context>[];
198
- protected persistCookiesPerSession: boolean;
199
253
  protected static optionsShape: {
200
- // @ts-ignore optional peer dependency or compatibility with es2022
201
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
202
- // @ts-ignore optional peer dependency or compatibility with es2022
203
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
204
- // @ts-ignore optional peer dependency or compatibility with es2022
205
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
206
- // @ts-ignore optional peer dependency or compatibility with es2022
207
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
208
- // @ts-ignore optional peer dependency or compatibility with es2022
209
- headless: import("ow").AnyPredicate<string | boolean>;
210
- // @ts-ignore optional peer dependency or compatibility with es2022
211
- browserPoolOptions: import("ow").ObjectPredicate<object>;
212
- // @ts-ignore optional peer dependency or compatibility with es2022
213
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
214
- // @ts-ignore optional peer dependency or compatibility with es2022
215
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
216
- // @ts-ignore optional peer dependency or compatibility with es2022
217
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
218
- // @ts-ignore optional peer dependency or compatibility with es2022
219
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
220
- // @ts-ignore optional peer dependency or compatibility with es2022
221
- ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
222
- // @ts-ignore optional peer dependency or compatibility with es2022
223
- ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
224
- // @ts-ignore optional peer dependency or compatibility with es2022
225
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
226
- // @ts-ignore optional peer dependency or compatibility with es2022
227
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
228
- // @ts-ignore optional peer dependency or compatibility with es2022
229
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
230
- // @ts-ignore optional peer dependency or compatibility with es2022
231
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
232
- // @ts-ignore optional peer dependency or compatibility with es2022
233
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
234
- // @ts-ignore optional peer dependency or compatibility with es2022
235
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
236
- // @ts-ignore optional peer dependency or compatibility with es2022
237
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
238
- // @ts-ignore optional peer dependency or compatibility with es2022
239
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
240
- // @ts-ignore optional peer dependency or compatibility with es2022
241
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
242
- // @ts-ignore optional peer dependency or compatibility with es2022
243
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
244
- // @ts-ignore optional peer dependency or compatibility with es2022
245
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
246
- // @ts-ignore optional peer dependency or compatibility with es2022
247
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
248
- // @ts-ignore optional peer dependency or compatibility with es2022
249
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
250
- // @ts-ignore optional peer dependency or compatibility with es2022
251
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
252
- // @ts-ignore optional peer dependency or compatibility with es2022
253
- respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
254
- // @ts-ignore optional peer dependency or compatibility with es2022
255
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
258
- // @ts-ignore optional peer dependency or compatibility with es2022
259
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
260
- // @ts-ignore optional peer dependency or compatibility with es2022
261
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
262
- // @ts-ignore optional peer dependency or compatibility with es2022
263
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
266
- // @ts-ignore optional peer dependency or compatibility with es2022
267
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
268
- // @ts-ignore optional peer dependency or compatibility with es2022
269
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
270
- // @ts-ignore optional peer dependency or compatibility with es2022
271
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
254
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
255
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
256
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
257
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
258
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
259
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
260
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
261
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
262
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
263
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
264
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
265
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
266
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
267
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
268
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
269
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
270
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
271
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
272
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
273
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
274
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
275
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
276
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
277
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
278
+ requestQueue: z.ZodOptional<z.ZodEnum<{
279
+ deferred: "deferred";
280
+ writeThrough: "writeThrough";
281
+ }>>;
282
+ }, z.core.$strict>]>>;
283
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
284
+ // @ts-ignore optional peer dependency or compatibility with es2022
285
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
286
+ // @ts-ignore optional peer dependency or compatibility with es2022
287
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
288
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
289
+ // @ts-ignore optional peer dependency or compatibility with es2022
290
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
291
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
292
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
293
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
294
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
295
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
296
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
297
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
298
+ id: z.ZodOptional<z.ZodString>;
299
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
300
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
301
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
302
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
303
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
304
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
305
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
306
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
307
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
308
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
272
309
  };
310
+ /** @internal */
311
+ protected static optionsSchema: z.ZodObject<{
312
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
313
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
314
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
315
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
316
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
317
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
318
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
319
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
320
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
321
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
322
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
323
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
324
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
325
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
326
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
327
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
328
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
329
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
330
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
331
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
332
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
333
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
334
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
335
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
336
+ requestQueue: z.ZodOptional<z.ZodEnum<{
337
+ deferred: "deferred";
338
+ writeThrough: "writeThrough";
339
+ }>>;
340
+ }, z.core.$strict>]>>;
341
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
342
+ // @ts-ignore optional peer dependency or compatibility with es2022
343
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
344
+ // @ts-ignore optional peer dependency or compatibility with es2022
345
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
346
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
347
+ // @ts-ignore optional peer dependency or compatibility with es2022
348
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
349
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
350
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
351
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
352
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
353
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
354
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
355
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
356
+ id: z.ZodOptional<z.ZodString>;
357
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
358
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
359
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
360
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
361
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
362
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
363
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
364
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
365
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
366
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
367
+ }, z.core.$strict>;
273
368
  /**
274
369
  * All `BrowserCrawler` parameters are passed via an options object.
370
+ * @internal
275
371
  */
276
- protected constructor(options?: BrowserCrawlerOptions<Context>, config?: Configuration);
277
- protected _cleanupContext(crawlingContext: Context): Promise<void>;
372
+ protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
373
+ contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
374
+ /**
375
+ * Builds the pool the crawler owns, used only when the user injected no `browserPool`. Supplied by the
376
+ * concrete crawler, which is the only place that knows which browser plugin to run - that is also why
377
+ * `remoteBrowser` is handed to it rather than acted upon here.
378
+ */
379
+ browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
380
+ });
381
+ /** @internal */
382
+ protected getNavigationTimeoutMillis(): number;
383
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
278
384
  private containsSelectors;
279
- protected isRequestBlocked(crawlingContext: Context): Promise<string | false>;
385
+ private isRequestBlocked;
386
+ private preparePage;
387
+ private prepareNavigation;
388
+ private navigate;
389
+ private finalizeNavigation;
280
390
  /**
281
- * Wrapper around requestHandler that opens and closes pages etc.
391
+ * Copies cookies from the live browser page into the session cookie jar.
282
392
  */
283
- protected _runRequestHandler(crawlingContext: Context): Promise<void>;
284
- protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, createNewSession?: boolean): void;
285
- protected _handleNavigation(crawlingContext: Context): Promise<void>;
286
- protected _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string): Promise<void>;
393
+ private persistCookiesFromPage;
287
394
  /**
288
- * Marks session bad in case of navigation timeout.
395
+ * Runs the user request handler, then re-reads browser cookies so login flows /
396
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
397
+ * @internal
289
398
  */
290
- protected _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void>;
399
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
400
+ private handleBlockedRequestByContent;
401
+ private restoreRequestState;
402
+ private applyCookies;
291
403
  /**
292
- * Transforms proxy-related errors to `SessionError`.
404
+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
293
405
  */
294
- protected _throwIfProxyError(error: Error): void;
295
- protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
406
+ private handleNavigationTimeout;
296
407
  /**
297
- * Should be overridden in case of different automation library that does not support this response API.
408
+ * Transforms proxy-related errors to `SessionError`.
298
409
  */
299
- protected _responseHandler(crawlingContext: Context): Promise<void>;
300
- protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
301
- protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
410
+ private throwIfProxyError;
411
+ protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
412
+ private processResponse;
302
413
  /**
303
- * Function for cleaning up after all requests are processed.
304
- * @ignore
414
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
415
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
305
416
  */
306
417
  teardown(): Promise<void>;
418
+ destroy(): Promise<void>;
307
419
  }
308
- /** @internal */
309
- interface EnqueueLinksInternalOptions {
310
- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
311
- page: CommonPage;
312
- requestQueue: RequestProvider;
313
- robotsTxtFile?: RobotsTxtFile;
314
- onSkippedRequest?: SkippedRequestCallback;
315
- originalRequestUrl: string;
316
- finalRequestUrl?: string;
317
- }
318
- /** @internal */
319
- // @ts-ignore optional peer dependency or compatibility with es2022
320
- export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
321
420
  /**
322
421
  * Extracts URLs from a given page.
323
422
  * @ignore
@@ -326,4 +425,3 @@ export declare function extractUrlsFromPage(page: {
326
425
  $$eval: Function;
327
426
  }, selector: string, baseUrl: string): Promise<string[]>;
328
427
  export {};
329
- //# sourceMappingURL=browser-crawler.d.ts.map