@crawlee/browser 4.0.0-beta.19 → 4.0.0-beta.191

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,32 @@
1
- import type { Awaitable, BasicCrawlerOptions, BasicCrawlingContext, CrawlingContext, Dictionary, EnqueueLinksOptions, ErrorHandler, LoadedRequest, Request, RequestHandler, RequestProvider, SkippedRequestCallback } from '@crawlee/basic';
2
- import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
3
- import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
4
- import { BrowserPool } from '@crawlee/browser-pool';
5
- import type { BatchAddRequestsResult } from '@crawlee/types';
6
- import type { RobotsTxtFile } from '@crawlee/utils';
7
- import type { ReadonlyDeep } from 'type-fest';
1
+ import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
2
+ import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
+ import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
4
+ import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
+ import { z } from 'zod';
8
6
  import type { BrowserLaunchContext } from './browser-launcher.js';
9
7
  interface BaseResponse {
10
8
  status(): number;
9
+ /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
10
+ headers?(): Record<string, string>;
11
11
  }
12
- export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
13
- /**
14
- * An instance of the {@link BrowserController} that manages the browser instance and provides access to its API.
15
- */
16
- browserController: ProvidedController;
12
+ /**
13
+ * The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
14
+ * additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
15
+ * {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
16
+ * `destroy()` once the crawler itself is destroyed.
17
+ */
18
+ export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
19
+ releaseAllBrowsers: () => Promise<void>;
20
+ destroy: () => Promise<void>;
21
+ };
22
+ /**
23
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
24
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
25
+ * into a second, half-working way of configuring the same pool.
26
+ */
27
+ export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
28
+ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
29
+ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
17
30
  /**
18
31
  * The browser page object where the web page is loaded and rendered.
19
32
  */
@@ -21,19 +34,55 @@ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Re
21
34
  /**
22
35
  * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
23
36
  */
24
- request: LoadedRequest<Request<UserData>>;
37
+ request: LoadedRequest<CrawlingRequest<UserData>>;
25
38
  /**
26
39
  * The HTTP response object returned by the browser's navigation.
27
40
  */
28
41
  response: Response;
42
+ /**
43
+ * Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
44
+ * object (or return `{ gotoOptions: ... }`) to influence the navigation.
45
+ */
46
+ gotoOptions: GoToOptions;
47
+ /**
48
+ * Extracts URLs from the current page, without adding them to the request queue.
49
+ */
50
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
29
51
  /**
30
52
  * Helper function for extracting URLs from the current page and adding them to the request queue.
31
53
  */
32
- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
54
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
33
55
  }
34
- export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
35
- export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController extends BrowserController = BrowserController, Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
56
+ export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
57
+ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
36
58
  launchContext?: BrowserLaunchContext<any, any>;
59
+ /**
60
+ * The browser pool the crawler should serve its pages from. This is the single way to run a pool with
61
+ * non-default options: build one with the factory that matches your crawler
62
+ * ({@link playwrightBrowserPool}, {@link puppeteerBrowserPool}, {@link stagehandBrowserPool}) - it
63
+ * accepts every {@link BrowserPoolOptions|`BrowserPool` option} and supplies the correct browser plugin
64
+ * itself, so the pool can never mismatch the crawler.
65
+ *
66
+ * A pool passed in this way is borrowed, not owned: the crawler will not tear it down, which is what makes it
67
+ * shareable across crawlers. Since the crawler then builds nothing itself, the options that configure its own
68
+ * pool (`launchContext`, `headless`, `remoteBrowser`) are rejected rather than silently ignored.
69
+ *
70
+ * When omitted, the crawler builds - and tears down - a default pool for its own browser.
71
+ */
72
+ browserPool?: IBrowserPool<Page>;
73
+ /**
74
+ * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
75
+ *
76
+ * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
77
+ * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
78
+ * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
79
+ * or a {@link RemoteBrowserProvider}.
80
+ *
81
+ * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it
82
+ * across crawlers, build it with the remote factory for your crawler ({@link remotePlaywrightBrowserPool},
83
+ * {@link remotePuppeteerBrowserPool}, {@link remoteStagehandBrowserPool}) and pass it as `browserPool`.
84
+ */
85
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
37
86
  /**
38
87
  * Function that is called to process each request.
39
88
  *
@@ -44,7 +93,6 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
44
93
  * - {@link BrowserCrawlingContext.page|`page`} is an instance of the
45
94
  * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
46
95
  * Playwright [Page](https://playwright.dev/docs/api/class-page);
47
- * - {@link BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@link BrowserController};
48
96
  * - {@link BrowserCrawlingContext.response|`response`} is an instance of the
49
97
  * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
50
98
  * Playwright [Response](https://playwright.dev/docs/api/class-response),
@@ -59,9 +107,9 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
59
107
  * To make this work, we should **always**
60
108
  * let our function throw exceptions rather than catch them.
61
109
  * The exceptions are logged to the request using the
62
- * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
110
+ * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
63
111
  */
64
- requestHandler?: RequestHandler<ExtendedContext>;
112
+ requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
65
113
  /**
66
114
  * User-provided function that allows modifying the request object before it gets retried by the crawler.
67
115
  * It's executed before each retry for the requests that failed less than {@link BrowserCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
@@ -83,22 +131,16 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
83
131
  * represents the last error thrown during processing of the request.
84
132
  */
85
133
  failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
86
- /**
87
- * Custom options passed to the underlying {@link BrowserPool} constructor.
88
- * We can tweak those to fine-tune browser management.
89
- */
90
- browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
91
134
  /**
92
135
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
93
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
94
- * which are passed to the `page.goto()` function the crawler calls to navigate.
136
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
137
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
95
138
  *
96
139
  * **Example:**
97
140
  *
98
141
  * ```js
99
142
  * preNavigationHooks: [
100
- * async (crawlingContext, gotoOptions) => {
101
- * const { page } = crawlingContext;
143
+ * async ({ page, gotoOptions }) => {
102
144
  * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
103
145
  * gotoOptions.timeout = 60_000;
104
146
  * gotoOptions.waitUntil = 'domcontentloaded';
@@ -106,14 +148,22 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
106
148
  * ]
107
149
  * ```
108
150
  *
109
- * Modyfing `pageOptions` is supported only in Playwright incognito.
110
- * See {@link PrePageCreateHook}
151
+ * A hook may optionally return a partial object whose properties are merged into the crawling context,
152
+ * allowing the hook to override context members for subsequent hooks and pipeline stages.
153
+ *
154
+ * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
155
+ * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
156
+ * This means the members added by `extendContext` are already available here, but navigation-dependent
157
+ * members (e.g. `page`, `response`) are not.
111
158
  */
112
- preNavigationHooks?: BrowserHook<Context>[];
159
+ preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
113
160
  /**
114
161
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
115
162
  * The function accepts `crawlingContext` as the only parameter.
116
163
  *
164
+ * A hook may optionally return a partial object whose properties are merged into the crawling context.
165
+ * This is useful for overriding context members (e.g. `response`) after solving a challenge.
166
+ *
117
167
  * **Example:**
118
168
  *
119
169
  * ```js
@@ -124,24 +174,27 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
124
174
  * await solveCaptcha(page);
125
175
  * }
126
176
  * },
177
+ * async (crawlingContext) => {
178
+ * if (await needsRevalidation(crawlingContext)) {
179
+ * return { response: await crawlingContext.page.reload() };
180
+ * }
181
+ * },
127
182
  * ]
128
183
  * ```
129
184
  */
130
- postNavigationHooks?: BrowserHook<Context>[];
185
+ postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
131
186
  /**
132
- * Timeout in which page navigation needs to finish, in seconds.
187
+ * Timeout for the whole navigation phase, in seconds. A single window shared by the `preNavigationHooks`,
188
+ * the page navigation, and the `postNavigationHooks` - so a slow hook eats into the same budget the
189
+ * navigation uses. Separate from the
190
+ * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
191
+ * request handler.
133
192
  */
134
193
  navigationTimeoutSecs?: number;
135
194
  /**
136
- * Defines whether the cookies should be persisted for sessions.
137
- * This can only be used when `useSessionPool` is set to `true`.
138
- */
139
- persistCookiesPerSession?: boolean;
140
- /**
141
- * Whether to run browser in headless mode. Defaults to `true`.
142
- * Can be also set via {@link Configuration}.
195
+ * Defines whether the cookies should be persisted for sessions. Enabled by default.
143
196
  */
144
- headless?: boolean | 'new' | 'old';
197
+ saveResponseCookies?: boolean;
145
198
  /**
146
199
  * Whether to ignore custom elements (and their #shadow-roots) when processing the page content via `parseWithCheerio` helper.
147
200
  * By default, they are expanded automatically. Use this option to disable this behavior.
@@ -165,174 +218,213 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
165
218
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
166
219
  * which downloads the pages using raw HTTP requests and is about 10x faster.
167
220
  *
168
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
169
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
170
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
221
+ * The source URLs are represented by the {@link Request} objects that are fed from the
222
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
223
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
171
224
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
172
225
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
173
226
  *
174
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
175
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
176
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
227
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
228
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
229
+ * and pass the result as `requestManager`.
230
+ *
231
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
232
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
177
233
  *
178
234
  * The crawler finishes when there are no more {@link Request} objects to crawl.
179
235
  *
180
236
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
181
237
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
182
238
  *
183
- * New pages are only opened when there is enough free CPU and memory available,
184
- * using the functionality provided by the {@link AutoscaledPool} class.
185
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
186
- * parameter of the `BrowserCrawler` constructor.
187
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
188
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
189
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
239
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
240
+ * {@link ConcurrencySystem}.
241
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
242
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
243
+ * {@link ConcurrencySystem|`concurrencySystem`}.
190
244
  *
191
245
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
192
246
  *
193
247
  * @category Crawlers
194
248
  */
195
- export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController extends BrowserController = BrowserController, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
196
- readonly config: Configuration;
249
+ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
250
+ #private;
197
251
  /**
198
- * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
252
+ * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
253
+ * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
199
254
  */
200
- browserPool: BrowserPool<InternalBrowserPoolOptions>;
255
+ get browserPool(): IBrowserPool<Page>;
201
256
  launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
202
257
  protected readonly ignoreShadowRoots: boolean;
203
258
  protected readonly ignoreIframes: boolean;
204
- protected navigationTimeoutMillis: number;
205
- protected preNavigationHooks: BrowserHook<Context>[];
206
- protected postNavigationHooks: BrowserHook<Context>[];
207
- protected persistCookiesPerSession: boolean;
259
+ /**
260
+ * @internal
261
+ */
208
262
  protected static optionsShape: {
209
- // @ts-ignore optional peer dependency or compatibility with es2022
210
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
211
- // @ts-ignore optional peer dependency or compatibility with es2022
212
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
213
- // @ts-ignore optional peer dependency or compatibility with es2022
214
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
215
- // @ts-ignore optional peer dependency or compatibility with es2022
216
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
217
- // @ts-ignore optional peer dependency or compatibility with es2022
218
- headless: import("ow").AnyPredicate<string | boolean>;
219
- // @ts-ignore optional peer dependency or compatibility with es2022
220
- browserPoolOptions: import("ow").ObjectPredicate<object>;
221
- // @ts-ignore optional peer dependency or compatibility with es2022
222
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
223
- // @ts-ignore optional peer dependency or compatibility with es2022
224
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
225
- // @ts-ignore optional peer dependency or compatibility with es2022
226
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
227
- // @ts-ignore optional peer dependency or compatibility with es2022
228
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
229
- // @ts-ignore optional peer dependency or compatibility with es2022
230
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
231
- // @ts-ignore optional peer dependency or compatibility with es2022
232
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
233
- // @ts-ignore optional peer dependency or compatibility with es2022
234
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
235
- // @ts-ignore optional peer dependency or compatibility with es2022
236
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
237
- // @ts-ignore optional peer dependency or compatibility with es2022
238
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
239
- // @ts-ignore optional peer dependency or compatibility with es2022
240
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
241
- // @ts-ignore optional peer dependency or compatibility with es2022
242
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
243
- // @ts-ignore optional peer dependency or compatibility with es2022
244
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
245
- // @ts-ignore optional peer dependency or compatibility with es2022
246
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
247
- // @ts-ignore optional peer dependency or compatibility with es2022
248
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
249
- // @ts-ignore optional peer dependency or compatibility with es2022
250
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
251
- // @ts-ignore optional peer dependency or compatibility with es2022
252
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
253
- // @ts-ignore optional peer dependency or compatibility with es2022
254
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
255
- // @ts-ignore optional peer dependency or compatibility with es2022
256
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
257
- // @ts-ignore optional peer dependency or compatibility with es2022
258
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
259
- // @ts-ignore optional peer dependency or compatibility with es2022
260
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
261
- // @ts-ignore optional peer dependency or compatibility with es2022
262
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
263
- // @ts-ignore optional peer dependency or compatibility with es2022
264
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
265
- // @ts-ignore optional peer dependency or compatibility with es2022
266
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
267
- // @ts-ignore optional peer dependency or compatibility with es2022
268
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
269
- // @ts-ignore optional peer dependency or compatibility with es2022
270
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
271
- // @ts-ignore optional peer dependency or compatibility with es2022
272
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
273
- // @ts-ignore optional peer dependency or compatibility with es2022
274
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
275
- // @ts-ignore optional peer dependency or compatibility with es2022
276
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
277
- // @ts-ignore optional peer dependency or compatibility with es2022
278
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
279
- // @ts-ignore optional peer dependency or compatibility with es2022
280
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
281
- // @ts-ignore optional peer dependency or compatibility with es2022
282
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
263
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
264
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
265
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
266
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
267
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
268
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
269
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
270
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
271
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
272
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
273
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
274
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
275
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
276
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
277
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
278
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
279
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
280
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
281
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
282
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
283
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
284
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
285
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
286
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
287
+ requestQueue: z.ZodOptional<z.ZodEnum<{
288
+ deferred: "deferred";
289
+ writeThrough: "writeThrough";
290
+ }>>;
291
+ }, z.core.$strict>]>>;
292
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
293
+ // @ts-ignore optional peer dependency or compatibility with es2022
294
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
295
+ // @ts-ignore optional peer dependency or compatibility with es2022
296
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
297
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
298
+ // @ts-ignore optional peer dependency or compatibility with es2022
299
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
300
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
301
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
302
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
303
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
304
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
305
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
306
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
307
+ id: z.ZodOptional<z.ZodString>;
308
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
309
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
310
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
311
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
312
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
313
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
314
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
315
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
316
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
317
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
318
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
283
319
  };
320
+ /** @internal */
321
+ protected static optionsSchema: z.ZodObject<{
322
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
323
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
324
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
325
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
326
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
327
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
328
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
329
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
330
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
331
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
332
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
333
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
334
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
335
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
336
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
337
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
338
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
339
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
340
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
341
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
342
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
343
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
344
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
345
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
346
+ requestQueue: z.ZodOptional<z.ZodEnum<{
347
+ deferred: "deferred";
348
+ writeThrough: "writeThrough";
349
+ }>>;
350
+ }, z.core.$strict>]>>;
351
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
352
+ // @ts-ignore optional peer dependency or compatibility with es2022
353
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
354
+ // @ts-ignore optional peer dependency or compatibility with es2022
355
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
356
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
357
+ // @ts-ignore optional peer dependency or compatibility with es2022
358
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
359
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
360
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
361
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
362
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
363
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
364
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
365
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
366
+ id: z.ZodOptional<z.ZodString>;
367
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
368
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
369
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
370
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
371
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
372
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
373
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
374
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
375
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
376
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
377
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
378
+ }, z.core.$strict>;
284
379
  /**
285
380
  * All `BrowserCrawler` parameters are passed via an options object.
286
381
  */
287
- protected constructor(options: BrowserCrawlerOptions<Page, Response, ProvidedController, Context, ContextExtension, ExtendedContext> & {
382
+ protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
288
383
  contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
289
- }, config?: Configuration);
290
- protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>>;
384
+ /**
385
+ * Builds the pool the crawler owns, used only when the user injected no `browserPool`. Supplied by the
386
+ * concrete crawler, which is the only place that knows which browser plugin to run - that is also why
387
+ * `remoteBrowser` is handed to it rather than acted upon here.
388
+ */
389
+ browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
390
+ });
391
+ protected getNavigationTimeoutMillis(): number;
392
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
291
393
  private containsSelectors;
292
- protected isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>): Promise<string | false>;
394
+ private isRequestBlocked;
293
395
  private preparePage;
294
- private performNavigation;
396
+ private prepareNavigation;
397
+ private navigate;
398
+ private finalizeNavigation;
399
+ /**
400
+ * Copies cookies from the live browser page into the session cookie jar.
401
+ */
402
+ private persistCookiesFromPage;
403
+ /**
404
+ * Runs the user request handler, then re-reads browser cookies so login flows /
405
+ * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
406
+ */
407
+ protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
295
408
  private handleBlockedRequestByContent;
296
409
  private restoreRequestState;
297
- protected _applyCookies({ session, request, page, browserController }: BrowserCrawlingContext, preHooksCookies: string, postHooksCookies: string): Promise<void>;
410
+ private applyCookies;
298
411
  /**
299
- * Marks session bad in case of navigation timeout.
412
+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
300
413
  */
301
- protected _handleNavigationTimeout(crawlingContext: BrowserCrawlingContext, error: Error): Promise<void>;
414
+ private handleNavigationTimeout;
302
415
  /**
303
416
  * Transforms proxy-related errors to `SessionError`.
304
417
  */
305
- protected _throwIfProxyError(error: Error): void;
306
- protected abstract _navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
418
+ private throwIfProxyError;
419
+ protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
307
420
  private processResponse;
308
- protected _extendLaunchContext(_pageId: string, launchContext: LaunchContext): Promise<void>;
309
- protected _maybeAddSessionRetiredListener(_pageId: string, browserController: Context['browserController']): void;
310
421
  /**
311
- * Function for cleaning up after all requests are processed.
312
- * @ignore
422
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
423
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
313
424
  */
314
425
  teardown(): Promise<void>;
426
+ destroy(): Promise<void>;
315
427
  }
316
- /** @internal */
317
- interface EnqueueLinksInternalOptions {
318
- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
319
- page: CommonPage;
320
- requestQueue: RequestProvider;
321
- robotsTxtFile?: RobotsTxtFile;
322
- onSkippedRequest?: SkippedRequestCallback;
323
- originalRequestUrl: string;
324
- finalRequestUrl?: string;
325
- }
326
- /** @internal */
327
- interface BoundEnqueueLinksInternalOptions {
328
- enqueueLinks: BasicCrawlingContext['enqueueLinks'];
329
- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>;
330
- originalRequestUrl: string;
331
- finalRequestUrl?: string;
332
- page: CommonPage;
333
- }
334
- /** @internal */
335
- export declare function browserCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
336
428
  /**
337
429
  * Extracts URLs from a given page.
338
430
  * @ignore
@@ -341,4 +433,3 @@ export declare function extractUrlsFromPage(page: {
341
433
  $$eval: Function;
342
434
  }, selector: string, baseUrl: string): Promise<string[]>;
343
435
  export {};
344
- //# sourceMappingURL=browser-crawler.d.ts.map