@crawlee/puppeteer 4.0.0-beta.28 → 4.0.0-beta.281

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +4 -6
  3. package/index.js +2 -4
  4. package/internals/enqueue-links/click-elements.d.ts +30 -66
  5. package/internals/enqueue-links/click-elements.js +59 -70
  6. package/internals/puppeteer-browser-pool.d.ts +44 -0
  7. package/internals/puppeteer-browser-pool.js +34 -0
  8. package/internals/puppeteer-crawler.d.ts +163 -124
  9. package/internals/puppeteer-crawler.js +68 -72
  10. package/internals/puppeteer-launcher.d.ts +31 -19
  11. package/internals/puppeteer-launcher.js +20 -15
  12. package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -1
  13. package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +12 -9
  14. package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +29 -79
  15. package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +83 -182
  16. package/package.json +15 -19
  17. package/index.d.ts.map +0 -1
  18. package/index.js.map +0 -1
  19. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  20. package/internals/enqueue-links/click-elements.js.map +0 -1
  21. package/internals/puppeteer-crawler.d.ts.map +0 -1
  22. package/internals/puppeteer-crawler.js.map +0 -1
  23. package/internals/puppeteer-launcher.d.ts.map +0 -1
  24. package/internals/puppeteer-launcher.js.map +0 -1
  25. package/internals/utils/puppeteer_request_interception.d.ts.map +0 -1
  26. package/internals/utils/puppeteer_request_interception.js.map +0 -1
  27. package/internals/utils/puppeteer_utils.d.ts.map +0 -1
  28. package/internals/utils/puppeteer_utils.js.map +0 -1
@@ -1,45 +1,46 @@
1
- import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterRoutes } from '@crawlee/browser';
1
+ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
2
2
  import { BrowserCrawler } from '@crawlee/browser';
3
- import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';
4
3
  import type { Dictionary } from '@crawlee/types';
5
4
  // @ts-ignore optional peer dependency or compatibility with es2022
6
- import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
5
+ import type { HTTPResponse, Page } from 'puppeteer';
6
+ import { z } from 'zod';
7
7
  import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
8
- import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';
9
- export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {
8
+ import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
9
+ export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
10
+ export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
10
11
  }
11
- // @ts-ignore optional peer dependency or compatibility with es2022
12
- export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {
13
- }
14
- export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
15
- export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerController, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
16
- browserPlugins: [PuppeteerPlugin];
17
- }> {
12
+ export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
13
+ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
18
14
  /**
19
15
  * Options used by {@link launchPuppeteer} to start new Puppeteer instances.
20
16
  */
21
17
  launchContext?: PuppeteerLaunchContext;
18
+ /**
19
+ * Whether to run browser in headless mode. Defaults to `true`.
20
+ * Can be also set via {@link Configuration}.
21
+ */
22
+ headless?: boolean | 'new' | 'old';
22
23
  /**
23
24
  * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
24
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
25
- * which are passed to the `page.goto()` function the crawler calls to navigate.
25
+ * or browser properties before navigation. The function receives the `crawlingContext`; the options object
26
+ * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
27
+ * A hook may optionally return a partial object whose properties are merged into the crawling context
28
+ * (e.g. to override context members for subsequent hooks and pipeline stages).
26
29
  * Example:
27
30
  * ```
28
31
  * preNavigationHooks: [
29
- * async (crawlingContext, gotoOptions) => {
30
- * const { page } = crawlingContext;
32
+ * async ({ page, gotoOptions }) => {
31
33
  * await page.evaluate((attr) => { window.foo = attr; }, 'bar');
34
+ * gotoOptions.timeout = 60_000;
32
35
  * },
33
36
  * ]
34
37
  * ```
35
- *
36
- * Modyfing `pageOptions` is supported only in Playwright incognito.
37
- * See {@link PrePageCreateHook}
38
38
  */
39
- preNavigationHooks?: PuppeteerHook[];
39
+ preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
40
40
  /**
41
41
  * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
42
- * The function accepts `crawlingContext` as the only parameter.
42
+ * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
43
+ * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
43
44
  * Example:
44
45
  * ```
45
46
  * postNavigationHooks: [
@@ -52,7 +53,7 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
52
53
  * ]
53
54
  * ```
54
55
  */
55
- postNavigationHooks?: PuppeteerHook[];
56
+ postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
56
57
  }
57
58
  /**
58
59
  * Provides a simple framework for parallel crawling of web pages
@@ -65,24 +66,26 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
65
66
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
66
67
  * which downloads the pages using raw HTTP requests and is about 10x faster.
67
68
  *
68
- * The source URLs are represented using {@link Request} objects that are fed from
69
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
70
- * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
69
+ * The source URLs are represented using {@link Request} objects that are fed from the
70
+ * {@link IRequestManager|request manager} provided via the {@link PuppeteerCrawlerOptions.requestManager|`requestManager`}
71
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
72
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
73
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
74
+ * result as `requestManager`.
71
75
  *
72
- * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
73
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
74
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
76
+ * > The {@link PuppeteerCrawlerOptions.requestList|`requestList`} and {@link PuppeteerCrawlerOptions.requestQueue|`requestQueue`}
77
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
75
78
  *
76
79
  * The crawler finishes when there are no more {@link Request} objects to crawl.
77
80
  *
78
81
  * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
79
82
  * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.requestHandler} option.
80
83
  *
81
- * New pages are only opened when there is enough free CPU and memory available,
82
- * using the functionality provided by the {@link AutoscaledPool} class.
83
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
84
- * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
85
- * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
84
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
85
+ * {@link ConcurrencySystem}.
86
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
87
+ * `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured
88
+ * {@link ConcurrencySystem|`concurrencySystem`}.
86
89
  *
87
90
  * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
88
91
  *
@@ -117,99 +120,135 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
117
120
  * ```
118
121
  * @category Crawlers
119
122
  */
120
- export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, PuppeteerController, {
121
- browserPlugins: [PuppeteerPlugin];
122
- }, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext> {
123
+ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
124
+ #private;
125
+ /**
126
+ * @internal
127
+ */
123
128
  protected static optionsShape: {
124
- // @ts-ignore optional peer dependency or compatibility with es2022
125
- browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
126
- // @ts-ignore optional peer dependency or compatibility with es2022
127
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
128
- // @ts-ignore optional peer dependency or compatibility with es2022
129
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
130
- // @ts-ignore optional peer dependency or compatibility with es2022
131
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
132
- // @ts-ignore optional peer dependency or compatibility with es2022
133
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
134
- // @ts-ignore optional peer dependency or compatibility with es2022
135
- headless: import("ow").AnyPredicate<string | boolean>;
136
- // @ts-ignore optional peer dependency or compatibility with es2022
137
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
138
- // @ts-ignore optional peer dependency or compatibility with es2022
139
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
140
- // @ts-ignore optional peer dependency or compatibility with es2022
141
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
142
- // @ts-ignore optional peer dependency or compatibility with es2022
143
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
144
- // @ts-ignore optional peer dependency or compatibility with es2022
145
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
146
- // @ts-ignore optional peer dependency or compatibility with es2022
147
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
148
- // @ts-ignore optional peer dependency or compatibility with es2022
149
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
150
- // @ts-ignore optional peer dependency or compatibility with es2022
151
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
152
- // @ts-ignore optional peer dependency or compatibility with es2022
153
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
154
- // @ts-ignore optional peer dependency or compatibility with es2022
155
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
156
- // @ts-ignore optional peer dependency or compatibility with es2022
157
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
158
- // @ts-ignore optional peer dependency or compatibility with es2022
159
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
160
- // @ts-ignore optional peer dependency or compatibility with es2022
161
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
162
- // @ts-ignore optional peer dependency or compatibility with es2022
163
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
164
- // @ts-ignore optional peer dependency or compatibility with es2022
165
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
166
- // @ts-ignore optional peer dependency or compatibility with es2022
167
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
168
- // @ts-ignore optional peer dependency or compatibility with es2022
169
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
170
- // @ts-ignore optional peer dependency or compatibility with es2022
171
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
172
- // @ts-ignore optional peer dependency or compatibility with es2022
173
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
174
- // @ts-ignore optional peer dependency or compatibility with es2022
175
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
176
- // @ts-ignore optional peer dependency or compatibility with es2022
177
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
178
- // @ts-ignore optional peer dependency or compatibility with es2022
179
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
180
- // @ts-ignore optional peer dependency or compatibility with es2022
181
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
182
- // @ts-ignore optional peer dependency or compatibility with es2022
183
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
184
- // @ts-ignore optional peer dependency or compatibility with es2022
185
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
186
- // @ts-ignore optional peer dependency or compatibility with es2022
187
- storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
188
- // @ts-ignore optional peer dependency or compatibility with es2022
189
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
190
- // @ts-ignore optional peer dependency or compatibility with es2022
191
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
192
- // @ts-ignore optional peer dependency or compatibility with es2022
193
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
194
- // @ts-ignore optional peer dependency or compatibility with es2022
195
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
196
- // @ts-ignore optional peer dependency or compatibility with es2022
197
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
198
- // @ts-ignore optional peer dependency or compatibility with es2022
199
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
200
- // @ts-ignore optional peer dependency or compatibility with es2022
201
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
202
- // @ts-ignore optional peer dependency or compatibility with es2022
203
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
204
- // @ts-ignore optional peer dependency or compatibility with es2022
205
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
129
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
130
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
131
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
132
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
133
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
134
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
135
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
136
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
137
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
138
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
139
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
140
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
141
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
142
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
143
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
144
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
145
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
146
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
147
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
148
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
149
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
150
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
151
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
152
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
153
+ requestQueue: z.ZodOptional<z.ZodEnum<{
154
+ deferred: "deferred";
155
+ writeThrough: "writeThrough";
156
+ }>>;
157
+ }, z.core.$strict>]>>;
158
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
159
+ // @ts-ignore optional peer dependency or compatibility with es2022
160
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
161
+ // @ts-ignore optional peer dependency or compatibility with es2022
162
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
163
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
164
+ // @ts-ignore optional peer dependency or compatibility with es2022
165
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
166
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
167
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
168
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
169
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
170
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
171
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
172
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
173
+ id: z.ZodOptional<z.ZodString>;
174
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
175
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
176
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
177
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
178
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
179
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
180
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
181
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
182
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
183
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
184
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
185
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
206
186
  };
187
+ /** @internal */
188
+ protected static optionsSchema: z.ZodObject<{
189
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
190
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
191
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
192
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
193
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
194
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
195
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
196
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
197
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
198
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
199
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
200
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
201
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
202
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
203
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
204
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
205
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
206
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
207
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
208
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
209
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
210
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
211
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
212
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
213
+ requestQueue: z.ZodOptional<z.ZodEnum<{
214
+ deferred: "deferred";
215
+ writeThrough: "writeThrough";
216
+ }>>;
217
+ }, z.core.$strict>]>>;
218
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
219
+ // @ts-ignore optional peer dependency or compatibility with es2022
220
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
221
+ // @ts-ignore optional peer dependency or compatibility with es2022
222
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
223
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
224
+ // @ts-ignore optional peer dependency or compatibility with es2022
225
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
226
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
227
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
228
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
229
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
230
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
231
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
232
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
233
+ id: z.ZodOptional<z.ZodString>;
234
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
235
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
236
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
237
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
238
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
239
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
240
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
241
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
242
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
243
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
244
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
245
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
246
+ }, z.core.$strict>;
207
247
  /**
208
248
  * All `PuppeteerCrawler` parameters are passed via an options object.
209
249
  */
210
- constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
211
- private enhanceContext;
212
- protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
250
+ constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
251
+ protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
213
252
  }
214
253
  /**
215
254
  * Creates new {@link Router} instance that works based on request labels.
@@ -235,6 +274,6 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
235
274
  * await crawler.run();
236
275
  * ```
237
276
  */
238
- // @ts-ignore optional peer dependency or compatibility with es2022
239
- export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
240
- //# sourceMappingURL=puppeteer-crawler.d.ts.map
277
+ export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
278
+ export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
279
+ export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
@@ -1,8 +1,9 @@
1
1
  import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
2
2
  import { serviceLocator } from '@crawlee/core';
3
- import ow from 'ow';
4
- import { PuppeteerLauncher } from './puppeteer-launcher.js';
5
- import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
3
+ import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
5
+ import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
6
+ import * as puppeteerUtils from './utils/puppeteer-utils.js';
6
7
  /**
7
8
  * Provides a simple framework for parallel crawling of web pages
8
9
  * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
@@ -14,24 +15,26 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
14
15
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
15
16
  * which downloads the pages using raw HTTP requests and is about 10x faster.
16
17
  *
17
- * The source URLs are represented using {@link Request} objects that are fed from
18
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
19
- * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
18
+ * The source URLs are represented using {@link Request} objects that are fed from the
19
+ * {@link IRequestManager|request manager} provided via the {@link PuppeteerCrawlerOptions.requestManager|`requestManager`}
20
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
21
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
22
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
23
+ * result as `requestManager`.
20
24
  *
21
- * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
22
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
23
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
25
+ * > The {@link PuppeteerCrawlerOptions.requestList|`requestList`} and {@link PuppeteerCrawlerOptions.requestQueue|`requestQueue`}
26
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
24
27
  *
25
28
  * The crawler finishes when there are no more {@link Request} objects to crawl.
26
29
  *
27
30
  * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
28
31
  * and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.requestHandler} option.
29
32
  *
30
- * New pages are only opened when there is enough free CPU and memory available,
31
- * using the functionality provided by the {@link AutoscaledPool} class.
32
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PuppeteerCrawlerOptions.autoscaledPoolOptions}
33
- * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
34
- * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
33
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
34
+ * {@link ConcurrencySystem}.
35
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
36
+ * `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured
37
+ * {@link ConcurrencySystem|`concurrencySystem`}.
35
38
  *
36
39
  * Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
37
40
  *
@@ -67,46 +70,65 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
67
70
  * @category Crawlers
68
71
  */
69
72
  export class PuppeteerCrawler extends BrowserCrawler {
73
+ /**
74
+ * @internal
75
+ */
70
76
  static optionsShape = {
71
77
  ...BrowserCrawler.optionsShape,
72
- browserPoolOptions: ow.optional.object,
78
+ launchContext: schemas.anyObject.default(() => ({})),
79
+ // Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
80
+ // time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
81
+ headless: z.union([z.boolean(), z.string()]).optional(),
73
82
  };
83
+ /** @internal */
84
+ static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
74
85
  /**
75
86
  * All `PuppeteerCrawler` parameters are passed via an options object.
76
87
  */
77
88
  constructor(options = {}) {
78
- ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));
79
- const { launchContext = {}, headless, proxyConfiguration, ...browserCrawlerOptions } = options;
80
- const browserPoolOptions = {
81
- ...options.browserPoolOptions,
82
- };
89
+ const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
90
+ const { launchContext, headless, configuration, proxyConfiguration, ...browserCrawlerOptions } = parsedOptions;
83
91
  if (launchContext.proxyUrl) {
84
92
  throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
85
93
  'Use PuppeteerCrawlerOptions.proxyConfiguration');
86
94
  }
87
- // `browserPlugins` is working when it's not overridden by `launchContext`,
88
- // which for crawlers it is always overridden. Hence the error to use the other option.
89
- if (browserPoolOptions.browserPlugins) {
90
- throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
95
+ if (options.browserPool) {
96
+ // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
97
+ assertBrowserPoolNotConfigured(new.target.name, {
98
+ launchContext: options.launchContext,
99
+ headless: options.headless,
100
+ });
91
101
  }
92
- if (headless != null) {
93
- launchContext.launchOptions ??= {};
94
- launchContext.launchOptions.headless = headless;
95
- }
96
- const puppeteerLauncher = new PuppeteerLauncher(launchContext, options.configuration);
97
- browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];
98
102
  super({
99
103
  ...browserCrawlerOptions,
100
- launchContext,
104
+ configuration,
101
105
  proxyConfiguration,
102
- browserPoolOptions,
103
- contextPipelineBuilder: () => this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),
106
+ browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
107
+ contextPipelineBuilder: () => this.#buildContextPipeline(),
104
108
  });
105
109
  }
106
- async enhanceContext(context) {
110
+ #buildContextPipeline() {
111
+ return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
112
+ }
113
+ async #enhanceContext(context) {
107
114
  const waitForSelector = async (selector, timeoutMs = 5_000) => {
108
115
  await context.page.waitForSelector(selector, { timeout: timeoutMs });
109
116
  };
117
+ const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
118
+ if (selector) {
119
+ await waitForSelector(selector, timeoutMs);
120
+ }
121
+ return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
122
+ };
123
+ const querySelectorAll = async (selector, timeoutMs) => {
124
+ const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
125
+ // The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
126
+ // or an element under a shadow root/iframe the snapshot was told to ignore.
127
+ if (matches.length === 0) {
128
+ throw new Error(`Selector '${selector}' not found in the page snapshot.`);
129
+ }
130
+ return matches;
131
+ };
110
132
  return {
111
133
  injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
112
134
  injectJQuery: async () => {
@@ -118,15 +140,12 @@ export class PuppeteerCrawler extends BrowserCrawler {
118
140
  await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
119
141
  },
120
142
  waitForSelector,
121
- parseWithCheerio: async (selector, timeoutMs = 5_000) => {
122
- if (selector) {
123
- await waitForSelector(selector, timeoutMs);
124
- }
125
- return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
126
- },
143
+ parseWithCheerio,
144
+ querySelectorAll,
145
+ querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
127
146
  enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
128
147
  page: context.page,
129
- requestQueue: this.requestQueue,
148
+ requestManager: this.requestManager,
130
149
  ...options,
131
150
  }),
132
151
  blockRequests: async (options) => puppeteerUtils.blockRequests(context.page, options),
@@ -134,39 +153,16 @@ export class PuppeteerCrawler extends BrowserCrawler {
134
153
  addInterceptRequestHandler: async (handler) => puppeteerUtils.addInterceptRequestHandler(context.page, handler),
135
154
  removeInterceptRequestHandler: async (handler) => puppeteerUtils.removeInterceptRequestHandler(context.page, handler),
136
155
  infiniteScroll: async (options) => puppeteerUtils.infiniteScroll(context.page, options),
137
- saveSnapshot: async (options) => puppeteerUtils.saveSnapshot(context.page, { ...options, config: serviceLocator.getConfiguration() }),
138
- closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),
156
+ saveSnapshot: async (options) => puppeteerUtils.saveSnapshot(context.page, {
157
+ ...options,
158
+ configuration: serviceLocator.getConfiguration(),
159
+ }),
139
160
  };
140
161
  }
141
- async _navigationHandler(crawlingContext, gotoOptions) {
142
- return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
162
+ async navigationHandler(crawlingContext, gotoOptions) {
163
+ return puppeteerUtils.gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
143
164
  }
144
165
  }
145
- /**
146
- * Creates new {@link Router} instance that works based on request labels.
147
- * This instance can then serve as a `requestHandler` of your {@link PuppeteerCrawler}.
148
- * Defaults to the {@link PuppeteerCrawlingContext}.
149
- *
150
- * > Serves as a shortcut for using `Router.create<PuppeteerCrawlingContext>()`.
151
- *
152
- * ```ts
153
- * import { PuppeteerCrawler, createPuppeteerRouter } from 'crawlee';
154
- *
155
- * const router = createPuppeteerRouter();
156
- * router.addHandler('label-a', async (ctx) => {
157
- * ctx.log.info('...');
158
- * });
159
- * router.addDefaultHandler(async (ctx) => {
160
- * ctx.log.info('...');
161
- * });
162
- *
163
- * const crawler = new PuppeteerCrawler({
164
- * requestHandler: router,
165
- * });
166
- * await crawler.run();
167
- * ```
168
- */
169
- export function createPuppeteerRouter(routes) {
170
- return Router.create(routes);
166
+ export function createPuppeteerRouter(routesOrSchemas) {
167
+ return Router.create(routesOrSchemas);
171
168
  }
172
- //# sourceMappingURL=puppeteer-crawler.js.map