@crawlee/puppeteer 4.0.0-beta.28 → 4.0.0-beta.281
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +4 -6
- package/index.js +2 -4
- package/internals/enqueue-links/click-elements.d.ts +30 -66
- package/internals/enqueue-links/click-elements.js +59 -70
- package/internals/puppeteer-browser-pool.d.ts +44 -0
- package/internals/puppeteer-browser-pool.js +34 -0
- package/internals/puppeteer-crawler.d.ts +163 -124
- package/internals/puppeteer-crawler.js +68 -72
- package/internals/puppeteer-launcher.d.ts +31 -19
- package/internals/puppeteer-launcher.js +20 -15
- package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -1
- package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +12 -9
- package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +29 -79
- package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +83 -182
- package/package.json +15 -19
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/enqueue-links/click-elements.d.ts.map +0 -1
- package/internals/enqueue-links/click-elements.js.map +0 -1
- package/internals/puppeteer-crawler.d.ts.map +0 -1
- package/internals/puppeteer-crawler.js.map +0 -1
- package/internals/puppeteer-launcher.d.ts.map +0 -1
- package/internals/puppeteer-launcher.js.map +0 -1
- package/internals/utils/puppeteer_request_interception.d.ts.map +0 -1
- package/internals/utils/puppeteer_request_interception.js.map +0 -1
- package/internals/utils/puppeteer_utils.d.ts.map +0 -1
- package/internals/utils/puppeteer_utils.js.map +0 -1
|
@@ -1,45 +1,46 @@
|
|
|
1
|
-
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterRoutes } from '@crawlee/browser';
|
|
1
|
+
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
2
2
|
import { BrowserCrawler } from '@crawlee/browser';
|
|
3
|
-
import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
4
3
|
import type { Dictionary } from '@crawlee/types';
|
|
5
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
|
-
import type { HTTPResponse,
|
|
5
|
+
import type { HTTPResponse, Page } from 'puppeteer';
|
|
6
|
+
import { z } from 'zod';
|
|
7
7
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
8
|
-
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/
|
|
9
|
-
export
|
|
8
|
+
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
|
|
9
|
+
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
10
|
+
export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
|
10
11
|
}
|
|
11
|
-
|
|
12
|
-
export interface
|
|
13
|
-
}
|
|
14
|
-
export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
|
|
15
|
-
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerController, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
|
|
16
|
-
browserPlugins: [PuppeteerPlugin];
|
|
17
|
-
}> {
|
|
12
|
+
export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
|
|
13
|
+
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
18
14
|
/**
|
|
19
15
|
* Options used by {@link launchPuppeteer} to start new Puppeteer instances.
|
|
20
16
|
*/
|
|
21
17
|
launchContext?: PuppeteerLaunchContext;
|
|
18
|
+
/**
|
|
19
|
+
* Whether to run browser in headless mode. Defaults to `true`.
|
|
20
|
+
* Can be also set via {@link Configuration}.
|
|
21
|
+
*/
|
|
22
|
+
headless?: boolean | 'new' | 'old';
|
|
22
23
|
/**
|
|
23
24
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
24
|
-
* or browser properties before navigation. The function
|
|
25
|
-
*
|
|
25
|
+
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
26
|
+
* forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
|
|
27
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context
|
|
28
|
+
* (e.g. to override context members for subsequent hooks and pipeline stages).
|
|
26
29
|
* Example:
|
|
27
30
|
* ```
|
|
28
31
|
* preNavigationHooks: [
|
|
29
|
-
* async (
|
|
30
|
-
* const { page } = crawlingContext;
|
|
32
|
+
* async ({ page, gotoOptions }) => {
|
|
31
33
|
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
34
|
+
* gotoOptions.timeout = 60_000;
|
|
32
35
|
* },
|
|
33
36
|
* ]
|
|
34
37
|
* ```
|
|
35
|
-
*
|
|
36
|
-
* Modyfing `pageOptions` is supported only in Playwright incognito.
|
|
37
|
-
* See {@link PrePageCreateHook}
|
|
38
38
|
*/
|
|
39
|
-
preNavigationHooks?:
|
|
39
|
+
preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
40
40
|
/**
|
|
41
41
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
42
|
-
* The function accepts `crawlingContext` as the only parameter.
|
|
42
|
+
* The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
|
|
43
|
+
* whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
|
|
43
44
|
* Example:
|
|
44
45
|
* ```
|
|
45
46
|
* postNavigationHooks: [
|
|
@@ -52,7 +53,7 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
52
53
|
* ]
|
|
53
54
|
* ```
|
|
54
55
|
*/
|
|
55
|
-
postNavigationHooks?:
|
|
56
|
+
postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
56
57
|
}
|
|
57
58
|
/**
|
|
58
59
|
* Provides a simple framework for parallel crawling of web pages
|
|
@@ -65,24 +66,26 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
65
66
|
* If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
|
|
66
67
|
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
67
68
|
*
|
|
68
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
69
|
-
* {@link
|
|
70
|
-
*
|
|
69
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
70
|
+
* {@link IRequestManager|request manager} provided via the {@link PuppeteerCrawlerOptions.requestManager|`requestManager`}
|
|
71
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
72
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
73
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
74
|
+
* result as `requestManager`.
|
|
71
75
|
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
76
|
+
* > The {@link PuppeteerCrawlerOptions.requestList|`requestList`} and {@link PuppeteerCrawlerOptions.requestQueue|`requestQueue`}
|
|
77
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
75
78
|
*
|
|
76
79
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
77
80
|
*
|
|
78
81
|
* `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
|
|
79
82
|
* and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.requestHandler} option.
|
|
80
83
|
*
|
|
81
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
* {@link
|
|
84
|
+
* New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
|
|
85
|
+
* {@link ConcurrencySystem}.
|
|
86
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
87
|
+
* `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
88
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
86
89
|
*
|
|
87
90
|
* Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
|
|
88
91
|
*
|
|
@@ -117,99 +120,135 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
117
120
|
* ```
|
|
118
121
|
* @category Crawlers
|
|
119
122
|
*/
|
|
120
|
-
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse,
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
124
|
+
#private;
|
|
125
|
+
/**
|
|
126
|
+
* @internal
|
|
127
|
+
*/
|
|
123
128
|
protected static optionsShape: {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
155
|
-
|
|
156
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
182
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
183
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
184
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
185
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
186
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
187
|
-
storageClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
188
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
189
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
190
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
191
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
192
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
193
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
194
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
195
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
196
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
197
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
198
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
199
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
200
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
201
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
202
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
203
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
204
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
205
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
129
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
130
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
131
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
132
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
133
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
134
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
135
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
136
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
137
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
138
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
139
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
140
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
141
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
142
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
143
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
144
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
145
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
146
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
147
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
148
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
149
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
150
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
151
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
152
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
153
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
154
|
+
deferred: "deferred";
|
|
155
|
+
writeThrough: "writeThrough";
|
|
156
|
+
}>>;
|
|
157
|
+
}, z.core.$strict>]>>;
|
|
158
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
159
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
160
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
161
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
162
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
163
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
164
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
165
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
166
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
167
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
168
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
169
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
170
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
171
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
172
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
173
|
+
id: z.ZodOptional<z.ZodString>;
|
|
174
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
175
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
176
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
177
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
178
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
179
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
180
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
181
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
182
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
183
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
184
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
185
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
206
186
|
};
|
|
187
|
+
/** @internal */
|
|
188
|
+
protected static optionsSchema: z.ZodObject<{
|
|
189
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
190
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
191
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
192
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
193
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
194
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
195
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
196
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
197
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
198
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
199
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
200
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
201
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
202
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
203
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
204
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
205
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
206
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
207
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
208
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
209
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
210
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
211
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
212
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
213
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
214
|
+
deferred: "deferred";
|
|
215
|
+
writeThrough: "writeThrough";
|
|
216
|
+
}>>;
|
|
217
|
+
}, z.core.$strict>]>>;
|
|
218
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
219
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
220
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
221
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
222
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
223
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
224
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
225
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
226
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
227
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
228
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
229
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
230
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
231
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
232
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
233
|
+
id: z.ZodOptional<z.ZodString>;
|
|
234
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
235
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
236
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
237
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
238
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
239
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
240
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
241
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
242
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
243
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
244
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
245
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
246
|
+
}, z.core.$strict>;
|
|
207
247
|
/**
|
|
208
248
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
209
249
|
*/
|
|
210
|
-
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
|
|
211
|
-
|
|
212
|
-
protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
|
|
250
|
+
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
251
|
+
protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
|
|
213
252
|
}
|
|
214
253
|
/**
|
|
215
254
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -235,6 +274,6 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
235
274
|
* await crawler.run();
|
|
236
275
|
* ```
|
|
237
276
|
*/
|
|
238
|
-
|
|
239
|
-
export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData
|
|
240
|
-
|
|
277
|
+
export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
278
|
+
export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
279
|
+
export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
|
|
2
2
|
import { serviceLocator } from '@crawlee/core';
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
|
|
6
|
+
import * as puppeteerUtils from './utils/puppeteer-utils.js';
|
|
6
7
|
/**
|
|
7
8
|
* Provides a simple framework for parallel crawling of web pages
|
|
8
9
|
* using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
|
|
@@ -14,24 +15,26 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
|
|
|
14
15
|
* If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
|
|
15
16
|
* which downloads the pages using raw HTTP requests and is about 10x faster.
|
|
16
17
|
*
|
|
17
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
18
|
-
* {@link
|
|
19
|
-
*
|
|
18
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
19
|
+
* {@link IRequestManager|request manager} provided via the {@link PuppeteerCrawlerOptions.requestManager|`requestManager`}
|
|
20
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
21
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
22
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
23
|
+
* result as `requestManager`.
|
|
20
24
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
25
|
+
* > The {@link PuppeteerCrawlerOptions.requestList|`requestList`} and {@link PuppeteerCrawlerOptions.requestQueue|`requestQueue`}
|
|
26
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
24
27
|
*
|
|
25
28
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
26
29
|
*
|
|
27
30
|
* `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
|
|
28
31
|
* and then calls the function provided by user as the {@link PuppeteerCrawlerOptions.requestHandler} option.
|
|
29
32
|
*
|
|
30
|
-
* New pages are only opened when there is enough free CPU and memory available,
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* {@link
|
|
33
|
+
* New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
|
|
34
|
+
* {@link ConcurrencySystem}.
|
|
35
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
36
|
+
* `PuppeteerCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
37
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
35
38
|
*
|
|
36
39
|
* Note that the pool of Puppeteer instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
|
|
37
40
|
*
|
|
@@ -67,46 +70,65 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
|
|
|
67
70
|
* @category Crawlers
|
|
68
71
|
*/
|
|
69
72
|
export class PuppeteerCrawler extends BrowserCrawler {
|
|
73
|
+
/**
|
|
74
|
+
* @internal
|
|
75
|
+
*/
|
|
70
76
|
static optionsShape = {
|
|
71
77
|
...BrowserCrawler.optionsShape,
|
|
72
|
-
|
|
78
|
+
launchContext: schemas.anyObject.default(() => ({})),
|
|
79
|
+
// Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
|
|
80
|
+
// time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
|
|
81
|
+
headless: z.union([z.boolean(), z.string()]).optional(),
|
|
73
82
|
};
|
|
83
|
+
/** @internal */
|
|
84
|
+
static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
|
|
74
85
|
/**
|
|
75
86
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
76
87
|
*/
|
|
77
88
|
constructor(options = {}) {
|
|
78
|
-
|
|
79
|
-
const { launchContext
|
|
80
|
-
const browserPoolOptions = {
|
|
81
|
-
...options.browserPoolOptions,
|
|
82
|
-
};
|
|
89
|
+
const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
|
|
90
|
+
const { launchContext, headless, configuration, proxyConfiguration, ...browserCrawlerOptions } = parsedOptions;
|
|
83
91
|
if (launchContext.proxyUrl) {
|
|
84
92
|
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
|
|
85
93
|
'Use PuppeteerCrawlerOptions.proxyConfiguration');
|
|
86
94
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
95
|
+
if (options.browserPool) {
|
|
96
|
+
// The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
|
|
97
|
+
assertBrowserPoolNotConfigured(new.target.name, {
|
|
98
|
+
launchContext: options.launchContext,
|
|
99
|
+
headless: options.headless,
|
|
100
|
+
});
|
|
91
101
|
}
|
|
92
|
-
if (headless != null) {
|
|
93
|
-
launchContext.launchOptions ??= {};
|
|
94
|
-
launchContext.launchOptions.headless = headless;
|
|
95
|
-
}
|
|
96
|
-
const puppeteerLauncher = new PuppeteerLauncher(launchContext, options.configuration);
|
|
97
|
-
browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];
|
|
98
102
|
super({
|
|
99
103
|
...browserCrawlerOptions,
|
|
100
|
-
|
|
104
|
+
configuration,
|
|
101
105
|
proxyConfiguration,
|
|
102
|
-
|
|
103
|
-
contextPipelineBuilder: () => this
|
|
106
|
+
browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
|
|
107
|
+
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
104
108
|
});
|
|
105
109
|
}
|
|
106
|
-
|
|
110
|
+
#buildContextPipeline() {
|
|
111
|
+
return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
|
|
112
|
+
}
|
|
113
|
+
async #enhanceContext(context) {
|
|
107
114
|
const waitForSelector = async (selector, timeoutMs = 5_000) => {
|
|
108
115
|
await context.page.waitForSelector(selector, { timeout: timeoutMs });
|
|
109
116
|
};
|
|
117
|
+
const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
|
|
118
|
+
if (selector) {
|
|
119
|
+
await waitForSelector(selector, timeoutMs);
|
|
120
|
+
}
|
|
121
|
+
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
122
|
+
};
|
|
123
|
+
const querySelectorAll = async (selector, timeoutMs) => {
|
|
124
|
+
const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
|
|
125
|
+
// The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
|
|
126
|
+
// or an element under a shadow root/iframe the snapshot was told to ignore.
|
|
127
|
+
if (matches.length === 0) {
|
|
128
|
+
throw new Error(`Selector '${selector}' not found in the page snapshot.`);
|
|
129
|
+
}
|
|
130
|
+
return matches;
|
|
131
|
+
};
|
|
110
132
|
return {
|
|
111
133
|
injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
|
|
112
134
|
injectJQuery: async () => {
|
|
@@ -118,15 +140,12 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
118
140
|
await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
|
|
119
141
|
},
|
|
120
142
|
waitForSelector,
|
|
121
|
-
parseWithCheerio
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}
|
|
125
|
-
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
126
|
-
},
|
|
143
|
+
parseWithCheerio,
|
|
144
|
+
querySelectorAll,
|
|
145
|
+
querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
|
|
127
146
|
enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
|
|
128
147
|
page: context.page,
|
|
129
|
-
|
|
148
|
+
requestManager: this.requestManager,
|
|
130
149
|
...options,
|
|
131
150
|
}),
|
|
132
151
|
blockRequests: async (options) => puppeteerUtils.blockRequests(context.page, options),
|
|
@@ -134,39 +153,16 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
134
153
|
addInterceptRequestHandler: async (handler) => puppeteerUtils.addInterceptRequestHandler(context.page, handler),
|
|
135
154
|
removeInterceptRequestHandler: async (handler) => puppeteerUtils.removeInterceptRequestHandler(context.page, handler),
|
|
136
155
|
infiniteScroll: async (options) => puppeteerUtils.infiniteScroll(context.page, options),
|
|
137
|
-
saveSnapshot: async (options) => puppeteerUtils.saveSnapshot(context.page, {
|
|
138
|
-
|
|
156
|
+
saveSnapshot: async (options) => puppeteerUtils.saveSnapshot(context.page, {
|
|
157
|
+
...options,
|
|
158
|
+
configuration: serviceLocator.getConfiguration(),
|
|
159
|
+
}),
|
|
139
160
|
};
|
|
140
161
|
}
|
|
141
|
-
async
|
|
142
|
-
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
162
|
+
async navigationHandler(crawlingContext, gotoOptions) {
|
|
163
|
+
return puppeteerUtils.gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
143
164
|
}
|
|
144
165
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
* This instance can then serve as a `requestHandler` of your {@link PuppeteerCrawler}.
|
|
148
|
-
* Defaults to the {@link PuppeteerCrawlingContext}.
|
|
149
|
-
*
|
|
150
|
-
* > Serves as a shortcut for using `Router.create<PuppeteerCrawlingContext>()`.
|
|
151
|
-
*
|
|
152
|
-
* ```ts
|
|
153
|
-
* import { PuppeteerCrawler, createPuppeteerRouter } from 'crawlee';
|
|
154
|
-
*
|
|
155
|
-
* const router = createPuppeteerRouter();
|
|
156
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
157
|
-
* ctx.log.info('...');
|
|
158
|
-
* });
|
|
159
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
160
|
-
* ctx.log.info('...');
|
|
161
|
-
* });
|
|
162
|
-
*
|
|
163
|
-
* const crawler = new PuppeteerCrawler({
|
|
164
|
-
* requestHandler: router,
|
|
165
|
-
* });
|
|
166
|
-
* await crawler.run();
|
|
167
|
-
* ```
|
|
168
|
-
*/
|
|
169
|
-
export function createPuppeteerRouter(routes) {
|
|
170
|
-
return Router.create(routes);
|
|
166
|
+
export function createPuppeteerRouter(routesOrSchemas) {
|
|
167
|
+
return Router.create(routesOrSchemas);
|
|
171
168
|
}
|
|
172
|
-
//# sourceMappingURL=puppeteer-crawler.js.map
|