@crawlee/puppeteer 4.0.0-beta.124 → 4.0.0-beta.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
5
|
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
package/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
5
|
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { Configuration } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type { Page } from 'puppeteer';
|
|
5
|
+
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
6
|
+
/** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
|
|
7
|
+
export type PuppeteerBrowserPool = BrowserPool<{
|
|
8
|
+
browserPlugins: [PuppeteerPlugin];
|
|
9
|
+
}, [PuppeteerPlugin]>;
|
|
10
|
+
export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
|
|
11
|
+
/** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
|
|
12
|
+
launchContext?: PuppeteerLaunchContext;
|
|
13
|
+
/**
|
|
14
|
+
* Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
|
|
15
|
+
* Defaults to `true`, and can also be set via {@link Configuration}.
|
|
16
|
+
*/
|
|
17
|
+
headless?: boolean | 'new' | 'old';
|
|
18
|
+
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
|
+
configuration?: Configuration;
|
|
20
|
+
}
|
|
21
|
+
export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
25
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
26
|
+
*
|
|
27
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
28
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
29
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
30
|
+
*
|
|
31
|
+
* **Example usage:**
|
|
32
|
+
*
|
|
33
|
+
* ```javascript
|
|
34
|
+
* const crawler = new PuppeteerCrawler({
|
|
35
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
36
|
+
* requestHandler: async ({ page }) => { ... },
|
|
37
|
+
* });
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
41
|
+
*
|
|
42
|
+
* @category Browser management
|
|
43
|
+
*/
|
|
44
|
+
export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
|
|
45
|
+
/**
|
|
46
|
+
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
47
|
+
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
48
|
+
*
|
|
49
|
+
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
50
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
51
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
52
|
+
*
|
|
53
|
+
* @category Browser management
|
|
54
|
+
*/
|
|
55
|
+
export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { PuppeteerLauncher } from './puppeteer-launcher.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
4
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
5
|
+
*
|
|
6
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
7
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
8
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
9
|
+
*
|
|
10
|
+
* **Example usage:**
|
|
11
|
+
*
|
|
12
|
+
* ```javascript
|
|
13
|
+
* const crawler = new PuppeteerCrawler({
|
|
14
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
15
|
+
* requestHandler: async ({ page }) => { ... },
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
20
|
+
*
|
|
21
|
+
* @category Browser management
|
|
22
|
+
*/
|
|
23
|
+
export function puppeteerBrowserPool(options = {}) {
|
|
24
|
+
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
25
|
+
return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
29
|
+
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
30
|
+
*
|
|
31
|
+
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
32
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
33
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
34
|
+
*
|
|
35
|
+
* @category Browser management
|
|
36
|
+
*/
|
|
37
|
+
export function remotePuppeteerBrowserPool(options) {
|
|
38
|
+
const { launchContext, headless, configuration, ...remoteOptions } = options;
|
|
39
|
+
return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
|
|
40
|
+
}
|
|
41
|
+
function puppeteerLauncher(launchContext = {}, headless, configuration) {
|
|
42
|
+
return new PuppeteerLauncher(headless == null
|
|
43
|
+
? launchContext
|
|
44
|
+
: {
|
|
45
|
+
...launchContext,
|
|
46
|
+
launchOptions: { ...launchContext.launchOptions, headless: headless },
|
|
47
|
+
}, configuration);
|
|
48
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
2
2
|
import { BrowserCrawler } from '@crawlee/browser';
|
|
3
|
-
import type { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
4
3
|
import type { Dictionary } from '@crawlee/types';
|
|
5
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
5
|
import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
|
|
@@ -13,13 +12,16 @@ export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
|
13
12
|
export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
|
14
13
|
}
|
|
15
14
|
export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
|
|
16
|
-
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']
|
|
17
|
-
browserPlugins: [PuppeteerPlugin];
|
|
18
|
-
}, Routes> {
|
|
15
|
+
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
19
16
|
/**
|
|
20
17
|
* Options used by {@link launchPuppeteer} to start new Puppeteer instances.
|
|
21
18
|
*/
|
|
22
19
|
launchContext?: PuppeteerLaunchContext;
|
|
20
|
+
/**
|
|
21
|
+
* Whether to run browser in headless mode. Defaults to `true`.
|
|
22
|
+
* Can be also set via {@link Configuration}.
|
|
23
|
+
*/
|
|
24
|
+
headless?: boolean | 'new' | 'old';
|
|
23
25
|
/**
|
|
24
26
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
25
27
|
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
@@ -120,17 +122,15 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
120
122
|
* ```
|
|
121
123
|
* @category Crawlers
|
|
122
124
|
*/
|
|
123
|
-
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']
|
|
124
|
-
browserPlugins: [PuppeteerPlugin];
|
|
125
|
-
}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
125
|
+
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
126
126
|
protected static optionsShape: {
|
|
127
|
-
|
|
127
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
128
128
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
129
129
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
130
130
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
131
131
|
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
132
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
133
132
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
133
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
134
134
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
135
135
|
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
136
136
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -182,13 +182,13 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
182
182
|
id: z.ZodOptional<z.ZodString>;
|
|
183
183
|
};
|
|
184
184
|
protected static optionsSchema: z.ZodObject<{
|
|
185
|
-
|
|
185
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
186
186
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
187
187
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
188
188
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
189
189
|
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
190
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
191
190
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
191
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
192
192
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
193
193
|
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
194
194
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -242,7 +242,7 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
242
242
|
/**
|
|
243
243
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
244
244
|
*/
|
|
245
|
-
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
|
|
245
|
+
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
246
246
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
247
247
|
protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
|
|
248
248
|
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
|
|
2
|
-
import { parseArgument,
|
|
1
|
+
import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';
|
|
2
|
+
import { parseArgument, serviceLocator } from '@crawlee/core';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import {
|
|
4
|
+
import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';
|
|
5
5
|
import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
|
|
6
6
|
/**
|
|
7
7
|
* Provides a simple framework for parallel crawling of web pages
|
|
@@ -71,7 +71,9 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
|
|
|
71
71
|
export class PuppeteerCrawler extends BrowserCrawler {
|
|
72
72
|
static optionsShape = {
|
|
73
73
|
...BrowserCrawler.optionsShape,
|
|
74
|
-
|
|
74
|
+
// Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
|
|
75
|
+
// time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
|
|
76
|
+
headless: z.union([z.boolean(), z.string()]).optional(),
|
|
75
77
|
};
|
|
76
78
|
static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
|
|
77
79
|
/**
|
|
@@ -79,30 +81,26 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
79
81
|
*/
|
|
80
82
|
constructor(options = {}) {
|
|
81
83
|
const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
|
|
82
|
-
const { launchContext, headless, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
83
|
-
const browserPoolOptions = {
|
|
84
|
-
...parsedOptions.browserPoolOptions,
|
|
85
|
-
};
|
|
84
|
+
const { launchContext, headless, configuration, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
86
85
|
if (launchContext.proxyUrl) {
|
|
87
86
|
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
|
|
88
87
|
'Use PuppeteerCrawlerOptions.proxyConfiguration');
|
|
89
88
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
launchContext.launchOptions ??= {};
|
|
97
|
-
launchContext.launchOptions.headless = headless;
|
|
89
|
+
if (options.browserPool) {
|
|
90
|
+
// The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
|
|
91
|
+
assertBrowserPoolNotConfigured(new.target.name, {
|
|
92
|
+
launchContext: options.launchContext,
|
|
93
|
+
headless: options.headless,
|
|
94
|
+
});
|
|
98
95
|
}
|
|
99
|
-
const puppeteerLauncher = new PuppeteerLauncher(launchContext, parsedOptions.configuration);
|
|
100
|
-
browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];
|
|
101
96
|
super({
|
|
102
97
|
...browserCrawlerOptions,
|
|
103
98
|
launchContext,
|
|
99
|
+
configuration,
|
|
104
100
|
proxyConfiguration,
|
|
105
|
-
|
|
101
|
+
browserPoolBuilder: (remoteBrowser) => remoteBrowser
|
|
102
|
+
? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
|
|
103
|
+
: puppeteerBrowserPool({ launchContext, headless, configuration }),
|
|
106
104
|
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
107
105
|
});
|
|
108
106
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.126",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/types": "4.0.0-beta.
|
|
55
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/browser": "4.0.0-beta.126",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.126",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.126",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.126",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.126",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"devtools-protocol": "*",
|
|
58
58
|
"jquery": "^3.7.1",
|
|
@@ -78,5 +78,5 @@
|
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
80
|
},
|
|
81
|
-
"gitHead": "
|
|
81
|
+
"gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
|
|
82
82
|
}
|