@crawlee/playwright 4.0.0-beta.125 → 4.0.0-beta.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts
CHANGED
package/index.js
CHANGED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { Configuration } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type { Page } from 'playwright';
|
|
5
|
+
import type { PlaywrightLaunchContext } from './playwright-launcher.js';
|
|
6
|
+
/** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
|
|
7
|
+
export type PlaywrightBrowserPool = BrowserPool<{
|
|
8
|
+
browserPlugins: [PlaywrightPlugin];
|
|
9
|
+
}, [PlaywrightPlugin]>;
|
|
10
|
+
export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
|
|
11
|
+
/** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
|
|
12
|
+
launchContext?: PlaywrightLaunchContext;
|
|
13
|
+
/**
|
|
14
|
+
* Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
|
|
15
|
+
* Defaults to `true`, and can also be set via {@link Configuration}.
|
|
16
|
+
*/
|
|
17
|
+
headless?: boolean;
|
|
18
|
+
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
|
+
configuration?: Configuration;
|
|
20
|
+
}
|
|
21
|
+
export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
|
|
25
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
26
|
+
*
|
|
27
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
28
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
29
|
+
* given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
|
|
30
|
+
*
|
|
31
|
+
* **Example usage:**
|
|
32
|
+
*
|
|
33
|
+
* ```javascript
|
|
34
|
+
* const crawler = new PlaywrightCrawler({
|
|
35
|
+
* browserPool: playwrightBrowserPool({
|
|
36
|
+
* maxOpenPagesPerBrowser: 1,
|
|
37
|
+
* launchContext: { launcher: firefox },
|
|
38
|
+
* }),
|
|
39
|
+
* requestHandler: async ({ page }) => { ... },
|
|
40
|
+
* });
|
|
41
|
+
* ```
|
|
42
|
+
*
|
|
43
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
44
|
+
*
|
|
45
|
+
* @category Browser management
|
|
46
|
+
*/
|
|
47
|
+
export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
|
|
48
|
+
/**
|
|
49
|
+
* The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
|
|
50
|
+
* service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
|
|
51
|
+
*
|
|
52
|
+
* A {@link PlaywrightCrawler} accepts the same connection details directly via
|
|
53
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
54
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
55
|
+
*
|
|
56
|
+
* **Example usage:**
|
|
57
|
+
*
|
|
58
|
+
* ```javascript
|
|
59
|
+
* const crawler = new PlaywrightCrawler({
|
|
60
|
+
* browserPool: remotePlaywrightBrowserPool({
|
|
61
|
+
* endpoint: 'wss://production-sfo.browserless.io?token=xxx',
|
|
62
|
+
* maxOpenBrowsers: 2,
|
|
63
|
+
* browserPoolOptions: { useFingerprints: false },
|
|
64
|
+
* }),
|
|
65
|
+
* requestHandler: async ({ page }) => { ... },
|
|
66
|
+
* });
|
|
67
|
+
* ```
|
|
68
|
+
*
|
|
69
|
+
* @category Browser management
|
|
70
|
+
*/
|
|
71
|
+
export declare function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { PlaywrightLauncher } from './playwright-launcher.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
|
|
4
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
5
|
+
*
|
|
6
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
7
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
8
|
+
* given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
|
|
9
|
+
*
|
|
10
|
+
* **Example usage:**
|
|
11
|
+
*
|
|
12
|
+
* ```javascript
|
|
13
|
+
* const crawler = new PlaywrightCrawler({
|
|
14
|
+
* browserPool: playwrightBrowserPool({
|
|
15
|
+
* maxOpenPagesPerBrowser: 1,
|
|
16
|
+
* launchContext: { launcher: firefox },
|
|
17
|
+
* }),
|
|
18
|
+
* requestHandler: async ({ page }) => { ... },
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*
|
|
22
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
23
|
+
*
|
|
24
|
+
* @category Browser management
|
|
25
|
+
*/
|
|
26
|
+
export function playwrightBrowserPool(options = {}) {
|
|
27
|
+
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
28
|
+
return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
|
|
32
|
+
* service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
|
|
33
|
+
*
|
|
34
|
+
* A {@link PlaywrightCrawler} accepts the same connection details directly via
|
|
35
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
36
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
37
|
+
*
|
|
38
|
+
* **Example usage:**
|
|
39
|
+
*
|
|
40
|
+
* ```javascript
|
|
41
|
+
* const crawler = new PlaywrightCrawler({
|
|
42
|
+
* browserPool: remotePlaywrightBrowserPool({
|
|
43
|
+
* endpoint: 'wss://production-sfo.browserless.io?token=xxx',
|
|
44
|
+
* maxOpenBrowsers: 2,
|
|
45
|
+
* browserPoolOptions: { useFingerprints: false },
|
|
46
|
+
* }),
|
|
47
|
+
* requestHandler: async ({ page }) => { ... },
|
|
48
|
+
* });
|
|
49
|
+
* ```
|
|
50
|
+
*
|
|
51
|
+
* @category Browser management
|
|
52
|
+
*/
|
|
53
|
+
export function remotePlaywrightBrowserPool(options) {
|
|
54
|
+
const { launchContext, headless, configuration, ...remoteOptions } = options;
|
|
55
|
+
return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
|
|
56
|
+
}
|
|
57
|
+
function playwrightLauncher(launchContext = {}, headless, configuration) {
|
|
58
|
+
return new PlaywrightLauncher(headless == null
|
|
59
|
+
? launchContext
|
|
60
|
+
: { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
|
|
61
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
2
2
|
import { BrowserCrawler } from '@crawlee/browser';
|
|
3
|
-
import type { PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
4
3
|
import type { Dictionary } from '@crawlee/types';
|
|
5
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
5
|
import type { Download, LaunchOptions, Page, Response } from 'playwright';
|
|
@@ -12,13 +11,16 @@ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
|
12
11
|
export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
|
|
13
12
|
}
|
|
14
13
|
export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
|
|
15
|
-
export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
|
|
16
|
-
browserPlugins: [PlaywrightPlugin];
|
|
17
|
-
}, Routes, StatisticStateExtension> {
|
|
14
|
+
export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
18
15
|
/**
|
|
19
16
|
* The same options as used by {@link launchPlaywright}.
|
|
20
17
|
*/
|
|
21
18
|
launchContext?: PlaywrightLaunchContext;
|
|
19
|
+
/**
|
|
20
|
+
* Whether to run browser in headless mode. Defaults to `true`.
|
|
21
|
+
* Can be also set via {@link Configuration}.
|
|
22
|
+
*/
|
|
23
|
+
headless?: boolean;
|
|
22
24
|
/**
|
|
23
25
|
* Function that is called to process each request.
|
|
24
26
|
*
|
|
@@ -142,18 +144,16 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
|
|
|
142
144
|
* ```
|
|
143
145
|
* @category Crawlers
|
|
144
146
|
*/
|
|
145
|
-
export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, {
|
|
146
|
-
browserPlugins: [PlaywrightPlugin];
|
|
147
|
-
}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
147
|
+
export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
148
148
|
protected static optionsShape: {
|
|
149
|
-
|
|
149
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
150
150
|
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
151
151
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
152
152
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
153
153
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
154
154
|
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
155
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
156
155
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
156
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
157
157
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
158
158
|
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
159
159
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -205,14 +205,14 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
|
|
|
205
205
|
id: z.ZodOptional<z.ZodString>;
|
|
206
206
|
};
|
|
207
207
|
protected static optionsSchema: z.ZodObject<{
|
|
208
|
-
|
|
208
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
209
209
|
launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
210
210
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
211
211
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
212
212
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
213
213
|
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
214
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
215
214
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
215
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
216
216
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
217
217
|
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
218
218
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';
|
|
1
|
+
import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator, } from '@crawlee/browser';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import {
|
|
3
|
+
import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
|
|
4
4
|
import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
5
5
|
/**
|
|
6
6
|
* Provides a simple framework for parallel crawling of web pages
|
|
@@ -70,7 +70,7 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
|
|
|
70
70
|
export class PlaywrightCrawler extends BrowserCrawler {
|
|
71
71
|
static optionsShape = {
|
|
72
72
|
...BrowserCrawler.optionsShape,
|
|
73
|
-
|
|
73
|
+
headless: z.boolean().optional(),
|
|
74
74
|
launcher: schemas.anyObject.optional(),
|
|
75
75
|
};
|
|
76
76
|
static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
|
|
@@ -79,29 +79,25 @@ export class PlaywrightCrawler extends BrowserCrawler {
|
|
|
79
79
|
*/
|
|
80
80
|
constructor(options = {}) {
|
|
81
81
|
const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
|
|
82
|
-
const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
83
|
-
const browserPoolOptions = {
|
|
84
|
-
...parsedOptions.browserPoolOptions,
|
|
85
|
-
};
|
|
82
|
+
const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
86
83
|
if (launchContext.proxyUrl) {
|
|
87
84
|
throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
|
|
88
85
|
'Use PlaywrightCrawlerOptions.proxyConfiguration');
|
|
89
86
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
launchContext.launchOptions ??= {};
|
|
97
|
-
launchContext.launchOptions.headless = headless;
|
|
87
|
+
if (options.browserPool) {
|
|
88
|
+
// The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
|
|
89
|
+
assertBrowserPoolNotConfigured(new.target.name, {
|
|
90
|
+
launchContext: options.launchContext,
|
|
91
|
+
headless: options.headless,
|
|
92
|
+
});
|
|
98
93
|
}
|
|
99
|
-
const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);
|
|
100
|
-
browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
|
|
101
94
|
super({
|
|
102
95
|
...browserCrawlerOptions,
|
|
103
96
|
launchContext,
|
|
104
|
-
|
|
97
|
+
configuration,
|
|
98
|
+
browserPoolBuilder: (remoteBrowser) => remoteBrowser
|
|
99
|
+
? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
|
|
100
|
+
: playwrightBrowserPool({ launchContext, headless, configuration }),
|
|
105
101
|
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
106
102
|
});
|
|
107
103
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/playwright",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.126",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,13 +49,13 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^2.0.3",
|
|
51
51
|
"@apify/timeout": "^0.4.4",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
54
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
55
|
-
"@crawlee/cheerio": "4.0.0-beta.
|
|
56
|
-
"@crawlee/core": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.126",
|
|
53
|
+
"@crawlee/browser": "4.0.0-beta.126",
|
|
54
|
+
"@crawlee/browser-pool": "4.0.0-beta.126",
|
|
55
|
+
"@crawlee/cheerio": "4.0.0-beta.126",
|
|
56
|
+
"@crawlee/core": "4.0.0-beta.126",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.126",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.126",
|
|
59
59
|
"cheerio": "^1.0.0",
|
|
60
60
|
"jquery": "^3.7.1",
|
|
61
61
|
"ml-logistic-regression": "^2.0.0",
|
|
@@ -84,5 +84,5 @@
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
},
|
|
87
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
|
|
88
88
|
}
|