@crawlee/puppeteer 4.0.0-beta.222 → 4.0.0-beta.224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +3 -3
- package/index.js +1 -1
- package/internals/enqueue-links/click-elements.js +1 -1
- package/internals/puppeteer-browser-pool.d.ts +4 -15
- package/internals/puppeteer-browser-pool.js +0 -14
- package/internals/puppeteer-crawler.d.ts +1 -1
- package/internals/puppeteer-crawler.js +3 -5
- package/internals/puppeteer-launcher.js +2 -1
- package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +2 -2
- package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +1 -1
- package/package.json +7 -7
- /package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -0
- /package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +0 -0
package/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ export * from '@crawlee/browser';
|
|
|
2
2
|
export * from './internals/puppeteer-browser-pool.js';
|
|
3
3
|
export * from './internals/puppeteer-crawler.js';
|
|
4
4
|
export * from './internals/puppeteer-launcher.js';
|
|
5
|
-
export type { InterceptHandler } from './internals/utils/
|
|
6
|
-
export * as puppeteerUtils from './internals/utils/
|
|
7
|
-
export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/
|
|
5
|
+
export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
|
|
6
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
7
|
+
export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
|
|
8
8
|
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
|
package/index.js
CHANGED
|
@@ -2,4 +2,4 @@ export * from '@crawlee/browser';
|
|
|
2
2
|
export * from './internals/puppeteer-browser-pool.js';
|
|
3
3
|
export * from './internals/puppeteer-crawler.js';
|
|
4
4
|
export * from './internals/puppeteer-launcher.js';
|
|
5
|
-
export * as puppeteerUtils from './internals/utils/
|
|
5
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
@@ -2,7 +2,7 @@ import { URL } from 'node:url';
|
|
|
2
2
|
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
|
|
3
3
|
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/
|
|
5
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
|
|
6
6
|
const STARTING_Z_INDEX = 2147400000;
|
|
7
7
|
const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
8
8
|
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Configuration } from '@crawlee/browser';
|
|
2
|
-
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin
|
|
1
|
+
import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
4
|
import type { Page } from 'puppeteer';
|
|
5
5
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
@@ -17,8 +17,8 @@ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'b
|
|
|
17
17
|
headless?: boolean | 'new' | 'old';
|
|
18
18
|
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
19
|
configuration?: Configuration;
|
|
20
|
-
}
|
|
21
|
-
|
|
20
|
+
/** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
|
|
21
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
@@ -42,14 +42,3 @@ export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowser
|
|
|
42
42
|
* @category Browser management
|
|
43
43
|
*/
|
|
44
44
|
export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
|
|
45
|
-
/**
|
|
46
|
-
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
47
|
-
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
48
|
-
*
|
|
49
|
-
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
50
|
-
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
51
|
-
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
52
|
-
*
|
|
53
|
-
* @category Browser management
|
|
54
|
-
*/
|
|
55
|
-
export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
@@ -24,20 +24,6 @@ export function puppeteerBrowserPool(options = {}) {
|
|
|
24
24
|
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
25
25
|
return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
26
26
|
}
|
|
27
|
-
/**
|
|
28
|
-
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
29
|
-
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
30
|
-
*
|
|
31
|
-
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
32
|
-
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
33
|
-
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
34
|
-
*
|
|
35
|
-
* @category Browser management
|
|
36
|
-
*/
|
|
37
|
-
export function remotePuppeteerBrowserPool(options) {
|
|
38
|
-
const { launchContext, headless, configuration, ...remoteOptions } = options;
|
|
39
|
-
return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
|
|
40
|
-
}
|
|
41
27
|
function puppeteerLauncher(launchContext = {}, headless, configuration) {
|
|
42
28
|
return new PuppeteerLauncher(headless == null
|
|
43
29
|
? launchContext
|
|
@@ -5,7 +5,7 @@ import type { Dictionary } from '@crawlee/types';
|
|
|
5
5
|
import type { HTTPResponse, Page } from 'puppeteer';
|
|
6
6
|
import { z } from 'zod';
|
|
7
7
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
8
|
-
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/
|
|
8
|
+
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
|
|
9
9
|
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
10
10
|
export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
|
11
11
|
}
|
|
@@ -2,8 +2,8 @@ import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
|
|
|
2
2
|
import { serviceLocator } from '@crawlee/core';
|
|
3
3
|
import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { puppeteerBrowserPool
|
|
6
|
-
import * as puppeteerUtils from './utils/
|
|
5
|
+
import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
|
|
6
|
+
import * as puppeteerUtils from './utils/puppeteer-utils.js';
|
|
7
7
|
/**
|
|
8
8
|
* Provides a simple framework for parallel crawling of web pages
|
|
9
9
|
* using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
|
|
@@ -103,9 +103,7 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
103
103
|
...browserCrawlerOptions,
|
|
104
104
|
configuration,
|
|
105
105
|
proxyConfiguration,
|
|
106
|
-
browserPoolBuilder: (remoteBrowser) => remoteBrowser
|
|
107
|
-
? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
|
|
108
|
-
: puppeteerBrowserPool({ launchContext, headless, configuration }),
|
|
106
|
+
browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
|
|
109
107
|
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
110
108
|
});
|
|
111
109
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
2
|
-
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
2
|
+
import { PuppeteerPlugin, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
3
|
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
/**
|
|
@@ -28,6 +28,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
|
|
|
28
28
|
}, configuration);
|
|
29
29
|
this.configuration = configuration;
|
|
30
30
|
this.Plugin = PuppeteerPlugin;
|
|
31
|
+
this.RemotePlugin = RemotePuppeteerPlugin;
|
|
31
32
|
}
|
|
32
33
|
getDefaultHeadlessOption() {
|
|
33
34
|
const headless = super.getDefaultHeadlessOption();
|
|
@@ -25,8 +25,8 @@ import type { CheerioAPI } from 'cheerio';
|
|
|
25
25
|
import type { HTTPResponse, Page } from 'puppeteer';
|
|
26
26
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
|
|
27
27
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
28
|
-
import type { InterceptHandler } from './
|
|
29
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './
|
|
28
|
+
import type { InterceptHandler } from './puppeteer-request-interception.js';
|
|
29
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
|
|
30
30
|
export interface DirectNavigationOptions {
|
|
31
31
|
/**
|
|
32
32
|
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
|
|
@@ -26,7 +26,7 @@ import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/intern
|
|
|
26
26
|
import { z } from 'zod';
|
|
27
27
|
import { LruCache } from '@apify/datastructures';
|
|
28
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
29
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './
|
|
29
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
|
|
30
30
|
const require = createRequire(import.meta.url);
|
|
31
31
|
const jqueryPath = require.resolve('jquery');
|
|
32
32
|
const MAX_INJECT_FILE_CACHE_SIZE = 10;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.224",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^3.0.1",
|
|
51
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/types": "4.0.0-beta.
|
|
55
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/browser": "4.0.0-beta.224",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.224",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.224",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.224",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.224",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"devtools-protocol": "*",
|
|
58
58
|
"jquery": "^3.7.1",
|
|
@@ -74,5 +74,5 @@
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "669f726126327dee3b1d6b950663fd4fceace651"
|
|
78
78
|
}
|
/package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts}
RENAMED
|
File without changes
|
/package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js}
RENAMED
|
File without changes
|