@crawlee/puppeteer 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +3 -5
- package/index.js +1 -3
- package/internals/enqueue-links/click-elements.d.ts +6 -7
- package/internals/enqueue-links/click-elements.js +14 -8
- package/internals/puppeteer-browser-pool.d.ts +4 -15
- package/internals/puppeteer-browser-pool.js +0 -14
- package/internals/puppeteer-crawler.d.ts +32 -46
- package/internals/puppeteer-crawler.js +35 -21
- package/internals/puppeteer-launcher.d.ts +8 -4
- package/internals/puppeteer-launcher.js +8 -2
- package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +6 -2
- package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +22 -69
- package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +12 -132
- package/package.json +14 -17
- /package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
package/index.d.ts
CHANGED
|
@@ -2,9 +2,7 @@ export * from '@crawlee/browser';
|
|
|
2
2
|
export * from './internals/puppeteer-browser-pool.js';
|
|
3
3
|
export * from './internals/puppeteer-crawler.js';
|
|
4
4
|
export * from './internals/puppeteer-launcher.js';
|
|
5
|
-
export
|
|
6
|
-
export
|
|
7
|
-
export
|
|
8
|
-
export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
|
|
9
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
5
|
+
export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
|
|
6
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
7
|
+
export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
|
|
10
8
|
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
|
package/index.js
CHANGED
|
@@ -2,6 +2,4 @@ export * from '@crawlee/browser';
|
|
|
2
2
|
export * from './internals/puppeteer-browser-pool.js';
|
|
3
3
|
export * from './internals/puppeteer-crawler.js';
|
|
4
4
|
export * from './internals/puppeteer-launcher.js';
|
|
5
|
-
export * as
|
|
6
|
-
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
7
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
5
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
|
|
2
2
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
import type { ClickOptions, Page
|
|
4
|
+
import type { ClickOptions, Page } from 'puppeteer';
|
|
5
5
|
export interface EnqueueLinksByClickingElementsOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
@@ -142,7 +142,7 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
142
142
|
* **Example usage**
|
|
143
143
|
*
|
|
144
144
|
* ```javascript
|
|
145
|
-
* await
|
|
145
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
146
146
|
* page,
|
|
147
147
|
* requestManager,
|
|
148
148
|
* selector: 'a.product-detail',
|
|
@@ -169,20 +169,19 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
|
|
|
169
169
|
* Clicks all elements of given page matching given selector.
|
|
170
170
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
171
171
|
* Returns a list of all target URLs.
|
|
172
|
+
*
|
|
173
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
172
174
|
* @ignore
|
|
173
175
|
*/
|
|
174
176
|
export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
|
|
175
|
-
/**
|
|
176
|
-
* We're only interested in pages created by the page we're currently clicking in.
|
|
177
|
-
* There will generally be a lot of other targets being created in the browser.
|
|
178
|
-
*/
|
|
179
|
-
export declare function isTargetRelevant(page: Page, target: Target): boolean;
|
|
180
177
|
/**
|
|
181
178
|
* Click all elements matching the given selector. To be able to do this using
|
|
182
179
|
* Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
|
|
183
180
|
* so we first move them to the top of the page's stacking context and then click.
|
|
184
181
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
185
182
|
* for large element sets, this will take considerable amount of time.
|
|
183
|
+
*
|
|
184
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
186
185
|
* @ignore
|
|
187
186
|
*/
|
|
188
187
|
export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions,
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
4
|
import { z } from 'zod';
|
|
4
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/
|
|
5
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
|
|
5
6
|
const STARTING_Z_INDEX = 2147400000;
|
|
6
7
|
const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
7
8
|
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
@@ -47,7 +48,7 @@ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
|
47
48
|
* **Example usage**
|
|
48
49
|
*
|
|
49
50
|
* ```javascript
|
|
50
|
-
* await
|
|
51
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
51
52
|
* page,
|
|
52
53
|
* requestManager,
|
|
53
54
|
* selector: 'a.product-detail',
|
|
@@ -65,6 +66,7 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
65
66
|
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
66
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
67
68
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
69
|
+
const hasOnSkippedRequest = onSkippedRequest !== undefined;
|
|
68
70
|
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
69
71
|
const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
|
|
70
72
|
const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
|
|
@@ -76,15 +78,15 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
76
78
|
});
|
|
77
79
|
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
78
80
|
const skippedByFilters = [];
|
|
79
|
-
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (
|
|
81
|
+
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
|
|
80
82
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
81
|
-
await Promise.all(skippedByFilters.map(async (
|
|
83
|
+
await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
|
|
82
84
|
}
|
|
83
85
|
if (transformRequestFunction) {
|
|
84
86
|
const skippedByTransform = [];
|
|
85
|
-
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
|
|
87
|
+
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
|
|
86
88
|
if (onSkippedRequest && skippedByTransform.length > 0) {
|
|
87
|
-
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(
|
|
89
|
+
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
const requests = filteredOptions.map((opts) => new Request(opts));
|
|
@@ -95,6 +97,8 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
95
97
|
* Clicks all elements of given page matching given selector.
|
|
96
98
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
97
99
|
* Returns a list of all target URLs.
|
|
100
|
+
*
|
|
101
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
98
102
|
* @ignore
|
|
99
103
|
*/
|
|
100
104
|
export async function clickElementsAndInterceptNavigationRequests(options) {
|
|
@@ -173,7 +177,7 @@ function createTargetCreatedHandler(page, requests) {
|
|
|
173
177
|
* We're only interested in pages created by the page we're currently clicking in.
|
|
174
178
|
* There will generally be a lot of other targets being created in the browser.
|
|
175
179
|
*/
|
|
176
|
-
|
|
180
|
+
function isTargetRelevant(page, target) {
|
|
177
181
|
// oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
|
|
178
182
|
return target.type() === 'page' && page.target() === target.opener();
|
|
179
183
|
}
|
|
@@ -218,6 +222,8 @@ async function preventHistoryNavigation(page) {
|
|
|
218
222
|
* so we first move them to the top of the page's stacking context and then click.
|
|
219
223
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
220
224
|
* for large element sets, this will take considerable amount of time.
|
|
225
|
+
*
|
|
226
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
221
227
|
* @ignore
|
|
222
228
|
*/
|
|
223
229
|
export async function clickElements(page, selector, clickOptions) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Configuration } from '@crawlee/browser';
|
|
2
|
-
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin
|
|
1
|
+
import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
4
|
import type { Page } from 'puppeteer';
|
|
5
5
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
@@ -17,8 +17,8 @@ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'b
|
|
|
17
17
|
headless?: boolean | 'new' | 'old';
|
|
18
18
|
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
19
|
configuration?: Configuration;
|
|
20
|
-
}
|
|
21
|
-
|
|
20
|
+
/** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
|
|
21
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
24
24
|
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
@@ -42,14 +42,3 @@ export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowser
|
|
|
42
42
|
* @category Browser management
|
|
43
43
|
*/
|
|
44
44
|
export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
|
|
45
|
-
/**
|
|
46
|
-
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
47
|
-
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
48
|
-
*
|
|
49
|
-
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
50
|
-
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
51
|
-
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
52
|
-
*
|
|
53
|
-
* @category Browser management
|
|
54
|
-
*/
|
|
55
|
-
export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
@@ -24,20 +24,6 @@ export function puppeteerBrowserPool(options = {}) {
|
|
|
24
24
|
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
25
25
|
return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
26
26
|
}
|
|
27
|
-
/**
|
|
28
|
-
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
29
|
-
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
30
|
-
*
|
|
31
|
-
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
32
|
-
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
33
|
-
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
34
|
-
*
|
|
35
|
-
* @category Browser management
|
|
36
|
-
*/
|
|
37
|
-
export function remotePuppeteerBrowserPool(options) {
|
|
38
|
-
const { launchContext, headless, configuration, ...remoteOptions } = options;
|
|
39
|
-
return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
|
|
40
|
-
}
|
|
41
27
|
function puppeteerLauncher(launchContext = {}, headless, configuration) {
|
|
42
28
|
return new PuppeteerLauncher(headless == null
|
|
43
29
|
? launchContext
|
|
@@ -2,12 +2,10 @@ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUse
|
|
|
2
2
|
import { BrowserCrawler } from '@crawlee/browser';
|
|
3
3
|
import type { Dictionary } from '@crawlee/types';
|
|
4
4
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
5
|
-
import type { HTTPResponse,
|
|
5
|
+
import type { HTTPResponse, Page } from 'puppeteer';
|
|
6
6
|
import { z } from 'zod';
|
|
7
|
-
import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
|
|
8
7
|
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
9
|
-
import type {
|
|
10
|
-
import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
|
|
8
|
+
import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
|
|
11
9
|
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
12
10
|
export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
|
13
11
|
}
|
|
@@ -122,20 +120,12 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
122
120
|
* ```
|
|
123
121
|
* @category Crawlers
|
|
124
122
|
*/
|
|
125
|
-
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse,
|
|
123
|
+
export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
124
|
+
#private;
|
|
125
|
+
/**
|
|
126
|
+
* @internal
|
|
127
|
+
*/
|
|
126
128
|
protected static optionsShape: {
|
|
127
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
128
|
-
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
129
|
-
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
130
|
-
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
131
|
-
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
132
|
-
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
133
|
-
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
134
|
-
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
135
|
-
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
136
|
-
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
137
|
-
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
138
|
-
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
139
129
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
140
130
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
141
131
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -167,26 +157,23 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
167
157
|
}, z.core.$strict>]>>;
|
|
168
158
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
169
159
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
170
|
-
httpClient: z.ZodOptional<z.
|
|
160
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
171
161
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
172
|
-
configuration: z.ZodOptional<z.
|
|
162
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
173
163
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
174
164
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
175
|
-
eventManager: z.ZodOptional<z.
|
|
165
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
176
166
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
177
167
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
178
168
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
169
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
179
170
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
180
171
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
181
172
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
182
173
|
id: z.ZodOptional<z.ZodString>;
|
|
183
|
-
};
|
|
184
|
-
protected static optionsSchema: z.ZodObject<{
|
|
185
|
-
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
186
174
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
187
175
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
188
176
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
189
|
-
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
190
177
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
191
178
|
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
192
179
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
@@ -194,6 +181,11 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
194
181
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
195
182
|
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
196
183
|
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
184
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
185
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
186
|
+
};
|
|
187
|
+
/** @internal */
|
|
188
|
+
protected static optionsSchema: z.ZodObject<{
|
|
197
189
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
198
190
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
199
191
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -225,43 +217,37 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
225
217
|
}, z.core.$strict>]>>;
|
|
226
218
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
227
219
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
228
|
-
httpClient: z.ZodOptional<z.
|
|
220
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
229
221
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
230
|
-
configuration: z.ZodOptional<z.
|
|
222
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
231
223
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
232
224
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
233
|
-
eventManager: z.ZodOptional<z.
|
|
225
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
234
226
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
235
227
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
236
228
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
229
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
237
230
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
238
231
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
239
232
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
240
233
|
id: z.ZodOptional<z.ZodString>;
|
|
234
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
235
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
236
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
237
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
238
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
239
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
240
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
241
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
242
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
243
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
244
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
245
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
241
246
|
}, z.core.$strict>;
|
|
242
247
|
/**
|
|
243
248
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
244
249
|
*/
|
|
245
250
|
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
246
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
247
|
-
protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
|
|
248
|
-
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
|
|
249
|
-
injectJQuery: () => Promise<void>;
|
|
250
|
-
waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
|
|
251
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
252
|
-
parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
|
|
253
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
254
|
-
enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
|
|
255
|
-
blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
|
|
256
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
257
|
-
compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/puppeteer_utils.js").CompiledScriptFunction;
|
|
258
|
-
addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
|
|
259
|
-
removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
|
|
260
|
-
infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
|
|
261
|
-
saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
|
|
262
|
-
closeCookieModals: () => Promise<void>;
|
|
263
|
-
}>;
|
|
264
|
-
private enhanceContext;
|
|
265
251
|
protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
|
|
266
252
|
}
|
|
267
253
|
/**
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
|
|
2
|
+
import { serviceLocator } from '@crawlee/core';
|
|
3
|
+
import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
4
|
import { z } from 'zod';
|
|
4
|
-
import { puppeteerBrowserPool
|
|
5
|
-
import
|
|
5
|
+
import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
|
|
6
|
+
import * as puppeteerUtils from './utils/puppeteer-utils.js';
|
|
6
7
|
/**
|
|
7
8
|
* Provides a simple framework for parallel crawling of web pages
|
|
8
9
|
* using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
|
|
@@ -69,19 +70,24 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
|
|
|
69
70
|
* @category Crawlers
|
|
70
71
|
*/
|
|
71
72
|
export class PuppeteerCrawler extends BrowserCrawler {
|
|
73
|
+
/**
|
|
74
|
+
* @internal
|
|
75
|
+
*/
|
|
72
76
|
static optionsShape = {
|
|
73
77
|
...BrowserCrawler.optionsShape,
|
|
78
|
+
launchContext: schemas.anyObject.default(() => ({})),
|
|
74
79
|
// Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
|
|
75
80
|
// time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
|
|
76
81
|
headless: z.union([z.boolean(), z.string()]).optional(),
|
|
77
82
|
};
|
|
83
|
+
/** @internal */
|
|
78
84
|
static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
|
|
79
85
|
/**
|
|
80
86
|
* All `PuppeteerCrawler` parameters are passed via an options object.
|
|
81
87
|
*/
|
|
82
88
|
constructor(options = {}) {
|
|
83
89
|
const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
|
|
84
|
-
const { launchContext, headless, configuration, proxyConfiguration,
|
|
90
|
+
const { launchContext, headless, configuration, proxyConfiguration, ...browserCrawlerOptions } = parsedOptions;
|
|
85
91
|
if (launchContext.proxyUrl) {
|
|
86
92
|
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
|
|
87
93
|
'Use PuppeteerCrawlerOptions.proxyConfiguration');
|
|
@@ -95,22 +101,34 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
95
101
|
}
|
|
96
102
|
super({
|
|
97
103
|
...browserCrawlerOptions,
|
|
98
|
-
launchContext,
|
|
99
104
|
configuration,
|
|
100
105
|
proxyConfiguration,
|
|
101
|
-
browserPoolBuilder: (remoteBrowser) => remoteBrowser
|
|
102
|
-
|
|
103
|
-
: puppeteerBrowserPool({ launchContext, headless, configuration }),
|
|
104
|
-
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
106
|
+
browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
|
|
107
|
+
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
105
108
|
});
|
|
106
109
|
}
|
|
107
|
-
buildContextPipeline() {
|
|
108
|
-
return
|
|
110
|
+
#buildContextPipeline() {
|
|
111
|
+
return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
|
|
109
112
|
}
|
|
110
|
-
async enhanceContext(context) {
|
|
113
|
+
async #enhanceContext(context) {
|
|
111
114
|
const waitForSelector = async (selector, timeoutMs = 5_000) => {
|
|
112
115
|
await context.page.waitForSelector(selector, { timeout: timeoutMs });
|
|
113
116
|
};
|
|
117
|
+
const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
|
|
118
|
+
if (selector) {
|
|
119
|
+
await waitForSelector(selector, timeoutMs);
|
|
120
|
+
}
|
|
121
|
+
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
122
|
+
};
|
|
123
|
+
const querySelectorAll = async (selector, timeoutMs) => {
|
|
124
|
+
const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
|
|
125
|
+
// The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
|
|
126
|
+
// or an element under a shadow root/iframe the snapshot was told to ignore.
|
|
127
|
+
if (matches.length === 0) {
|
|
128
|
+
throw new Error(`Selector '${selector}' not found in the page snapshot.`);
|
|
129
|
+
}
|
|
130
|
+
return matches;
|
|
131
|
+
};
|
|
114
132
|
return {
|
|
115
133
|
injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
|
|
116
134
|
injectJQuery: async () => {
|
|
@@ -122,12 +140,9 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
122
140
|
await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
|
|
123
141
|
},
|
|
124
142
|
waitForSelector,
|
|
125
|
-
parseWithCheerio
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
}
|
|
129
|
-
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
130
|
-
},
|
|
143
|
+
parseWithCheerio,
|
|
144
|
+
querySelectorAll,
|
|
145
|
+
querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
|
|
131
146
|
enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
|
|
132
147
|
page: context.page,
|
|
133
148
|
requestManager: this.requestManager,
|
|
@@ -142,11 +157,10 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
142
157
|
...options,
|
|
143
158
|
configuration: serviceLocator.getConfiguration(),
|
|
144
159
|
}),
|
|
145
|
-
closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),
|
|
146
160
|
};
|
|
147
161
|
}
|
|
148
162
|
async navigationHandler(crawlingContext, gotoOptions) {
|
|
149
|
-
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
163
|
+
return puppeteerUtils.gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
|
|
150
164
|
}
|
|
151
165
|
}
|
|
152
166
|
export function createPuppeteerRouter(routesOrSchemas) {
|
|
@@ -68,9 +68,10 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
|
|
|
68
68
|
*/
|
|
69
69
|
export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
|
|
70
70
|
readonly configuration: Configuration;
|
|
71
|
+
/**
|
|
72
|
+
* @internal
|
|
73
|
+
*/
|
|
71
74
|
protected static optionsShape: {
|
|
72
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
73
|
-
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
74
75
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
75
76
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
76
77
|
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -80,10 +81,11 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
|
|
|
80
81
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
81
82
|
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
82
83
|
userAgent: z.ZodOptional<z.ZodString>;
|
|
83
|
-
};
|
|
84
|
-
protected static optionsSchema: z.ZodObject<{
|
|
85
84
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
86
85
|
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
86
|
+
};
|
|
87
|
+
/** @internal */
|
|
88
|
+
protected static optionsSchema: z.ZodObject<{
|
|
87
89
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
88
90
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
89
91
|
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -93,6 +95,8 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
|
|
|
93
95
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
94
96
|
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
95
97
|
userAgent: z.ZodOptional<z.ZodString>;
|
|
98
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
99
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
|
|
96
100
|
}, z.core.$strict>;
|
|
97
101
|
/**
|
|
98
102
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { BrowserLauncher, Configuration
|
|
2
|
-
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
1
|
+
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
2
|
+
import { PuppeteerPlugin, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
4
|
import { z } from 'zod';
|
|
4
5
|
/**
|
|
5
6
|
* `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
|
|
@@ -7,10 +8,14 @@ import { z } from 'zod';
|
|
|
7
8
|
*/
|
|
8
9
|
export class PuppeteerLauncher extends BrowserLauncher {
|
|
9
10
|
configuration;
|
|
11
|
+
/**
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
10
14
|
static optionsShape = {
|
|
11
15
|
...BrowserLauncher.optionsShape,
|
|
12
16
|
launcher: schemas.anyObject.optional(),
|
|
13
17
|
};
|
|
18
|
+
/** @internal */
|
|
14
19
|
static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
|
|
15
20
|
/**
|
|
16
21
|
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
|
|
@@ -23,6 +28,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
|
|
|
23
28
|
}, configuration);
|
|
24
29
|
this.configuration = configuration;
|
|
25
30
|
this.Plugin = PuppeteerPlugin;
|
|
31
|
+
this.RemotePlugin = RemotePuppeteerPlugin;
|
|
26
32
|
}
|
|
27
33
|
getDefaultHeadlessOption() {
|
|
28
34
|
const headless = super.getDefaultHeadlessOption();
|
package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js}
RENAMED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import {
|
|
2
|
+
import { serviceLocator } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
4
|
const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
|
|
4
5
|
// We use weak maps here so that the content gets discarded after page gets closed.
|
|
5
6
|
const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
|
|
@@ -62,7 +63,10 @@ async function handleRequest(request, interceptRequestHandlers) {
|
|
|
62
63
|
const originalContinue = request.continue.bind(request);
|
|
63
64
|
request.continue = async (overrides = {}) => {
|
|
64
65
|
wasContinued = true;
|
|
65
|
-
const headers = browserifyHeaders({
|
|
66
|
+
const headers = browserifyHeaders({
|
|
67
|
+
...accumulatedOverrides.headers,
|
|
68
|
+
...(overrides.headers instanceof Headers ? Object.fromEntries(overrides.headers) : overrides.headers),
|
|
69
|
+
});
|
|
66
70
|
Object.assign(accumulatedOverrides, overrides, { headers });
|
|
67
71
|
};
|
|
68
72
|
const { abort, respond } = request;
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* **Example usage:**
|
|
6
6
|
*
|
|
7
7
|
* ```javascript
|
|
8
|
-
* import { launchPuppeteer,
|
|
8
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
9
9
|
*
|
|
10
10
|
* // Open https://www.example.com in Puppeteer
|
|
11
11
|
* const browser = await launchPuppeteer();
|
|
@@ -13,21 +13,21 @@
|
|
|
13
13
|
* await page.goto('https://www.example.com');
|
|
14
14
|
*
|
|
15
15
|
* // Inject jQuery into a page
|
|
16
|
-
* await
|
|
16
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
17
17
|
* ```
|
|
18
18
|
* @module puppeteerUtils
|
|
19
19
|
*/
|
|
20
20
|
import type { Request } from '@crawlee/browser';
|
|
21
21
|
import { Configuration } from '@crawlee/browser';
|
|
22
22
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
23
|
-
import {
|
|
24
|
-
import type {
|
|
23
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
24
|
+
import type { AnyNode } from 'domhandler';
|
|
25
25
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
26
|
-
import type { HTTPResponse, Page
|
|
26
|
+
import type { HTTPResponse, Page } from 'puppeteer';
|
|
27
27
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
|
|
28
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
29
|
-
import type { InterceptHandler } from './
|
|
30
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './
|
|
29
|
+
import type { InterceptHandler } from './puppeteer-request-interception.js';
|
|
30
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
|
|
31
31
|
export interface DirectNavigationOptions {
|
|
32
32
|
/**
|
|
33
33
|
* Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
|
|
@@ -101,7 +101,7 @@ export declare function injectFile(page: Page, filePath: string, options?: Injec
|
|
|
101
101
|
*
|
|
102
102
|
* **Example usage:**
|
|
103
103
|
* ```javascript
|
|
104
|
-
* await
|
|
104
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
105
105
|
* const title = await page.evaluate(() => {
|
|
106
106
|
* return $('head title').text();
|
|
107
107
|
* });
|
|
@@ -122,14 +122,14 @@ export declare function injectJQuery(page: Page, options?: {
|
|
|
122
122
|
*
|
|
123
123
|
* **Example usage:**
|
|
124
124
|
* ```javascript
|
|
125
|
-
* const $ = await
|
|
125
|
+
* const $ = await puppeteerUtils.parseWithCheerio(page);
|
|
126
126
|
* const title = $('title').text();
|
|
127
127
|
* ```
|
|
128
128
|
*
|
|
129
129
|
* @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
|
|
130
130
|
* @param ignoreShadowRoots
|
|
131
131
|
*/
|
|
132
|
-
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<
|
|
132
|
+
export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
|
|
133
133
|
/**
|
|
134
134
|
* Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
|
|
135
135
|
* This is useful to speed up crawling of websites, since it reduces the amount
|
|
@@ -156,13 +156,13 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
|
|
|
156
156
|
*
|
|
157
157
|
* **Example usage**
|
|
158
158
|
* ```javascript
|
|
159
|
-
* import { launchPuppeteer,
|
|
159
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
160
160
|
*
|
|
161
161
|
* const browser = await launchPuppeteer();
|
|
162
162
|
* const page = await browser.newPage();
|
|
163
163
|
*
|
|
164
164
|
* // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
|
|
165
|
-
* await
|
|
165
|
+
* await puppeteerUtils.blockRequests(page, {
|
|
166
166
|
* extraUrlPatterns: ['adsbygoogle.js'],
|
|
167
167
|
* });
|
|
168
168
|
*
|
|
@@ -173,33 +173,6 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
|
|
|
173
173
|
* @param [options]
|
|
174
174
|
*/
|
|
175
175
|
export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
|
|
176
|
-
/**
|
|
177
|
-
* @internal
|
|
178
|
-
*/
|
|
179
|
-
export declare function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
|
|
180
|
-
/**
|
|
181
|
-
* `blockResources()` has a high impact on performance in recent versions of Puppeteer.
|
|
182
|
-
* Until this resolves, please use `utils.puppeteer.blockRequests()`.
|
|
183
|
-
* @deprecated
|
|
184
|
-
*/
|
|
185
|
-
export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
|
|
186
|
-
/**
|
|
187
|
-
* *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
|
|
188
|
-
* performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
|
|
189
|
-
*
|
|
190
|
-
* Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
|
|
191
|
-
* *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
|
|
192
|
-
* This issue should be resolved or atleast mitigated in future iterations of this feature.
|
|
193
|
-
* @param page
|
|
194
|
-
* Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
|
|
195
|
-
* @param cache
|
|
196
|
-
* Object in which responses are stored
|
|
197
|
-
* @param responseUrlRules
|
|
198
|
-
* List of rules that are used to check if the response should be cached.
|
|
199
|
-
* String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
|
|
200
|
-
* @deprecated
|
|
201
|
-
*/
|
|
202
|
-
export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
|
|
203
176
|
/**
|
|
204
177
|
* Compiles a Puppeteer script into an async function that may be executed at any time
|
|
205
178
|
* by providing it with the following object:
|
|
@@ -315,7 +288,6 @@ export interface SaveSnapshotOptions {
|
|
|
315
288
|
* @param [options]
|
|
316
289
|
*/
|
|
317
290
|
export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
|
318
|
-
export declare function closeCookieModals(page: Page): Promise<void>;
|
|
319
291
|
export interface PuppeteerContextUtils {
|
|
320
292
|
/**
|
|
321
293
|
* Injects a JavaScript file into current `page`.
|
|
@@ -365,6 +337,15 @@ export interface PuppeteerContextUtils {
|
|
|
365
337
|
* ```
|
|
366
338
|
*/
|
|
367
339
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
340
|
+
/**
|
|
341
|
+
* Like {@link PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
|
|
342
|
+
* as a Cheerio object. Throws when nothing matches.
|
|
343
|
+
*/
|
|
344
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
345
|
+
/**
|
|
346
|
+
* Like {@link PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
|
|
347
|
+
*/
|
|
348
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
368
349
|
/**
|
|
369
350
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
370
351
|
* When provided with the `selector` argument, it waits for it to be available first.
|
|
@@ -377,7 +358,7 @@ export interface PuppeteerContextUtils {
|
|
|
377
358
|
* });
|
|
378
359
|
* ```
|
|
379
360
|
*/
|
|
380
|
-
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<
|
|
361
|
+
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
381
362
|
/**
|
|
382
363
|
* The function finds elements matching a specific CSS selector in a Puppeteer page,
|
|
383
364
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -548,33 +529,5 @@ export interface PuppeteerContextUtils {
|
|
|
548
529
|
* Saves a full screenshot and HTML of the current page into a Key-Value store.
|
|
549
530
|
*/
|
|
550
531
|
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
|
551
|
-
/**
|
|
552
|
-
* Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
|
|
553
|
-
*
|
|
554
|
-
* Note that this method requires the idcac-playwright package to be installed.
|
|
555
|
-
* Crawlee does not include it by default due to licensing issues.
|
|
556
|
-
*
|
|
557
|
-
* To use this method, please install the package manually by running:
|
|
558
|
-
*
|
|
559
|
-
* ```bash
|
|
560
|
-
* npm install idcac-playwright
|
|
561
|
-
* ```
|
|
562
|
-
*/
|
|
563
|
-
closeCookieModals(): Promise<void>;
|
|
564
532
|
}
|
|
565
533
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
|
|
566
|
-
/** @internal */
|
|
567
|
-
export declare const puppeteerUtils: {
|
|
568
|
-
injectFile: typeof injectFile;
|
|
569
|
-
injectJQuery: typeof injectJQuery;
|
|
570
|
-
enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
|
|
571
|
-
blockRequests: typeof blockRequests;
|
|
572
|
-
compileScript: typeof compileScript;
|
|
573
|
-
gotoExtended: typeof gotoExtended;
|
|
574
|
-
addInterceptRequestHandler: typeof addInterceptRequestHandler;
|
|
575
|
-
removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
|
|
576
|
-
infiniteScroll: typeof infiniteScroll;
|
|
577
|
-
saveSnapshot: typeof saveSnapshot;
|
|
578
|
-
parseWithCheerio: typeof parseWithCheerio;
|
|
579
|
-
closeCookieModals: typeof closeCookieModals;
|
|
580
|
-
};
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* **Example usage:**
|
|
6
6
|
*
|
|
7
7
|
* ```javascript
|
|
8
|
-
* import { launchPuppeteer,
|
|
8
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
9
9
|
*
|
|
10
10
|
* // Open https://www.example.com in Puppeteer
|
|
11
11
|
* const browser = await launchPuppeteer();
|
|
@@ -13,19 +13,20 @@
|
|
|
13
13
|
* await page.goto('https://www.example.com');
|
|
14
14
|
*
|
|
15
15
|
* // Inject jQuery into a page
|
|
16
|
-
* await
|
|
16
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
17
17
|
* ```
|
|
18
18
|
* @module puppeteerUtils
|
|
19
19
|
*/
|
|
20
20
|
import { readFile } from 'node:fs/promises';
|
|
21
21
|
import { createRequire } from 'node:module';
|
|
22
22
|
import vm from 'node:vm';
|
|
23
|
-
import { Configuration, KeyValueStore,
|
|
24
|
-
import {
|
|
23
|
+
import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';
|
|
24
|
+
import { sleep } from '@crawlee/utils';
|
|
25
|
+
import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
|
|
25
26
|
import { z } from 'zod';
|
|
26
27
|
import { LruCache } from '@apify/datastructures';
|
|
27
28
|
import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
|
|
28
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './
|
|
29
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
|
|
29
30
|
const require = createRequire(import.meta.url);
|
|
30
31
|
const jqueryPath = require.resolve('jquery');
|
|
31
32
|
const MAX_INJECT_FILE_CACHE_SIZE = 10;
|
|
@@ -34,7 +35,6 @@ const filePathSchema = z.string();
|
|
|
34
35
|
const injectFileOptionsSchema = z.strictObject({
|
|
35
36
|
surviveNavigations: z.boolean().optional(),
|
|
36
37
|
});
|
|
37
|
-
const responseUrlRulesSchema = schemas.arrayOf(z.union([z.string(), z.instanceof(RegExp)]), 'strings or RegExps');
|
|
38
38
|
const gotoExtendedRequestSchema = z.looseObject({
|
|
39
39
|
url: z.url(),
|
|
40
40
|
method: z.string().optional(),
|
|
@@ -107,7 +107,7 @@ export async function injectFile(page, filePath, options = {}) {
|
|
|
107
107
|
*
|
|
108
108
|
* **Example usage:**
|
|
109
109
|
* ```javascript
|
|
110
|
-
* await
|
|
110
|
+
* await puppeteerUtils.injectJQuery(page);
|
|
111
111
|
* const title = await page.evaluate(() => {
|
|
112
112
|
* return $('head title').text();
|
|
113
113
|
* });
|
|
@@ -129,7 +129,7 @@ export async function injectJQuery(page, options) {
|
|
|
129
129
|
*
|
|
130
130
|
* **Example usage:**
|
|
131
131
|
* ```javascript
|
|
132
|
-
* const $ = await
|
|
132
|
+
* const $ = await puppeteerUtils.parseWithCheerio(page);
|
|
133
133
|
* const title = $('title').text();
|
|
134
134
|
* ```
|
|
135
135
|
*
|
|
@@ -170,6 +170,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
170
170
|
? null
|
|
171
171
|
: (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
|
|
172
172
|
const pageContent = html || (await page.content());
|
|
173
|
+
// Full cheerio (parse5) import for browser compliant parser
|
|
173
174
|
const { load } = await import('cheerio');
|
|
174
175
|
return load(pageContent);
|
|
175
176
|
}
|
|
@@ -199,13 +200,13 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
|
|
|
199
200
|
*
|
|
200
201
|
* **Example usage**
|
|
201
202
|
* ```javascript
|
|
202
|
-
* import { launchPuppeteer,
|
|
203
|
+
* import { launchPuppeteer, puppeteerUtils } from 'crawlee';
|
|
203
204
|
*
|
|
204
205
|
* const browser = await launchPuppeteer();
|
|
205
206
|
* const page = await browser.newPage();
|
|
206
207
|
*
|
|
207
208
|
* // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
|
|
208
|
-
* await
|
|
209
|
+
* await puppeteerUtils.blockRequests(page, {
|
|
209
210
|
* extraUrlPatterns: ['adsbygoogle.js'],
|
|
210
211
|
* });
|
|
211
212
|
*
|
|
@@ -225,7 +226,7 @@ export async function blockRequests(page, options = {}) {
|
|
|
225
226
|
/**
|
|
226
227
|
* @internal
|
|
227
228
|
*/
|
|
228
|
-
|
|
229
|
+
async function sendCDPCommand(page, command, ...args) {
|
|
229
230
|
// In puppeteer 16.x and 17.x, the `_client` method is completely omitted from the types. It's still there and works the same way, but it is hidden.
|
|
230
231
|
// Puppeteer <= 17
|
|
231
232
|
if (Reflect.has(page, '_client')) {
|
|
@@ -239,83 +240,6 @@ export async function sendCDPCommand(page, command, ...args) {
|
|
|
239
240
|
const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
|
|
240
241
|
throw new Error(`Cannot detect CDP client for Puppeteer ${parsed.version}. You should report this to Crawlee, mentioning the puppeteer version you are using.`);
|
|
241
242
|
}
|
|
242
|
-
/**
|
|
243
|
-
* `blockResources()` has a high impact on performance in recent versions of Puppeteer.
|
|
244
|
-
* Until this resolves, please use `utils.puppeteer.blockRequests()`.
|
|
245
|
-
* @deprecated
|
|
246
|
-
*/
|
|
247
|
-
export const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
|
|
248
|
-
serviceLocator
|
|
249
|
-
.getLogger()
|
|
250
|
-
.deprecated('utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. ' +
|
|
251
|
-
'Until this resolves, please use utils.puppeteer.blockRequests()');
|
|
252
|
-
await addInterceptRequestHandler(page, async (request) => {
|
|
253
|
-
const type = request.resourceType();
|
|
254
|
-
if (resourceTypes.includes(type))
|
|
255
|
-
await request.abort();
|
|
256
|
-
else
|
|
257
|
-
await request.continue();
|
|
258
|
-
});
|
|
259
|
-
};
|
|
260
|
-
/**
|
|
261
|
-
* *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
|
|
262
|
-
* performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
|
|
263
|
-
*
|
|
264
|
-
* Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
|
|
265
|
-
* *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
|
|
266
|
-
* This issue should be resolved or atleast mitigated in future iterations of this feature.
|
|
267
|
-
* @param page
|
|
268
|
-
* Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
|
|
269
|
-
* @param cache
|
|
270
|
-
* Object in which responses are stored
|
|
271
|
-
* @param responseUrlRules
|
|
272
|
-
* List of rules that are used to check if the response should be cached.
|
|
273
|
-
* String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
|
|
274
|
-
* @deprecated
|
|
275
|
-
*/
|
|
276
|
-
export async function cacheResponses(page, cache, responseUrlRules) {
|
|
277
|
-
parseArgument(page, validators.browserPage);
|
|
278
|
-
parseArgument(cache, schemas.anyObject);
|
|
279
|
-
parseArgument(responseUrlRules, responseUrlRulesSchema);
|
|
280
|
-
serviceLocator
|
|
281
|
-
.getLogger()
|
|
282
|
-
.deprecated('utils.puppeteer.cacheResponses() has a high impact on performance ' +
|
|
283
|
-
"in recent versions of Puppeteer so it's use is discouraged until this issue resolves.");
|
|
284
|
-
await addInterceptRequestHandler(page, async (request) => {
|
|
285
|
-
const url = request.url();
|
|
286
|
-
if (cache[url]) {
|
|
287
|
-
await request.respond(cache[url]);
|
|
288
|
-
return;
|
|
289
|
-
}
|
|
290
|
-
await request.continue();
|
|
291
|
-
});
|
|
292
|
-
page.on('response', async (response) => {
|
|
293
|
-
const url = response.url();
|
|
294
|
-
// Response is already cached, do nothing
|
|
295
|
-
if (cache[url])
|
|
296
|
-
return;
|
|
297
|
-
const shouldCache = responseUrlRules.some((rule) => {
|
|
298
|
-
if (typeof rule === 'string')
|
|
299
|
-
return url.includes(rule);
|
|
300
|
-
if (rule instanceof RegExp)
|
|
301
|
-
return rule.test(url);
|
|
302
|
-
return false;
|
|
303
|
-
});
|
|
304
|
-
try {
|
|
305
|
-
if (shouldCache) {
|
|
306
|
-
const buffer = await response.buffer();
|
|
307
|
-
cache[url] = {
|
|
308
|
-
status: response.status(),
|
|
309
|
-
headers: response.headers(),
|
|
310
|
-
body: buffer,
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
catch {
|
|
315
|
-
// ignore errors, usually means that buffer is empty or broken connection
|
|
316
|
-
}
|
|
317
|
-
});
|
|
318
|
-
}
|
|
319
243
|
/**
|
|
320
244
|
* Compiles a Puppeteer script into an async function that may be executed at any time
|
|
321
245
|
* by providing it with the following object:
|
|
@@ -545,48 +469,4 @@ export async function saveSnapshot(page, options = {}) {
|
|
|
545
469
|
throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
|
|
546
470
|
}
|
|
547
471
|
}
|
|
548
|
-
let idcacPlaywright = null;
|
|
549
|
-
async function getIdcacPlaywright() {
|
|
550
|
-
if (idcacPlaywright)
|
|
551
|
-
return idcacPlaywright;
|
|
552
|
-
try {
|
|
553
|
-
idcacPlaywright = await import('idcac-playwright');
|
|
554
|
-
}
|
|
555
|
-
catch (error) {
|
|
556
|
-
getLog().warning(`Failed to import 'idcac-playwright'.
|
|
557
|
-
|
|
558
|
-
We recently made idcac-playwright an optional dependency due to licensing issues.
|
|
559
|
-
To use this feature, please install it manually by running
|
|
560
|
-
|
|
561
|
-
npm install idcac-playwright
|
|
562
|
-
|
|
563
|
-
Original error message follows:
|
|
564
|
-
|
|
565
|
-
${error.message}
|
|
566
|
-
`);
|
|
567
|
-
}
|
|
568
|
-
return idcacPlaywright;
|
|
569
|
-
}
|
|
570
|
-
export async function closeCookieModals(page) {
|
|
571
|
-
parseArgument(page, validators.browserPage);
|
|
572
|
-
const idcac = await getIdcacPlaywright();
|
|
573
|
-
if (idcac?.getInjectableScript()) {
|
|
574
|
-
await page.evaluate(idcac.getInjectableScript());
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
472
|
export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
|
|
578
|
-
/** @internal */
|
|
579
|
-
export const puppeteerUtils = {
|
|
580
|
-
injectFile,
|
|
581
|
-
injectJQuery,
|
|
582
|
-
enqueueLinksByClickingElements,
|
|
583
|
-
blockRequests,
|
|
584
|
-
compileScript,
|
|
585
|
-
gotoExtended,
|
|
586
|
-
addInterceptRequestHandler,
|
|
587
|
-
removeInterceptRequestHandler,
|
|
588
|
-
infiniteScroll,
|
|
589
|
-
saveSnapshot,
|
|
590
|
-
parseWithCheerio,
|
|
591
|
-
closeCookieModals,
|
|
592
|
-
};
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -47,26 +47,23 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@apify/datastructures": "^
|
|
51
|
-
"@crawlee/browser": "4.0.0-rc.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-rc.
|
|
53
|
-
"@crawlee/core": "4.0.0-rc.
|
|
54
|
-
"@crawlee/types": "4.0.0-rc.
|
|
55
|
-
"@crawlee/utils": "4.0.0-rc.
|
|
56
|
-
"cheerio": "^1.
|
|
57
|
-
"devtools-protocol": "
|
|
50
|
+
"@apify/datastructures": "^3.0.1",
|
|
51
|
+
"@crawlee/browser": "4.0.0-rc.2",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-rc.2",
|
|
53
|
+
"@crawlee/core": "4.0.0-rc.2",
|
|
54
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
55
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
56
|
+
"cheerio": "^1.2.0",
|
|
57
|
+
"devtools-protocol": "^0.0.1734725",
|
|
58
|
+
"domhandler": "^6.0.1",
|
|
58
59
|
"jquery": "^3.7.1",
|
|
59
|
-
"tslib": "^2.
|
|
60
|
-
"zod": "^4.
|
|
60
|
+
"tslib": "^2.9.1",
|
|
61
|
+
"zod": "^4.6.7"
|
|
61
62
|
},
|
|
62
63
|
"peerDependencies": {
|
|
63
|
-
"idcac-playwright": "^0.2.0",
|
|
64
64
|
"puppeteer": "*"
|
|
65
65
|
},
|
|
66
66
|
"peerDependenciesMeta": {
|
|
67
|
-
"idcac-playwright": {
|
|
68
|
-
"optional": true
|
|
69
|
-
},
|
|
70
67
|
"puppeteer": {
|
|
71
68
|
"optional": true
|
|
72
69
|
}
|
|
@@ -78,5 +75,5 @@
|
|
|
78
75
|
}
|
|
79
76
|
}
|
|
80
77
|
},
|
|
81
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
82
79
|
}
|
/package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts}
RENAMED
|
File without changes
|