@crawlee/puppeteer 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
package/index.d.ts CHANGED
@@ -2,9 +2,7 @@ export * from '@crawlee/browser';
2
2
  export * from './internals/puppeteer-browser-pool.js';
3
3
  export * from './internals/puppeteer-crawler.js';
4
4
  export * from './internals/puppeteer-launcher.js';
5
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
6
- export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
7
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
8
- export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
9
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
5
+ export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
6
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
7
+ export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
10
8
  export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
package/index.js CHANGED
@@ -2,6 +2,4 @@ export * from '@crawlee/browser';
2
2
  export * from './internals/puppeteer-browser-pool.js';
3
3
  export * from './internals/puppeteer-crawler.js';
4
4
  export * from './internals/puppeteer-launcher.js';
5
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
6
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
7
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
5
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
@@ -1,7 +1,7 @@
1
1
  import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
2
2
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
- import type { ClickOptions, Page, Target } from 'puppeteer';
4
+ import type { ClickOptions, Page } from 'puppeteer';
5
5
  export interface EnqueueLinksByClickingElementsOptions {
6
6
  /**
7
7
  * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
@@ -142,7 +142,7 @@ export interface EnqueueLinksByClickingElementsOptions {
142
142
  * **Example usage**
143
143
  *
144
144
  * ```javascript
145
- * await utils.puppeteer.enqueueLinksByClickingElements({
145
+ * await puppeteerUtils.enqueueLinksByClickingElements({
146
146
  * page,
147
147
  * requestManager,
148
148
  * selector: 'a.product-detail',
@@ -169,20 +169,19 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
169
169
  * Clicks all elements of given page matching given selector.
170
170
  * Catches and intercepts all initiated navigation requests and opened pages.
171
171
  * Returns a list of all target URLs.
172
+ *
173
+ * Not part of the public API — exported only so tests can import this module directly.
172
174
  * @ignore
173
175
  */
174
176
  export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
175
- /**
176
- * We're only interested in pages created by the page we're currently clicking in.
177
- * There will generally be a lot of other targets being created in the browser.
178
- */
179
- export declare function isTargetRelevant(page: Page, target: Target): boolean;
180
177
  /**
181
178
  * Click all elements matching the given selector. To be able to do this using
182
179
  * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
183
180
  * so we first move them to the top of the page's stacking context and then click.
184
181
  * We do all in series to prevent elements from hiding one another. Therefore,
185
182
  * for large element sets, this will take considerable amount of time.
183
+ *
184
+ * Not part of the public API — exported only so tests can import this module directly.
186
185
  * @ignore
187
186
  */
188
187
  export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
@@ -1,7 +1,8 @@
1
1
  import { URL } from 'node:url';
2
- import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request, schemas, serviceLocator, } from '@crawlee/browser';
2
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
4
  import { z } from 'zod';
4
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
5
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
5
6
  const STARTING_Z_INDEX = 2147400000;
6
7
  const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
7
8
  const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
@@ -47,7 +48,7 @@ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
47
48
  * **Example usage**
48
49
  *
49
50
  * ```javascript
50
- * await utils.puppeteer.enqueueLinksByClickingElements({
51
+ * await puppeteerUtils.enqueueLinksByClickingElements({
51
52
  * page,
52
53
  * requestManager,
53
54
  * selector: 'a.product-detail',
@@ -65,6 +66,7 @@ export async function enqueueLinksByClickingElements(options) {
65
66
  const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
66
67
  const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
67
68
  const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
69
+ const hasOnSkippedRequest = onSkippedRequest !== undefined;
68
70
  const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
69
71
  const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
70
72
  const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
@@ -76,15 +78,15 @@ export async function enqueueLinksByClickingElements(options) {
76
78
  });
77
79
  const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
78
80
  const skippedByFilters = [];
79
- let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
81
+ let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
80
82
  if (onSkippedRequest && skippedByFilters.length > 0) {
81
- await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
83
+ await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
82
84
  }
83
85
  if (transformRequestFunction) {
84
86
  const skippedByTransform = [];
85
- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
87
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
86
88
  if (onSkippedRequest && skippedByTransform.length > 0) {
87
- await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
89
+ await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
88
90
  }
89
91
  }
90
92
  const requests = filteredOptions.map((opts) => new Request(opts));
@@ -95,6 +97,8 @@ export async function enqueueLinksByClickingElements(options) {
95
97
  * Clicks all elements of given page matching given selector.
96
98
  * Catches and intercepts all initiated navigation requests and opened pages.
97
99
  * Returns a list of all target URLs.
100
+ *
101
+ * Not part of the public API — exported only so tests can import this module directly.
98
102
  * @ignore
99
103
  */
100
104
  export async function clickElementsAndInterceptNavigationRequests(options) {
@@ -173,7 +177,7 @@ function createTargetCreatedHandler(page, requests) {
173
177
  * We're only interested in pages created by the page we're currently clicking in.
174
178
  * There will generally be a lot of other targets being created in the browser.
175
179
  */
176
- export function isTargetRelevant(page, target) {
180
+ function isTargetRelevant(page, target) {
177
181
  // oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
178
182
  return target.type() === 'page' && page.target() === target.opener();
179
183
  }
@@ -218,6 +222,8 @@ async function preventHistoryNavigation(page) {
218
222
  * so we first move them to the top of the page's stacking context and then click.
219
223
  * We do all in series to prevent elements from hiding one another. Therefore,
220
224
  * for large element sets, this will take considerable amount of time.
225
+ *
226
+ * Not part of the public API — exported only so tests can import this module directly.
221
227
  * @ignore
222
228
  */
223
229
  export async function clickElements(page, selector, clickOptions) {
@@ -1,5 +1,5 @@
1
- import type { Configuration } from '@crawlee/browser';
2
- import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
1
+ import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
4
  import type { Page } from 'puppeteer';
5
5
  import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
@@ -17,8 +17,8 @@ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'b
17
17
  headless?: boolean | 'new' | 'old';
18
18
  /** Configuration to read the browser defaults from. Defaults to the global configuration. */
19
19
  configuration?: Configuration;
20
- }
21
- export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
20
+ /** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
21
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
22
22
  }
23
23
  /**
24
24
  * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
@@ -42,14 +42,3 @@ export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowser
42
42
  * @category Browser management
43
43
  */
44
44
  export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
45
- /**
46
- * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
47
- * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
48
- *
49
- * A {@link PuppeteerCrawler} accepts the same connection details directly via
50
- * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
51
- * tune the wrapping pool, or to share one remote pool between crawlers.
52
- *
53
- * @category Browser management
54
- */
55
- export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
@@ -24,20 +24,6 @@ export function puppeteerBrowserPool(options = {}) {
24
24
  const { launchContext, headless, configuration, ...poolOptions } = options;
25
25
  return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
26
26
  }
27
- /**
28
- * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
29
- * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
30
- *
31
- * A {@link PuppeteerCrawler} accepts the same connection details directly via
32
- * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
33
- * tune the wrapping pool, or to share one remote pool between crawlers.
34
- *
35
- * @category Browser management
36
- */
37
- export function remotePuppeteerBrowserPool(options) {
38
- const { launchContext, headless, configuration, ...remoteOptions } = options;
39
- return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
40
- }
41
27
  function puppeteerLauncher(launchContext = {}, headless, configuration) {
42
28
  return new PuppeteerLauncher(headless == null
43
29
  ? launchContext
@@ -2,12 +2,10 @@ import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUse
2
2
  import { BrowserCrawler } from '@crawlee/browser';
3
3
  import type { Dictionary } from '@crawlee/types';
4
4
  // @ts-ignore optional peer dependency or compatibility with es2022
5
- import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
5
+ import type { HTTPResponse, Page } from 'puppeteer';
6
6
  import { z } from 'zod';
7
- import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
8
7
  import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
9
- import type { InterceptHandler } from './utils/puppeteer_request_interception.js';
10
- import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
8
+ import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
11
9
  export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
12
10
  export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
13
11
  }
@@ -122,20 +120,12 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
122
120
  * ```
123
121
  * @category Crawlers
124
122
  */
125
- export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
123
+ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
124
+ #private;
125
+ /**
126
+ * @internal
127
+ */
126
128
  protected static optionsShape: {
127
- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
128
- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
129
- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
130
- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
131
- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
132
- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
133
- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
134
- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
135
- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
136
- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
137
- ignoreIframes: z.ZodDefault<z.ZodBoolean>;
138
- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
139
129
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
140
130
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
141
131
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -167,26 +157,23 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
167
157
  }, z.core.$strict>]>>;
168
158
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
169
159
  // @ts-ignore optional peer dependency or compatibility with es2022
170
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
160
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
171
161
  // @ts-ignore optional peer dependency or compatibility with es2022
172
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
162
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
173
163
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
174
164
  // @ts-ignore optional peer dependency or compatibility with es2022
175
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
165
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
176
166
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
177
167
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
178
168
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
169
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
179
170
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
180
171
  keepAlive: z.ZodOptional<z.ZodBoolean>;
181
172
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
182
173
  id: z.ZodOptional<z.ZodString>;
183
- };
184
- protected static optionsSchema: z.ZodObject<{
185
- headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
186
174
  navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
187
175
  preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
188
176
  postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
189
- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
190
177
  browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
191
178
  browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
192
179
  remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -194,6 +181,11 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
194
181
  proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
195
182
  ignoreIframes: z.ZodDefault<z.ZodBoolean>;
196
183
  ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
184
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
185
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
186
+ };
187
+ /** @internal */
188
+ protected static optionsSchema: z.ZodObject<{
197
189
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
198
190
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
199
191
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -225,43 +217,37 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
225
217
  }, z.core.$strict>]>>;
226
218
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
227
219
  // @ts-ignore optional peer dependency or compatibility with es2022
228
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
220
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
229
221
  // @ts-ignore optional peer dependency or compatibility with es2022
230
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
222
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
231
223
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
232
224
  // @ts-ignore optional peer dependency or compatibility with es2022
233
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
225
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
234
226
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
235
227
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
236
228
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
229
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
237
230
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
238
231
  keepAlive: z.ZodOptional<z.ZodBoolean>;
239
232
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
240
233
  id: z.ZodOptional<z.ZodString>;
234
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
235
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
236
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
237
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
238
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
239
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
240
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
241
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
242
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
243
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
244
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
245
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
241
246
  }, z.core.$strict>;
242
247
  /**
243
248
  * All `PuppeteerCrawler` parameters are passed via an options object.
244
249
  */
245
250
  constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
246
- // @ts-ignore optional peer dependency or compatibility with es2022
247
- protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
248
- injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
249
- injectJQuery: () => Promise<void>;
250
- waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
251
- // @ts-ignore optional peer dependency or compatibility with es2022
252
- parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
253
- // @ts-ignore optional peer dependency or compatibility with es2022
254
- enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
255
- blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/puppeteer_utils.js").CompiledScriptFunction;
258
- addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
259
- removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
260
- infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
261
- saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
262
- closeCookieModals: () => Promise<void>;
263
- }>;
264
- private enhanceContext;
265
251
  protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
266
252
  }
267
253
  /**
@@ -1,8 +1,9 @@
1
- import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';
2
- import { parseArgument, serviceLocator } from '@crawlee/core';
1
+ import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
2
+ import { serviceLocator } from '@crawlee/core';
3
+ import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
3
4
  import { z } from 'zod';
4
- import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';
5
- import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
5
+ import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
6
+ import * as puppeteerUtils from './utils/puppeteer-utils.js';
6
7
  /**
7
8
  * Provides a simple framework for parallel crawling of web pages
8
9
  * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
@@ -69,19 +70,24 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
69
70
  * @category Crawlers
70
71
  */
71
72
  export class PuppeteerCrawler extends BrowserCrawler {
73
+ /**
74
+ * @internal
75
+ */
72
76
  static optionsShape = {
73
77
  ...BrowserCrawler.optionsShape,
78
+ launchContext: schemas.anyObject.default(() => ({})),
74
79
  // Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
75
80
  // time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
76
81
  headless: z.union([z.boolean(), z.string()]).optional(),
77
82
  };
83
+ /** @internal */
78
84
  static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
79
85
  /**
80
86
  * All `PuppeteerCrawler` parameters are passed via an options object.
81
87
  */
82
88
  constructor(options = {}) {
83
89
  const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
84
- const { launchContext, headless, configuration, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
90
+ const { launchContext, headless, configuration, proxyConfiguration, ...browserCrawlerOptions } = parsedOptions;
85
91
  if (launchContext.proxyUrl) {
86
92
  throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
87
93
  'Use PuppeteerCrawlerOptions.proxyConfiguration');
@@ -95,22 +101,34 @@ export class PuppeteerCrawler extends BrowserCrawler {
95
101
  }
96
102
  super({
97
103
  ...browserCrawlerOptions,
98
- launchContext,
99
104
  configuration,
100
105
  proxyConfiguration,
101
- browserPoolBuilder: (remoteBrowser) => remoteBrowser
102
- ? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
103
- : puppeteerBrowserPool({ launchContext, headless, configuration }),
104
- contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
106
+ browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
107
+ contextPipelineBuilder: () => this.#buildContextPipeline(),
105
108
  });
106
109
  }
107
- buildContextPipeline() {
108
- return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
110
+ #buildContextPipeline() {
111
+ return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
109
112
  }
110
- async enhanceContext(context) {
113
+ async #enhanceContext(context) {
111
114
  const waitForSelector = async (selector, timeoutMs = 5_000) => {
112
115
  await context.page.waitForSelector(selector, { timeout: timeoutMs });
113
116
  };
117
+ const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
118
+ if (selector) {
119
+ await waitForSelector(selector, timeoutMs);
120
+ }
121
+ return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
122
+ };
123
+ const querySelectorAll = async (selector, timeoutMs) => {
124
+ const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
125
+ // The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
126
+ // or an element under a shadow root/iframe the snapshot was told to ignore.
127
+ if (matches.length === 0) {
128
+ throw new Error(`Selector '${selector}' not found in the page snapshot.`);
129
+ }
130
+ return matches;
131
+ };
114
132
  return {
115
133
  injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
116
134
  injectJQuery: async () => {
@@ -122,12 +140,9 @@ export class PuppeteerCrawler extends BrowserCrawler {
122
140
  await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
123
141
  },
124
142
  waitForSelector,
125
- parseWithCheerio: async (selector, timeoutMs = 5_000) => {
126
- if (selector) {
127
- await waitForSelector(selector, timeoutMs);
128
- }
129
- return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
130
- },
143
+ parseWithCheerio,
144
+ querySelectorAll,
145
+ querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
131
146
  enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
132
147
  page: context.page,
133
148
  requestManager: this.requestManager,
@@ -142,11 +157,10 @@ export class PuppeteerCrawler extends BrowserCrawler {
142
157
  ...options,
143
158
  configuration: serviceLocator.getConfiguration(),
144
159
  }),
145
- closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),
146
160
  };
147
161
  }
148
162
  async navigationHandler(crawlingContext, gotoOptions) {
149
- return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
163
+ return puppeteerUtils.gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
150
164
  }
151
165
  }
152
166
  export function createPuppeteerRouter(routesOrSchemas) {
@@ -68,9 +68,10 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
68
68
  */
69
69
  export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
70
70
  readonly configuration: Configuration;
71
+ /**
72
+ * @internal
73
+ */
71
74
  protected static optionsShape: {
72
- // @ts-ignore optional peer dependency or compatibility with es2022
73
- launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
74
75
  proxyUrl: z.ZodOptional<z.ZodURL>;
75
76
  useChrome: z.ZodOptional<z.ZodBoolean>;
76
77
  useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
@@ -80,10 +81,11 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
80
81
  // @ts-ignore optional peer dependency or compatibility with es2022
81
82
  launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
82
83
  userAgent: z.ZodOptional<z.ZodString>;
83
- };
84
- protected static optionsSchema: z.ZodObject<{
85
84
  // @ts-ignore optional peer dependency or compatibility with es2022
86
85
  launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
86
+ };
87
+ /** @internal */
88
+ protected static optionsSchema: z.ZodObject<{
87
89
  proxyUrl: z.ZodOptional<z.ZodURL>;
88
90
  useChrome: z.ZodOptional<z.ZodBoolean>;
89
91
  useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
@@ -93,6 +95,8 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
93
95
  // @ts-ignore optional peer dependency or compatibility with es2022
94
96
  launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
95
97
  userAgent: z.ZodOptional<z.ZodString>;
98
+ // @ts-ignore optional peer dependency or compatibility with es2022
99
+ launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
96
100
  }, z.core.$strict>;
97
101
  /**
98
102
  * All `PuppeteerLauncher` parameters are passed via an launchContext object.
@@ -1,5 +1,6 @@
1
- import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
2
- import { PuppeteerPlugin } from '@crawlee/browser-pool';
1
+ import { BrowserLauncher, Configuration } from '@crawlee/browser';
2
+ import { PuppeteerPlugin, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
4
  import { z } from 'zod';
4
5
  /**
5
6
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
@@ -7,10 +8,14 @@ import { z } from 'zod';
7
8
  */
8
9
  export class PuppeteerLauncher extends BrowserLauncher {
9
10
  configuration;
11
+ /**
12
+ * @internal
13
+ */
10
14
  static optionsShape = {
11
15
  ...BrowserLauncher.optionsShape,
12
16
  launcher: schemas.anyObject.optional(),
13
17
  };
18
+ /** @internal */
14
19
  static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
15
20
  /**
16
21
  * All `PuppeteerLauncher` parameters are passed via an launchContext object.
@@ -23,6 +28,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
23
28
  }, configuration);
24
29
  this.configuration = configuration;
25
30
  this.Plugin = PuppeteerPlugin;
31
+ this.RemotePlugin = RemotePuppeteerPlugin;
26
32
  }
27
33
  getDefaultHeadlessOption() {
28
34
  const headless = super.getDefaultHeadlessOption();
@@ -1,5 +1,6 @@
1
1
  import { EventEmitter } from 'node:events';
2
- import { parseArgument, schemas, serviceLocator } from '@crawlee/browser';
2
+ import { serviceLocator } from '@crawlee/browser';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
3
4
  const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
4
5
  // We use weak maps here so that the content gets discarded after page gets closed.
5
6
  const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
@@ -62,7 +63,10 @@ async function handleRequest(request, interceptRequestHandlers) {
62
63
  const originalContinue = request.continue.bind(request);
63
64
  request.continue = async (overrides = {}) => {
64
65
  wasContinued = true;
65
- const headers = browserifyHeaders({ ...accumulatedOverrides.headers, ...overrides.headers });
66
+ const headers = browserifyHeaders({
67
+ ...accumulatedOverrides.headers,
68
+ ...(overrides.headers instanceof Headers ? Object.fromEntries(overrides.headers) : overrides.headers),
69
+ });
66
70
  Object.assign(accumulatedOverrides, overrides, { headers });
67
71
  };
68
72
  const { abort, respond } = request;
@@ -5,7 +5,7 @@
5
5
  * **Example usage:**
6
6
  *
7
7
  * ```javascript
8
- * import { launchPuppeteer, utils } from 'crawlee';
8
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
9
9
  *
10
10
  * // Open https://www.example.com in Puppeteer
11
11
  * const browser = await launchPuppeteer();
@@ -13,21 +13,21 @@
13
13
  * await page.goto('https://www.example.com');
14
14
  *
15
15
  * // Inject jQuery into a page
16
- * await utils.puppeteer.injectJQuery(page);
16
+ * await puppeteerUtils.injectJQuery(page);
17
17
  * ```
18
18
  * @module puppeteerUtils
19
19
  */
20
20
  import type { Request } from '@crawlee/browser';
21
21
  import { Configuration } from '@crawlee/browser';
22
22
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
23
- import { type CheerioRoot } from '@crawlee/utils/internal';
24
- import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
23
+ import type { Cheerio, CheerioAPI } from 'cheerio';
24
+ import type { AnyNode } from 'domhandler';
25
25
  // @ts-ignore optional peer dependency or compatibility with es2022
26
- import type { HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
26
+ import type { HTTPResponse, Page } from 'puppeteer';
27
27
  import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
28
28
  import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
29
- import type { InterceptHandler } from './puppeteer_request_interception.js';
30
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
29
+ import type { InterceptHandler } from './puppeteer-request-interception.js';
30
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
31
31
  export interface DirectNavigationOptions {
32
32
  /**
33
33
  * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
@@ -101,7 +101,7 @@ export declare function injectFile(page: Page, filePath: string, options?: Injec
101
101
  *
102
102
  * **Example usage:**
103
103
  * ```javascript
104
- * await utils.puppeteer.injectJQuery(page);
104
+ * await puppeteerUtils.injectJQuery(page);
105
105
  * const title = await page.evaluate(() => {
106
106
  * return $('head title').text();
107
107
  * });
@@ -122,14 +122,14 @@ export declare function injectJQuery(page: Page, options?: {
122
122
  *
123
123
  * **Example usage:**
124
124
  * ```javascript
125
- * const $ = await utils.puppeteer.parseWithCheerio(page);
125
+ * const $ = await puppeteerUtils.parseWithCheerio(page);
126
126
  * const title = $('title').text();
127
127
  * ```
128
128
  *
129
129
  * @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
130
130
  * @param ignoreShadowRoots
131
131
  */
132
- export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
132
+ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
133
133
  /**
134
134
  * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
135
135
  * This is useful to speed up crawling of websites, since it reduces the amount
@@ -156,13 +156,13 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
156
156
  *
157
157
  * **Example usage**
158
158
  * ```javascript
159
- * import { launchPuppeteer, utils } from 'crawlee';
159
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
160
160
  *
161
161
  * const browser = await launchPuppeteer();
162
162
  * const page = await browser.newPage();
163
163
  *
164
164
  * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
165
- * await utils.puppeteer.blockRequests(page, {
165
+ * await puppeteerUtils.blockRequests(page, {
166
166
  * extraUrlPatterns: ['adsbygoogle.js'],
167
167
  * });
168
168
  *
@@ -173,33 +173,6 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
173
173
  * @param [options]
174
174
  */
175
175
  export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
176
- /**
177
- * @internal
178
- */
179
- export declare function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
180
- /**
181
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
182
- * Until this resolves, please use `utils.puppeteer.blockRequests()`.
183
- * @deprecated
184
- */
185
- export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
186
- /**
187
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
188
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
189
- *
190
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
191
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
192
- * This issue should be resolved or atleast mitigated in future iterations of this feature.
193
- * @param page
194
- * Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
195
- * @param cache
196
- * Object in which responses are stored
197
- * @param responseUrlRules
198
- * List of rules that are used to check if the response should be cached.
199
- * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
200
- * @deprecated
201
- */
202
- export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
203
176
  /**
204
177
  * Compiles a Puppeteer script into an async function that may be executed at any time
205
178
  * by providing it with the following object:
@@ -315,7 +288,6 @@ export interface SaveSnapshotOptions {
315
288
  * @param [options]
316
289
  */
317
290
  export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
318
- export declare function closeCookieModals(page: Page): Promise<void>;
319
291
  export interface PuppeteerContextUtils {
320
292
  /**
321
293
  * Injects a JavaScript file into current `page`.
@@ -365,6 +337,15 @@ export interface PuppeteerContextUtils {
365
337
  * ```
366
338
  */
367
339
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
340
+ /**
341
+ * Like {@link PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
342
+ * as a Cheerio object. Throws when nothing matches.
343
+ */
344
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
345
+ /**
346
+ * Like {@link PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
347
+ */
348
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
368
349
  /**
369
350
  * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
370
351
  * When provided with the `selector` argument, it waits for it to be available first.
@@ -377,7 +358,7 @@ export interface PuppeteerContextUtils {
377
358
  * });
378
359
  * ```
379
360
  */
380
- parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
361
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
381
362
  /**
382
363
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
383
364
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -548,33 +529,5 @@ export interface PuppeteerContextUtils {
548
529
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
549
530
  */
550
531
  saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
551
- /**
552
- * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
553
- *
554
- * Note that this method requires the idcac-playwright package to be installed.
555
- * Crawlee does not include it by default due to licensing issues.
556
- *
557
- * To use this method, please install the package manually by running:
558
- *
559
- * ```bash
560
- * npm install idcac-playwright
561
- * ```
562
- */
563
- closeCookieModals(): Promise<void>;
564
532
  }
565
533
  export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
566
- /** @internal */
567
- export declare const puppeteerUtils: {
568
- injectFile: typeof injectFile;
569
- injectJQuery: typeof injectJQuery;
570
- enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
571
- blockRequests: typeof blockRequests;
572
- compileScript: typeof compileScript;
573
- gotoExtended: typeof gotoExtended;
574
- addInterceptRequestHandler: typeof addInterceptRequestHandler;
575
- removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
576
- infiniteScroll: typeof infiniteScroll;
577
- saveSnapshot: typeof saveSnapshot;
578
- parseWithCheerio: typeof parseWithCheerio;
579
- closeCookieModals: typeof closeCookieModals;
580
- };
@@ -5,7 +5,7 @@
5
5
  * **Example usage:**
6
6
  *
7
7
  * ```javascript
8
- * import { launchPuppeteer, utils } from 'crawlee';
8
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
9
9
  *
10
10
  * // Open https://www.example.com in Puppeteer
11
11
  * const browser = await launchPuppeteer();
@@ -13,19 +13,20 @@
13
13
  * await page.goto('https://www.example.com');
14
14
  *
15
15
  * // Inject jQuery into a page
16
- * await utils.puppeteer.injectJQuery(page);
16
+ * await puppeteerUtils.injectJQuery(page);
17
17
  * ```
18
18
  * @module puppeteerUtils
19
19
  */
20
20
  import { readFile } from 'node:fs/promises';
21
21
  import { createRequire } from 'node:module';
22
22
  import vm from 'node:vm';
23
- import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, validators } from '@crawlee/browser';
24
- import { expandShadowRoots, sleep } from '@crawlee/utils';
23
+ import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';
24
+ import { sleep } from '@crawlee/utils';
25
+ import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
25
26
  import { z } from 'zod';
26
27
  import { LruCache } from '@apify/datastructures';
27
28
  import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
28
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
29
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
29
30
  const require = createRequire(import.meta.url);
30
31
  const jqueryPath = require.resolve('jquery');
31
32
  const MAX_INJECT_FILE_CACHE_SIZE = 10;
@@ -34,7 +35,6 @@ const filePathSchema = z.string();
34
35
  const injectFileOptionsSchema = z.strictObject({
35
36
  surviveNavigations: z.boolean().optional(),
36
37
  });
37
- const responseUrlRulesSchema = schemas.arrayOf(z.union([z.string(), z.instanceof(RegExp)]), 'strings or RegExps');
38
38
  const gotoExtendedRequestSchema = z.looseObject({
39
39
  url: z.url(),
40
40
  method: z.string().optional(),
@@ -107,7 +107,7 @@ export async function injectFile(page, filePath, options = {}) {
107
107
  *
108
108
  * **Example usage:**
109
109
  * ```javascript
110
- * await utils.puppeteer.injectJQuery(page);
110
+ * await puppeteerUtils.injectJQuery(page);
111
111
  * const title = await page.evaluate(() => {
112
112
  * return $('head title').text();
113
113
  * });
@@ -129,7 +129,7 @@ export async function injectJQuery(page, options) {
129
129
  *
130
130
  * **Example usage:**
131
131
  * ```javascript
132
- * const $ = await utils.puppeteer.parseWithCheerio(page);
132
+ * const $ = await puppeteerUtils.parseWithCheerio(page);
133
133
  * const title = $('title').text();
134
134
  * ```
135
135
  *
@@ -170,6 +170,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
170
170
  ? null
171
171
  : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
172
172
  const pageContent = html || (await page.content());
173
+ // Full cheerio (parse5) import for browser compliant parser
173
174
  const { load } = await import('cheerio');
174
175
  return load(pageContent);
175
176
  }
@@ -199,13 +200,13 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
199
200
  *
200
201
  * **Example usage**
201
202
  * ```javascript
202
- * import { launchPuppeteer, utils } from 'crawlee';
203
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
203
204
  *
204
205
  * const browser = await launchPuppeteer();
205
206
  * const page = await browser.newPage();
206
207
  *
207
208
  * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
208
- * await utils.puppeteer.blockRequests(page, {
209
+ * await puppeteerUtils.blockRequests(page, {
209
210
  * extraUrlPatterns: ['adsbygoogle.js'],
210
211
  * });
211
212
  *
@@ -225,7 +226,7 @@ export async function blockRequests(page, options = {}) {
225
226
  /**
226
227
  * @internal
227
228
  */
228
- export async function sendCDPCommand(page, command, ...args) {
229
+ async function sendCDPCommand(page, command, ...args) {
229
230
  // In puppeteer 16.x and 17.x, the `_client` method is completely omitted from the types. It's still there and works the same way, but it is hidden.
230
231
  // Puppeteer <= 17
231
232
  if (Reflect.has(page, '_client')) {
@@ -239,83 +240,6 @@ export async function sendCDPCommand(page, command, ...args) {
239
240
  const parsed = JSON.parse(await readFile(jsonPath, 'utf-8'));
240
241
  throw new Error(`Cannot detect CDP client for Puppeteer ${parsed.version}. You should report this to Crawlee, mentioning the puppeteer version you are using.`);
241
242
  }
242
- /**
243
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
244
- * Until this resolves, please use `utils.puppeteer.blockRequests()`.
245
- * @deprecated
246
- */
247
- export const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
248
- serviceLocator
249
- .getLogger()
250
- .deprecated('utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. ' +
251
- 'Until this resolves, please use utils.puppeteer.blockRequests()');
252
- await addInterceptRequestHandler(page, async (request) => {
253
- const type = request.resourceType();
254
- if (resourceTypes.includes(type))
255
- await request.abort();
256
- else
257
- await request.continue();
258
- });
259
- };
260
- /**
261
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
262
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
263
- *
264
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
265
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
266
- * This issue should be resolved or atleast mitigated in future iterations of this feature.
267
- * @param page
268
- * Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
269
- * @param cache
270
- * Object in which responses are stored
271
- * @param responseUrlRules
272
- * List of rules that are used to check if the response should be cached.
273
- * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
274
- * @deprecated
275
- */
276
- export async function cacheResponses(page, cache, responseUrlRules) {
277
- parseArgument(page, validators.browserPage);
278
- parseArgument(cache, schemas.anyObject);
279
- parseArgument(responseUrlRules, responseUrlRulesSchema);
280
- serviceLocator
281
- .getLogger()
282
- .deprecated('utils.puppeteer.cacheResponses() has a high impact on performance ' +
283
- "in recent versions of Puppeteer so it's use is discouraged until this issue resolves.");
284
- await addInterceptRequestHandler(page, async (request) => {
285
- const url = request.url();
286
- if (cache[url]) {
287
- await request.respond(cache[url]);
288
- return;
289
- }
290
- await request.continue();
291
- });
292
- page.on('response', async (response) => {
293
- const url = response.url();
294
- // Response is already cached, do nothing
295
- if (cache[url])
296
- return;
297
- const shouldCache = responseUrlRules.some((rule) => {
298
- if (typeof rule === 'string')
299
- return url.includes(rule);
300
- if (rule instanceof RegExp)
301
- return rule.test(url);
302
- return false;
303
- });
304
- try {
305
- if (shouldCache) {
306
- const buffer = await response.buffer();
307
- cache[url] = {
308
- status: response.status(),
309
- headers: response.headers(),
310
- body: buffer,
311
- };
312
- }
313
- }
314
- catch {
315
- // ignore errors, usually means that buffer is empty or broken connection
316
- }
317
- });
318
- }
319
243
  /**
320
244
  * Compiles a Puppeteer script into an async function that may be executed at any time
321
245
  * by providing it with the following object:
@@ -545,48 +469,4 @@ export async function saveSnapshot(page, options = {}) {
545
469
  throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
546
470
  }
547
471
  }
548
- let idcacPlaywright = null;
549
- async function getIdcacPlaywright() {
550
- if (idcacPlaywright)
551
- return idcacPlaywright;
552
- try {
553
- idcacPlaywright = await import('idcac-playwright');
554
- }
555
- catch (error) {
556
- getLog().warning(`Failed to import 'idcac-playwright'.
557
-
558
- We recently made idcac-playwright an optional dependency due to licensing issues.
559
- To use this feature, please install it manually by running
560
-
561
- npm install idcac-playwright
562
-
563
- Original error message follows:
564
-
565
- ${error.message}
566
- `);
567
- }
568
- return idcacPlaywright;
569
- }
570
- export async function closeCookieModals(page) {
571
- parseArgument(page, validators.browserPage);
572
- const idcac = await getIdcacPlaywright();
573
- if (idcac?.getInjectableScript()) {
574
- await page.evaluate(idcac.getInjectableScript());
575
- }
576
- }
577
472
  export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
578
- /** @internal */
579
- export const puppeteerUtils = {
580
- injectFile,
581
- injectJQuery,
582
- enqueueLinksByClickingElements,
583
- blockRequests,
584
- compileScript,
585
- gotoExtended,
586
- addInterceptRequestHandler,
587
- removeInterceptRequestHandler,
588
- infiniteScroll,
589
- saveSnapshot,
590
- parseWithCheerio,
591
- closeCookieModals,
592
- };
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/puppeteer",
3
- "version": "4.0.0-rc.0",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -47,26 +47,23 @@
47
47
  "access": "public"
48
48
  },
49
49
  "dependencies": {
50
- "@apify/datastructures": "^2.0.3",
51
- "@crawlee/browser": "4.0.0-rc.0",
52
- "@crawlee/browser-pool": "4.0.0-rc.0",
53
- "@crawlee/core": "4.0.0-rc.0",
54
- "@crawlee/types": "4.0.0-rc.0",
55
- "@crawlee/utils": "4.0.0-rc.0",
56
- "cheerio": "^1.0.0",
57
- "devtools-protocol": "*",
50
+ "@apify/datastructures": "^3.0.1",
51
+ "@crawlee/browser": "4.0.0-rc.2",
52
+ "@crawlee/browser-pool": "4.0.0-rc.2",
53
+ "@crawlee/core": "4.0.0-rc.2",
54
+ "@crawlee/types": "4.0.0-rc.2",
55
+ "@crawlee/utils": "4.0.0-rc.2",
56
+ "cheerio": "^1.2.0",
57
+ "devtools-protocol": "^0.0.1734725",
58
+ "domhandler": "^6.0.1",
58
59
  "jquery": "^3.7.1",
59
- "tslib": "^2.8.1",
60
- "zod": "^4.4.3"
60
+ "tslib": "^2.9.1",
61
+ "zod": "^4.6.7"
61
62
  },
62
63
  "peerDependencies": {
63
- "idcac-playwright": "^0.2.0",
64
64
  "puppeteer": "*"
65
65
  },
66
66
  "peerDependenciesMeta": {
67
- "idcac-playwright": {
68
- "optional": true
69
- },
70
67
  "puppeteer": {
71
68
  "optional": true
72
69
  }
@@ -78,5 +75,5 @@
78
75
  }
79
76
  }
80
77
  },
81
- "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
78
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
82
79
  }