@crawlee/puppeteer 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts CHANGED
@@ -2,7 +2,7 @@ export * from '@crawlee/browser';
2
2
  export * from './internals/puppeteer-browser-pool.js';
3
3
  export * from './internals/puppeteer-crawler.js';
4
4
  export * from './internals/puppeteer-launcher.js';
5
- export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
6
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
7
- export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer_utils.js';
5
+ export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
6
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
7
+ export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
8
8
  export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
package/index.js CHANGED
@@ -2,4 +2,4 @@ export * from '@crawlee/browser';
2
2
  export * from './internals/puppeteer-browser-pool.js';
3
3
  export * from './internals/puppeteer-crawler.js';
4
4
  export * from './internals/puppeteer-launcher.js';
5
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
5
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
@@ -2,7 +2,7 @@ import { URL } from 'node:url';
2
2
  import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
3
3
  import { parseArgument, schemas } from '@crawlee/utils/internal';
4
4
  import { z } from 'zod';
5
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
5
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
6
6
  const STARTING_Z_INDEX = 2147400000;
7
7
  const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
8
8
  const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
@@ -1,5 +1,5 @@
1
- import type { Configuration } from '@crawlee/browser';
2
- import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
1
+ import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
4
  import type { Page } from 'puppeteer';
5
5
  import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
@@ -17,8 +17,8 @@ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'b
17
17
  headless?: boolean | 'new' | 'old';
18
18
  /** Configuration to read the browser defaults from. Defaults to the global configuration. */
19
19
  configuration?: Configuration;
20
- }
21
- export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
20
+ /** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
21
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
22
22
  }
23
23
  /**
24
24
  * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
@@ -42,14 +42,3 @@ export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowser
42
42
  * @category Browser management
43
43
  */
44
44
  export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
45
- /**
46
- * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
47
- * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
48
- *
49
- * A {@link PuppeteerCrawler} accepts the same connection details directly via
50
- * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
51
- * tune the wrapping pool, or to share one remote pool between crawlers.
52
- *
53
- * @category Browser management
54
- */
55
- export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
@@ -24,20 +24,6 @@ export function puppeteerBrowserPool(options = {}) {
24
24
  const { launchContext, headless, configuration, ...poolOptions } = options;
25
25
  return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
26
26
  }
27
- /**
28
- * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
29
- * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
30
- *
31
- * A {@link PuppeteerCrawler} accepts the same connection details directly via
32
- * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
33
- * tune the wrapping pool, or to share one remote pool between crawlers.
34
- *
35
- * @category Browser management
36
- */
37
- export function remotePuppeteerBrowserPool(options) {
38
- const { launchContext, headless, configuration, ...remoteOptions } = options;
39
- return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
40
- }
41
27
  function puppeteerLauncher(launchContext = {}, headless, configuration) {
42
28
  return new PuppeteerLauncher(headless == null
43
29
  ? launchContext
@@ -5,7 +5,7 @@ import type { Dictionary } from '@crawlee/types';
5
5
  import type { HTTPResponse, Page } from 'puppeteer';
6
6
  import { z } from 'zod';
7
7
  import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
8
- import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';
8
+ import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer-utils.js';
9
9
  export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
10
10
  export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
11
11
  }
@@ -2,8 +2,8 @@ import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
2
2
  import { serviceLocator } from '@crawlee/core';
3
3
  import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
4
4
  import { z } from 'zod';
5
- import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';
6
- import * as puppeteerUtils from './utils/puppeteer_utils.js';
5
+ import { puppeteerBrowserPool } from './puppeteer-browser-pool.js';
6
+ import * as puppeteerUtils from './utils/puppeteer-utils.js';
7
7
  /**
8
8
  * Provides a simple framework for parallel crawling of web pages
9
9
  * using headless Chrome with [Puppeteer](https://github.com/puppeteer/puppeteer).
@@ -103,9 +103,7 @@ export class PuppeteerCrawler extends BrowserCrawler {
103
103
  ...browserCrawlerOptions,
104
104
  configuration,
105
105
  proxyConfiguration,
106
- browserPoolBuilder: (remoteBrowser) => remoteBrowser
107
- ? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
108
- : puppeteerBrowserPool({ launchContext, headless, configuration }),
106
+ browserPoolBuilder: (remoteBrowser) => puppeteerBrowserPool({ launchContext, headless, configuration, remoteBrowser }),
109
107
  contextPipelineBuilder: () => this.#buildContextPipeline(),
110
108
  });
111
109
  }
@@ -116,6 +114,21 @@ export class PuppeteerCrawler extends BrowserCrawler {
116
114
  const waitForSelector = async (selector, timeoutMs = 5_000) => {
117
115
  await context.page.waitForSelector(selector, { timeout: timeoutMs });
118
116
  };
117
+ const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
118
+ if (selector) {
119
+ await waitForSelector(selector, timeoutMs);
120
+ }
121
+ return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
122
+ };
123
+ const querySelectorAll = async (selector, timeoutMs) => {
124
+ const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
125
+ // The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
126
+ // or an element under a shadow root/iframe the snapshot was told to ignore.
127
+ if (matches.length === 0) {
128
+ throw new Error(`Selector '${selector}' not found in the page snapshot.`);
129
+ }
130
+ return matches;
131
+ };
119
132
  return {
120
133
  injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
121
134
  injectJQuery: async () => {
@@ -127,12 +140,9 @@ export class PuppeteerCrawler extends BrowserCrawler {
127
140
  await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
128
141
  },
129
142
  waitForSelector,
130
- parseWithCheerio: async (selector, timeoutMs = 5_000) => {
131
- if (selector) {
132
- await waitForSelector(selector, timeoutMs);
133
- }
134
- return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
135
- },
143
+ parseWithCheerio,
144
+ querySelectorAll,
145
+ querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
136
146
  enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
137
147
  page: context.page,
138
148
  requestManager: this.requestManager,
@@ -1,5 +1,5 @@
1
1
  import { BrowserLauncher, Configuration } from '@crawlee/browser';
2
- import { PuppeteerPlugin } from '@crawlee/browser-pool';
2
+ import { PuppeteerPlugin, RemotePuppeteerPlugin } from '@crawlee/browser-pool';
3
3
  import { parseArgument, schemas } from '@crawlee/utils/internal';
4
4
  import { z } from 'zod';
5
5
  /**
@@ -28,6 +28,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
28
28
  }, configuration);
29
29
  this.configuration = configuration;
30
30
  this.Plugin = PuppeteerPlugin;
31
+ this.RemotePlugin = RemotePuppeteerPlugin;
31
32
  }
32
33
  getDefaultHeadlessOption() {
33
34
  const headless = super.getDefaultHeadlessOption();
@@ -63,7 +63,10 @@ async function handleRequest(request, interceptRequestHandlers) {
63
63
  const originalContinue = request.continue.bind(request);
64
64
  request.continue = async (overrides = {}) => {
65
65
  wasContinued = true;
66
- const headers = browserifyHeaders({ ...accumulatedOverrides.headers, ...overrides.headers });
66
+ const headers = browserifyHeaders({
67
+ ...accumulatedOverrides.headers,
68
+ ...(overrides.headers instanceof Headers ? Object.fromEntries(overrides.headers) : overrides.headers),
69
+ });
67
70
  Object.assign(accumulatedOverrides, overrides, { headers });
68
71
  };
69
72
  const { abort, respond } = request;
@@ -20,13 +20,14 @@
20
20
  import type { Request } from '@crawlee/browser';
21
21
  import { Configuration } from '@crawlee/browser';
22
22
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
23
- import type { CheerioAPI } from 'cheerio';
23
+ import type { Cheerio, CheerioAPI } from 'cheerio';
24
+ import type { AnyNode } from 'domhandler';
24
25
  // @ts-ignore optional peer dependency or compatibility with es2022
25
26
  import type { HTTPResponse, Page } from 'puppeteer';
26
27
  import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
27
28
  import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
28
- import type { InterceptHandler } from './puppeteer_request_interception.js';
29
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
29
+ import type { InterceptHandler } from './puppeteer-request-interception.js';
30
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
30
31
  export interface DirectNavigationOptions {
31
32
  /**
32
33
  * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
@@ -336,6 +337,15 @@ export interface PuppeteerContextUtils {
336
337
  * ```
337
338
  */
338
339
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
340
+ /**
341
+ * Like {@link PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
342
+ * as a Cheerio object. Throws when nothing matches.
343
+ */
344
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
345
+ /**
346
+ * Like {@link PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
347
+ */
348
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
339
349
  /**
340
350
  * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
341
351
  * When provided with the `selector` argument, it waits for it to be available first.
@@ -26,7 +26,7 @@ import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/intern
26
26
  import { z } from 'zod';
27
27
  import { LruCache } from '@apify/datastructures';
28
28
  import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
29
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
29
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer-request-interception.js';
30
30
  const require = createRequire(import.meta.url);
31
31
  const jqueryPath = require.resolve('jquery');
32
32
  const MAX_INJECT_FILE_CACHE_SIZE = 10;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/puppeteer",
3
- "version": "4.0.0-rc.1",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,16 +48,17 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/datastructures": "^3.0.1",
51
- "@crawlee/browser": "4.0.0-rc.1",
52
- "@crawlee/browser-pool": "4.0.0-rc.1",
53
- "@crawlee/core": "4.0.0-rc.1",
54
- "@crawlee/types": "4.0.0-rc.1",
55
- "@crawlee/utils": "4.0.0-rc.1",
56
- "cheerio": "^1.0.0",
57
- "devtools-protocol": "*",
51
+ "@crawlee/browser": "4.0.0-rc.2",
52
+ "@crawlee/browser-pool": "4.0.0-rc.2",
53
+ "@crawlee/core": "4.0.0-rc.2",
54
+ "@crawlee/types": "4.0.0-rc.2",
55
+ "@crawlee/utils": "4.0.0-rc.2",
56
+ "cheerio": "^1.2.0",
57
+ "devtools-protocol": "^0.0.1734725",
58
+ "domhandler": "^6.0.1",
58
59
  "jquery": "^3.7.1",
59
- "tslib": "^2.8.1",
60
- "zod": "^4.5.4"
60
+ "tslib": "^2.9.1",
61
+ "zod": "^4.6.7"
61
62
  },
62
63
  "peerDependencies": {
63
64
  "puppeteer": "*"
@@ -74,5 +75,5 @@
74
75
  }
75
76
  }
76
77
  },
77
- "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
78
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
78
79
  }