@crawlee/puppeteer 4.0.0-beta.231 → 4.0.0-beta.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -114,6 +114,21 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
114
114
|
const waitForSelector = async (selector, timeoutMs = 5_000) => {
|
|
115
115
|
await context.page.waitForSelector(selector, { timeout: timeoutMs });
|
|
116
116
|
};
|
|
117
|
+
const parseWithCheerio = async (selector, timeoutMs = 5_000) => {
|
|
118
|
+
if (selector) {
|
|
119
|
+
await waitForSelector(selector, timeoutMs);
|
|
120
|
+
}
|
|
121
|
+
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
122
|
+
};
|
|
123
|
+
const querySelectorAll = async (selector, timeoutMs) => {
|
|
124
|
+
const matches = (await parseWithCheerio(selector, timeoutMs))(selector);
|
|
125
|
+
// The browser located it but the Cheerio snapshot does not have it - a Puppeteer-only selector,
|
|
126
|
+
// or an element under a shadow root/iframe the snapshot was told to ignore.
|
|
127
|
+
if (matches.length === 0) {
|
|
128
|
+
throw new Error(`Selector '${selector}' not found in the page snapshot.`);
|
|
129
|
+
}
|
|
130
|
+
return matches;
|
|
131
|
+
};
|
|
117
132
|
return {
|
|
118
133
|
injectFile: async (filePath, options) => puppeteerUtils.injectFile(context.page, filePath, options),
|
|
119
134
|
injectJQuery: async () => {
|
|
@@ -125,12 +140,9 @@ export class PuppeteerCrawler extends BrowserCrawler {
|
|
|
125
140
|
await puppeteerUtils.injectJQuery(context.page, { surviveNavigations: false });
|
|
126
141
|
},
|
|
127
142
|
waitForSelector,
|
|
128
|
-
parseWithCheerio
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
return puppeteerUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
|
|
133
|
-
},
|
|
143
|
+
parseWithCheerio,
|
|
144
|
+
querySelectorAll,
|
|
145
|
+
querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
|
|
134
146
|
enqueueLinksByClickingElements: async (options) => puppeteerUtils.enqueueLinksByClickingElements({
|
|
135
147
|
page: context.page,
|
|
136
148
|
requestManager: this.requestManager,
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
import type { Request } from '@crawlee/browser';
|
|
21
21
|
import { Configuration } from '@crawlee/browser';
|
|
22
22
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
23
|
-
import type { CheerioAPI } from 'cheerio';
|
|
23
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
24
|
+
import type { AnyNode } from 'domhandler';
|
|
24
25
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
25
26
|
import type { HTTPResponse, Page } from 'puppeteer';
|
|
26
27
|
import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
|
|
@@ -336,6 +337,15 @@ export interface PuppeteerContextUtils {
|
|
|
336
337
|
* ```
|
|
337
338
|
*/
|
|
338
339
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
340
|
+
/**
|
|
341
|
+
* Like {@link PuppeteerContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
|
|
342
|
+
* as a Cheerio object. Throws when nothing matches.
|
|
343
|
+
*/
|
|
344
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
345
|
+
/**
|
|
346
|
+
* Like {@link PuppeteerContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
|
|
347
|
+
*/
|
|
348
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
339
349
|
/**
|
|
340
350
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
341
351
|
* When provided with the `selector` argument, it waits for it to be available first.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/puppeteer",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.232",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,13 +48,14 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/datastructures": "^3.0.1",
|
|
51
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/types": "4.0.0-beta.
|
|
55
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/browser": "4.0.0-beta.232",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.232",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.232",
|
|
54
|
+
"@crawlee/types": "4.0.0-beta.232",
|
|
55
|
+
"@crawlee/utils": "4.0.0-beta.232",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"devtools-protocol": "*",
|
|
58
|
+
"domhandler": "^6.0.0",
|
|
58
59
|
"jquery": "^3.7.1",
|
|
59
60
|
"tslib": "^2.8.1",
|
|
60
61
|
"zod": "^4.5.4"
|
|
@@ -74,5 +75,5 @@
|
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
},
|
|
77
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "5eab8f877242d46d7cae13b640d8e1adbf194911"
|
|
78
79
|
}
|