@crawlee/jsdom 3.18.1-beta.8 → 3.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -193,6 +193,7 @@ class JSDOMCrawler extends http_1.HttpCrawler {
|
|
|
193
193
|
this.log.debug(e.message);
|
|
194
194
|
}
|
|
195
195
|
}
|
|
196
|
+
const originalEnqueueLinks = crawlingContext.enqueueLinks;
|
|
196
197
|
return {
|
|
197
198
|
window,
|
|
198
199
|
get body() {
|
|
@@ -203,13 +204,16 @@ class JSDOMCrawler extends http_1.HttpCrawler {
|
|
|
203
204
|
},
|
|
204
205
|
enqueueLinks: async (enqueueOptions) => {
|
|
205
206
|
return domCrawlerEnqueueLinks({
|
|
206
|
-
|
|
207
|
+
// `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;
|
|
208
|
+
// pre-clamping it here would make the crawler log the internal limit as a user-provided one
|
|
209
|
+
options: enqueueOptions,
|
|
207
210
|
window,
|
|
208
211
|
requestQueue: await this.getRequestQueue(),
|
|
209
212
|
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
|
|
210
213
|
onSkippedRequest: this.handleSkippedRequest,
|
|
211
214
|
originalRequestUrl: crawlingContext.request.url,
|
|
212
215
|
finalRequestUrl: crawlingContext.request.loadedUrl,
|
|
216
|
+
enqueueLinks: originalEnqueueLinks,
|
|
213
217
|
});
|
|
214
218
|
},
|
|
215
219
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/jsdom",
|
|
3
|
-
"version": "3.18.1
|
|
3
|
+
"version": "3.18.1",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,9 +55,9 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.4.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/http": "3.18.1
|
|
59
|
-
"@crawlee/types": "3.18.1
|
|
60
|
-
"@crawlee/utils": "3.18.1
|
|
58
|
+
"@crawlee/http": "3.18.1",
|
|
59
|
+
"@crawlee/types": "3.18.1",
|
|
60
|
+
"@crawlee/utils": "3.18.1",
|
|
61
61
|
"@types/jsdom": "^21.0.0",
|
|
62
62
|
"cheerio": "1.0.0-rc.12",
|
|
63
63
|
"jsdom": "^26.0.0",
|
|
@@ -71,5 +71,5 @@
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
},
|
|
74
|
-
"gitHead": "
|
|
74
|
+
"gitHead": "5dbdf0e0095235cf9128dbf0a34613f4cca5ce74"
|
|
75
75
|
}
|