@crawlee/jsdom 4.0.0-beta.120 → 4.0.0-beta.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
|
|
2
2
|
import { HttpCrawler } from '@crawlee/http';
|
|
3
|
-
import type { Dictionary } from '@crawlee/types';
|
|
3
|
+
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
4
4
|
import { type CheerioRoot } from '@crawlee/utils/internal';
|
|
5
5
|
import { type RobotsTxtFile } from '@crawlee/utils';
|
|
6
6
|
import type { DOMWindow } from 'jsdom';
|
|
@@ -54,6 +54,10 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
54
54
|
* ```
|
|
55
55
|
*/
|
|
56
56
|
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
|
57
|
+
/**
|
|
58
|
+
* Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
|
|
59
|
+
*/
|
|
60
|
+
enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
57
61
|
}
|
|
58
62
|
export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
59
63
|
JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
|
|
@@ -162,7 +166,7 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
|
|
|
162
166
|
readonly body: string;
|
|
163
167
|
readonly document: Document;
|
|
164
168
|
} & {
|
|
165
|
-
enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<
|
|
169
|
+
enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
|
|
166
170
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
167
171
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
168
172
|
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
|
|
@@ -215,7 +215,7 @@ export class JSDOMCrawler extends HttpCrawler {
|
|
|
215
215
|
async addHelpers(crawlingContext) {
|
|
216
216
|
return {
|
|
217
217
|
enqueueLinks: async (enqueueOptions) => {
|
|
218
|
-
return domCrawlerEnqueueLinks({
|
|
218
|
+
return (await domCrawlerEnqueueLinks({
|
|
219
219
|
options: {
|
|
220
220
|
...enqueueOptions,
|
|
221
221
|
limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
|
|
@@ -226,7 +226,7 @@ export class JSDOMCrawler extends HttpCrawler {
|
|
|
226
226
|
onSkippedRequest: this.handleSkippedRequest,
|
|
227
227
|
originalRequestUrl: crawlingContext.request.url,
|
|
228
228
|
finalRequestUrl: crawlingContext.request.loadedUrl,
|
|
229
|
-
});
|
|
229
|
+
})); // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
|
|
230
230
|
},
|
|
231
231
|
async waitForSelector(selector, timeoutMs = 5_000) {
|
|
232
232
|
const cheerio = await import('cheerio');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/jsdom",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.121",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,9 +49,9 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.4.4",
|
|
51
51
|
"@apify/utilities": "^2.7.10",
|
|
52
|
-
"@crawlee/http": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/http": "4.0.0-beta.121",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.121",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.121",
|
|
55
55
|
"@types/jsdom": "^21.1.7",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"jsdom": "^26.1.0",
|
|
@@ -65,5 +65,5 @@
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
|
-
"gitHead": "
|
|
68
|
+
"gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
|
|
69
69
|
}
|