@crawlee/jsdom 4.0.0-beta.17 → 4.0.0-beta.170
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/index.d.ts +1 -1
- package/index.js +0 -1
- package/internals/jsdom-crawler.d.ts +208 -150
- package/internals/jsdom-crawler.js +52 -218
- package/internals/jsdom-parser.d.ts +29 -0
- package/internals/jsdom-parser.js +93 -0
- package/package.json +10 -10
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/jsdom-crawler.d.ts.map +0 -1
- package/internals/jsdom-crawler.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/index.d.ts
CHANGED
package/index.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import {
|
|
1
|
+
import type { CrawlingContext, DOMCrawlingContext, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
|
|
2
|
+
import { DOMCrawler } from '@crawlee/http';
|
|
3
3
|
import type { Dictionary } from '@crawlee/types';
|
|
4
|
-
import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
|
|
5
|
-
import type { DOMWindow } from 'jsdom';
|
|
6
4
|
import { VirtualConsole } from 'jsdom';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import type { JSDOMParseResult } from './jsdom-parser.js';
|
|
7
7
|
export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
8
|
-
JSONData extends Dictionary = any
|
|
8
|
+
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
9
|
+
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
|
|
9
10
|
export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
10
|
-
JSONData extends Dictionary = any
|
|
11
|
+
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
12
|
+
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
11
13
|
/**
|
|
12
14
|
* Download and run scripts.
|
|
13
15
|
*/
|
|
@@ -20,131 +22,208 @@ JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingConte
|
|
|
20
22
|
export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
21
23
|
JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
|
|
22
24
|
export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
23
|
-
JSONData extends Dictionary = any> extends
|
|
24
|
-
window: DOMWindow;
|
|
25
|
-
document: Document;
|
|
26
|
-
body: string;
|
|
27
|
-
/**
|
|
28
|
-
* Wait for an element matching the selector to appear.
|
|
29
|
-
* Timeout defaults to 5s.
|
|
30
|
-
*
|
|
31
|
-
* **Example usage:**
|
|
32
|
-
* ```ts
|
|
33
|
-
* async requestHandler({ waitForSelector, parseWithCheerio }) {
|
|
34
|
-
* await waitForSelector('article h1');
|
|
35
|
-
* const $ = await parseWithCheerio();
|
|
36
|
-
* const title = $('title').text();
|
|
37
|
-
* });
|
|
38
|
-
* ```
|
|
39
|
-
*/
|
|
40
|
-
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
41
|
-
/**
|
|
42
|
-
* Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
43
|
-
* When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
|
|
44
|
-
*
|
|
45
|
-
* **Example usage:**
|
|
46
|
-
* ```javascript
|
|
47
|
-
* async requestHandler({ parseWithCheerio }) {
|
|
48
|
-
* const $ = await parseWithCheerio();
|
|
49
|
-
* const title = $('title').text();
|
|
50
|
-
* });
|
|
51
|
-
* ```
|
|
52
|
-
*/
|
|
53
|
-
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
|
25
|
+
JSONData extends Dictionary = any> extends DOMCrawlingContext<JSDOMParseResult, UserData, JSONData> {
|
|
54
26
|
}
|
|
55
27
|
export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
56
28
|
JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
|
|
57
|
-
|
|
29
|
+
/**
|
|
30
|
+
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
|
|
31
|
+
* [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
|
|
32
|
+
* The URLs to crawl are fed either from a static list of URLs
|
|
33
|
+
* or from a dynamic queue of URLs enabling recursive crawling of websites.
|
|
34
|
+
*
|
|
35
|
+
* Since `JSDOMCrawler` uses raw HTTP requests to download web pages,
|
|
36
|
+
* it is very fast and efficient on data bandwidth. However, if the target website requires JavaScript
|
|
37
|
+
* to display the content, you might need to use {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead,
|
|
38
|
+
* because it loads the pages using full-featured headless Chrome browser.
|
|
39
|
+
*
|
|
40
|
+
* Alternatively, you can use {@link JSDOMCrawlerOptions.runScripts} to run website scripts in Node.
|
|
41
|
+
* JSDOM does not implement all the standards, so websites can break.
|
|
42
|
+
*
|
|
43
|
+
* **Limitation**:
|
|
44
|
+
* This crawler does not support proxies and cookies yet (each open starts with empty cookie store), and the user agent is always set to `Chrome`.
|
|
45
|
+
*
|
|
46
|
+
* `JSDOMCrawler` downloads each URL using a plain HTTP request,
|
|
47
|
+
* parses the HTML content using [JSDOM](https://www.npmjs.com/package/jsdom)
|
|
48
|
+
* and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
|
|
49
|
+
* using the `window` object.
|
|
50
|
+
*
|
|
51
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
52
|
+
* {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
|
|
53
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
54
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
55
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
56
|
+
* result as `requestManager`.
|
|
57
|
+
*
|
|
58
|
+
* > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
|
|
59
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
60
|
+
*
|
|
61
|
+
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
62
|
+
*
|
|
63
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
64
|
+
*
|
|
65
|
+
* ```
|
|
66
|
+
* preNavigationHooks: [
|
|
67
|
+
* (crawlingContext) => {
|
|
68
|
+
* // ...
|
|
69
|
+
* },
|
|
70
|
+
* ]
|
|
71
|
+
* ```
|
|
72
|
+
*
|
|
73
|
+
* By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
74
|
+
* and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
|
|
75
|
+
* and skips pages with other content types. If you want the crawler to process other content types,
|
|
76
|
+
* use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
|
|
77
|
+
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
78
|
+
* For more details, see {@link JSDOMCrawlerOptions.requestHandler}.
|
|
79
|
+
*
|
|
80
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
81
|
+
* {@link ConcurrencySystem}.
|
|
82
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
83
|
+
* `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
84
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
85
|
+
*
|
|
86
|
+
* **Example usage:**
|
|
87
|
+
*
|
|
88
|
+
* ```javascript
|
|
89
|
+
* const crawler = new JSDOMCrawler({
|
|
90
|
+
* async requestHandler({ request, window }) {
|
|
91
|
+
* await Dataset.pushData({
|
|
92
|
+
* url: request.url,
|
|
93
|
+
* title: window.document.title,
|
|
94
|
+
* });
|
|
95
|
+
* },
|
|
96
|
+
* });
|
|
97
|
+
*
|
|
98
|
+
* await crawler.run([
|
|
99
|
+
* 'http://crawlee.dev',
|
|
100
|
+
* ]);
|
|
101
|
+
* ```
|
|
102
|
+
* @category Crawlers
|
|
103
|
+
*/
|
|
104
|
+
export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends DOMCrawler<JSDOMParseResult, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
105
|
+
#private;
|
|
106
|
+
/**
|
|
107
|
+
* @internal
|
|
108
|
+
*/
|
|
58
109
|
protected static optionsShape: {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
116
|
-
proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
117
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
118
|
-
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
119
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
120
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
121
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
122
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
123
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
124
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
125
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
126
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
127
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
128
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
129
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
130
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
131
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
132
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
133
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
134
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
135
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
136
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
137
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
138
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
139
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
140
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
141
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
142
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
110
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
111
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
112
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
113
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
114
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
115
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
116
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
117
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
118
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
119
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
120
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
121
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
122
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
123
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
124
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
125
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
126
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
127
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
128
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
129
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
130
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
131
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
132
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
133
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
134
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
135
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
136
|
+
deferred: "deferred";
|
|
137
|
+
writeThrough: "writeThrough";
|
|
138
|
+
}>>;
|
|
139
|
+
}, z.core.$strict>]>>;
|
|
140
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
141
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
142
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
143
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
144
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
|
|
145
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
146
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
147
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
|
|
148
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
149
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
150
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
151
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
152
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
153
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
154
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
155
|
+
id: z.ZodOptional<z.ZodString>;
|
|
156
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
157
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
158
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
159
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
160
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
161
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
162
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
163
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
164
|
+
runScripts: z.ZodOptional<z.ZodBoolean>;
|
|
165
|
+
hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
|
|
143
166
|
};
|
|
144
|
-
|
|
145
|
-
protected
|
|
146
|
-
|
|
147
|
-
|
|
167
|
+
/** @internal */
|
|
168
|
+
protected static optionsSchema: z.ZodObject<{
|
|
169
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
170
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
171
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
172
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
173
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
174
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
175
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
176
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
177
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
178
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
179
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
180
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
181
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
182
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
183
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
184
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
185
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
186
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
187
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
188
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
189
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
190
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
191
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
192
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
193
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
194
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
195
|
+
deferred: "deferred";
|
|
196
|
+
writeThrough: "writeThrough";
|
|
197
|
+
}>>;
|
|
198
|
+
}, z.core.$strict>]>>;
|
|
199
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
200
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
201
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
202
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
203
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
|
|
204
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
205
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
206
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
|
|
207
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
208
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
209
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
210
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
211
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
212
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
213
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
214
|
+
id: z.ZodOptional<z.ZodString>;
|
|
215
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
216
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
217
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
218
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
219
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
220
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
221
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
222
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
223
|
+
runScripts: z.ZodOptional<z.ZodBoolean>;
|
|
224
|
+
hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
|
|
225
|
+
}, z.core.$strict>;
|
|
226
|
+
constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
|
|
148
227
|
/**
|
|
149
228
|
* Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
|
|
150
229
|
*
|
|
@@ -161,27 +240,7 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
|
|
|
161
240
|
*/
|
|
162
241
|
getVirtualConsole(): VirtualConsole;
|
|
163
242
|
private readonly jsdomErrorHandler;
|
|
164
|
-
private parseContent;
|
|
165
|
-
private addHelpers;
|
|
166
|
-
}
|
|
167
|
-
interface EnqueueLinksInternalOptions {
|
|
168
|
-
options?: EnqueueLinksOptions;
|
|
169
|
-
window: DOMWindow | null;
|
|
170
|
-
requestQueue: RequestProvider;
|
|
171
|
-
robotsTxtFile?: RobotsTxtFile;
|
|
172
|
-
onSkippedRequest?: SkippedRequestCallback;
|
|
173
|
-
originalRequestUrl: string;
|
|
174
|
-
finalRequestUrl?: string;
|
|
175
243
|
}
|
|
176
|
-
interface BoundEnqueueLinksInternalOptions {
|
|
177
|
-
enqueueLinks: BasicCrawlingContext['enqueueLinks'];
|
|
178
|
-
options?: EnqueueLinksOptions;
|
|
179
|
-
window: DOMWindow | null;
|
|
180
|
-
originalRequestUrl: string;
|
|
181
|
-
finalRequestUrl?: string;
|
|
182
|
-
}
|
|
183
|
-
/** @internal */
|
|
184
|
-
export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
|
185
244
|
/**
|
|
186
245
|
* Creates new {@link Router} instance that works based on request labels.
|
|
187
246
|
* This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
|
|
@@ -206,7 +265,6 @@ export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOpti
|
|
|
206
265
|
* await crawler.run();
|
|
207
266
|
* ```
|
|
208
267
|
*/
|
|
209
|
-
|
|
210
|
-
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData
|
|
211
|
-
export
|
|
212
|
-
//# sourceMappingURL=jsdom-crawler.d.ts.map
|
|
268
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
269
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
270
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
6
|
-
import { addTimeoutToPromise } from '@apify/timeout';
|
|
1
|
+
import { DOMCrawler, HttpCrawler, Router } from '@crawlee/http';
|
|
2
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
3
|
+
import { VirtualConsole } from 'jsdom';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { jsdomParser } from './jsdom-parser.js';
|
|
7
6
|
/**
|
|
8
7
|
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
|
|
9
8
|
* [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
|
|
@@ -26,38 +25,40 @@ import { addTimeoutToPromise } from '@apify/timeout';
|
|
|
26
25
|
* and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
|
|
27
26
|
* using the `window` object.
|
|
28
27
|
*
|
|
29
|
-
* The source URLs are represented using {@link Request} objects that are fed from
|
|
30
|
-
* {@link
|
|
31
|
-
*
|
|
28
|
+
* The source URLs are represented using {@link Request} objects that are fed from the
|
|
29
|
+
* {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
|
|
30
|
+
* constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
|
|
31
|
+
* as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
|
|
32
|
+
* {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
|
|
33
|
+
* result as `requestManager`.
|
|
32
34
|
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
35
|
+
* > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
|
|
36
|
+
* > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
|
|
36
37
|
*
|
|
37
38
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
38
39
|
*
|
|
39
|
-
* We can use the `preNavigationHooks` to adjust
|
|
40
|
+
* We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
|
|
40
41
|
*
|
|
41
42
|
* ```
|
|
42
43
|
* preNavigationHooks: [
|
|
43
|
-
* (crawlingContext
|
|
44
|
+
* (crawlingContext) => {
|
|
44
45
|
* // ...
|
|
45
46
|
* },
|
|
46
47
|
* ]
|
|
47
48
|
* ```
|
|
48
49
|
*
|
|
49
|
-
* By default, `JSDOMCrawler` only processes web pages with the `text/html`
|
|
50
|
-
* and `application/
|
|
50
|
+
* By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
|
|
51
|
+
* and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
|
|
51
52
|
* and skips pages with other content types. If you want the crawler to process other content types,
|
|
52
53
|
* use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
|
|
53
54
|
* Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
|
|
54
55
|
* For more details, see {@link JSDOMCrawlerOptions.requestHandler}.
|
|
55
56
|
*
|
|
56
|
-
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
* {@link
|
|
57
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
|
|
58
|
+
* {@link ConcurrencySystem}.
|
|
59
|
+
* Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
|
|
60
|
+
* `JSDOMCrawler` constructor, or, for finer control, by injecting a pre-configured
|
|
61
|
+
* {@link ConcurrencySystem|`concurrencySystem`}.
|
|
61
62
|
*
|
|
62
63
|
* **Example usage:**
|
|
63
64
|
*
|
|
@@ -77,36 +78,31 @@ import { addTimeoutToPromise } from '@apify/timeout';
|
|
|
77
78
|
* ```
|
|
78
79
|
* @category Crawlers
|
|
79
80
|
*/
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
});
|
|
85
|
-
export class JSDOMCrawler extends HttpCrawler {
|
|
81
|
+
export class JSDOMCrawler extends DOMCrawler {
|
|
82
|
+
/**
|
|
83
|
+
* @internal
|
|
84
|
+
*/
|
|
86
85
|
static optionsShape = {
|
|
87
86
|
...HttpCrawler.optionsShape,
|
|
88
|
-
runScripts:
|
|
89
|
-
hideInternalConsole:
|
|
87
|
+
runScripts: z.boolean().optional(),
|
|
88
|
+
hideInternalConsole: z.boolean().optional(),
|
|
90
89
|
};
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
90
|
+
/** @internal */
|
|
91
|
+
static optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);
|
|
92
|
+
#hideInternalConsole;
|
|
93
|
+
#virtualConsole = null;
|
|
94
|
+
constructor(options = {}) {
|
|
95
|
+
const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');
|
|
96
96
|
super({
|
|
97
97
|
...httpOptions,
|
|
98
|
-
contextPipelineBuilder
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
.compose({ action: async (context) => await this.addHelpers(context) }),
|
|
107
|
-
}, config);
|
|
108
|
-
this.runScripts = runScripts;
|
|
109
|
-
this.hideInternalConsole = hideInternalConsole;
|
|
98
|
+
contextPipelineBuilder,
|
|
99
|
+
parser: jsdomParser({
|
|
100
|
+
runScripts,
|
|
101
|
+
virtualConsole: () => this.getVirtualConsole(),
|
|
102
|
+
log: () => this.log,
|
|
103
|
+
}),
|
|
104
|
+
});
|
|
105
|
+
this.#hideInternalConsole = hideInternalConsole;
|
|
110
106
|
}
|
|
111
107
|
/**
|
|
112
108
|
* Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
|
|
@@ -123,180 +119,18 @@ export class JSDOMCrawler extends HttpCrawler {
|
|
|
123
119
|
* ```
|
|
124
120
|
*/
|
|
125
121
|
getVirtualConsole() {
|
|
126
|
-
if (this
|
|
127
|
-
return this
|
|
122
|
+
if (this.#virtualConsole) {
|
|
123
|
+
return this.#virtualConsole;
|
|
128
124
|
}
|
|
129
|
-
this
|
|
130
|
-
if (!this
|
|
131
|
-
this
|
|
125
|
+
this.#virtualConsole = new VirtualConsole();
|
|
126
|
+
if (!this.#hideInternalConsole) {
|
|
127
|
+
this.#virtualConsole.sendTo(console, { omitJSDOMErrors: true });
|
|
132
128
|
}
|
|
133
|
-
this
|
|
134
|
-
return this
|
|
135
|
-
}
|
|
136
|
-
jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', error);
|
|
137
|
-
async parseContent(crawlingContext) {
|
|
138
|
-
const isXml = crawlingContext.contentType.type.includes('xml');
|
|
139
|
-
// TODO handle non-string
|
|
140
|
-
const { window } = new JSDOM(crawlingContext.body.toString(), {
|
|
141
|
-
url: crawlingContext.response.url,
|
|
142
|
-
contentType: isXml ? 'text/xml' : 'text/html',
|
|
143
|
-
runScripts: this.runScripts ? 'dangerously' : undefined,
|
|
144
|
-
resources,
|
|
145
|
-
virtualConsole: this.getVirtualConsole(),
|
|
146
|
-
pretendToBeVisual: true,
|
|
147
|
-
});
|
|
148
|
-
// add some stubs in place of missing API so processing won't fail
|
|
149
|
-
Object.defineProperty(window, 'matchMedia', {
|
|
150
|
-
writable: true,
|
|
151
|
-
value: (query) => ({
|
|
152
|
-
matches: false,
|
|
153
|
-
media: query,
|
|
154
|
-
onchange: null,
|
|
155
|
-
addListener: () => { },
|
|
156
|
-
removeListener: () => { },
|
|
157
|
-
addEventListener: () => { },
|
|
158
|
-
removeEventListener: () => { },
|
|
159
|
-
dispatchEvent: () => { },
|
|
160
|
-
}),
|
|
161
|
-
});
|
|
162
|
-
window.document.createRange = () => {
|
|
163
|
-
const range = new window.Range();
|
|
164
|
-
range.getBoundingClientRect = () => ({});
|
|
165
|
-
range.getClientRects = () => ({ item: () => null, length: 0 });
|
|
166
|
-
return range;
|
|
167
|
-
};
|
|
168
|
-
if (this.runScripts) {
|
|
169
|
-
try {
|
|
170
|
-
await addTimeoutToPromise(async () => {
|
|
171
|
-
return new Promise((resolve) => {
|
|
172
|
-
window.addEventListener('load', () => {
|
|
173
|
-
resolve();
|
|
174
|
-
}, false);
|
|
175
|
-
}).catch();
|
|
176
|
-
}, 10_000, 'Window.load event not fired after 10 seconds.').catch();
|
|
177
|
-
}
|
|
178
|
-
catch (e) {
|
|
179
|
-
this.log.debug(e.message);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return {
|
|
183
|
-
window,
|
|
184
|
-
get body() {
|
|
185
|
-
return window.document.documentElement.outerHTML;
|
|
186
|
-
},
|
|
187
|
-
get document() {
|
|
188
|
-
return window.document;
|
|
189
|
-
},
|
|
190
|
-
};
|
|
191
|
-
}
|
|
192
|
-
async addHelpers(crawlingContext) {
|
|
193
|
-
return {
|
|
194
|
-
enqueueLinks: async (enqueueOptions) => {
|
|
195
|
-
return domCrawlerEnqueueLinks({
|
|
196
|
-
options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
|
|
197
|
-
window: crawlingContext.window,
|
|
198
|
-
requestQueue: await this.getRequestQueue(),
|
|
199
|
-
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
|
|
200
|
-
onSkippedRequest: this.handleSkippedRequest,
|
|
201
|
-
originalRequestUrl: crawlingContext.request.url,
|
|
202
|
-
finalRequestUrl: crawlingContext.request.loadedUrl,
|
|
203
|
-
});
|
|
204
|
-
},
|
|
205
|
-
async waitForSelector(selector, timeoutMs = 5_000) {
|
|
206
|
-
const $ = cheerio.load(crawlingContext.body);
|
|
207
|
-
if ($(selector).get().length === 0) {
|
|
208
|
-
if (timeoutMs) {
|
|
209
|
-
await sleep(50);
|
|
210
|
-
await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
214
|
-
}
|
|
215
|
-
},
|
|
216
|
-
async parseWithCheerio(selector, _timeoutMs = 5_000) {
|
|
217
|
-
const $ = cheerio.load(crawlingContext.body);
|
|
218
|
-
if (selector && $(selector).get().length === 0) {
|
|
219
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
220
|
-
}
|
|
221
|
-
return $;
|
|
222
|
-
},
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
/** @internal */
|
|
227
|
-
function containsEnqueueLinks(options) {
|
|
228
|
-
return !!options.enqueueLinks;
|
|
229
|
-
}
|
|
230
|
-
/** @internal */
|
|
231
|
-
export async function domCrawlerEnqueueLinks(options) {
|
|
232
|
-
const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
|
|
233
|
-
if (!window) {
|
|
234
|
-
throw new Error('Cannot enqueue links because the JSDOM is not available.');
|
|
235
|
-
}
|
|
236
|
-
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
237
|
-
enqueueStrategy: enqueueLinksOptions?.strategy,
|
|
238
|
-
finalRequestUrl,
|
|
239
|
-
originalRequestUrl,
|
|
240
|
-
userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
|
|
241
|
-
});
|
|
242
|
-
const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
|
|
243
|
-
if (containsEnqueueLinks(options)) {
|
|
244
|
-
return options.enqueueLinks({
|
|
245
|
-
urls,
|
|
246
|
-
baseUrl,
|
|
247
|
-
...enqueueLinksOptions,
|
|
248
|
-
});
|
|
129
|
+
this.#virtualConsole.on('jsdomError', this.jsdomErrorHandler);
|
|
130
|
+
return this.#virtualConsole;
|
|
249
131
|
}
|
|
250
|
-
|
|
251
|
-
requestQueue: options.requestQueue,
|
|
252
|
-
robotsTxtFile: options.robotsTxtFile,
|
|
253
|
-
onSkippedRequest: options.onSkippedRequest,
|
|
254
|
-
urls,
|
|
255
|
-
baseUrl,
|
|
256
|
-
...enqueueLinksOptions,
|
|
257
|
-
});
|
|
132
|
+
jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
|
|
258
133
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
* @ignore
|
|
262
|
-
*/
|
|
263
|
-
function extractUrlsFromWindow(window, selector, baseUrl) {
|
|
264
|
-
return Array.from(window.document.querySelectorAll(selector))
|
|
265
|
-
.map((e) => e.href)
|
|
266
|
-
.filter((href) => href !== undefined && href !== '')
|
|
267
|
-
.map((href) => {
|
|
268
|
-
if (href === undefined) {
|
|
269
|
-
return undefined;
|
|
270
|
-
}
|
|
271
|
-
return tryAbsoluteURL(href, baseUrl);
|
|
272
|
-
})
|
|
273
|
-
.filter((href) => href !== undefined && href !== '');
|
|
274
|
-
}
|
|
275
|
-
/**
|
|
276
|
-
* Creates new {@link Router} instance that works based on request labels.
|
|
277
|
-
* This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
|
|
278
|
-
* Defaults to the {@link JSDOMCrawlingContext}.
|
|
279
|
-
*
|
|
280
|
-
* > Serves as a shortcut for using `Router.create<JSDOMCrawlingContext>()`.
|
|
281
|
-
*
|
|
282
|
-
* ```ts
|
|
283
|
-
* import { JSDOMCrawler, createJSDOMRouter } from 'crawlee';
|
|
284
|
-
*
|
|
285
|
-
* const router = createJSDOMRouter();
|
|
286
|
-
* router.addHandler('label-a', async (ctx) => {
|
|
287
|
-
* ctx.log.info('...');
|
|
288
|
-
* });
|
|
289
|
-
* router.addDefaultHandler(async (ctx) => {
|
|
290
|
-
* ctx.log.info('...');
|
|
291
|
-
* });
|
|
292
|
-
*
|
|
293
|
-
* const crawler = new JSDOMCrawler({
|
|
294
|
-
* requestHandler: router,
|
|
295
|
-
* });
|
|
296
|
-
* await crawler.run();
|
|
297
|
-
* ```
|
|
298
|
-
*/
|
|
299
|
-
export function createJSDOMRouter(routes) {
|
|
300
|
-
return Router.create(routes);
|
|
134
|
+
export function createJSDOMRouter(routesOrSchemas) {
|
|
135
|
+
return Router.create(routesOrSchemas);
|
|
301
136
|
}
|
|
302
|
-
//# sourceMappingURL=jsdom-crawler.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { DOMParser } from '@crawlee/http';
|
|
2
|
+
import type { CrawleeLogger } from '@crawlee/types';
|
|
3
|
+
import type { DOMWindow, VirtualConsole } from 'jsdom';
|
|
4
|
+
export interface JSDOMParseResult {
|
|
5
|
+
window: DOMWindow;
|
|
6
|
+
document: Document;
|
|
7
|
+
body: string;
|
|
8
|
+
}
|
|
9
|
+
export interface JsdomParserOptions {
|
|
10
|
+
/**
|
|
11
|
+
* Download and run scripts.
|
|
12
|
+
*/
|
|
13
|
+
runScripts?: boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Resolves the `VirtualConsole` to hand to JSDOM. Called for every parse, so a crawler can create its console
|
|
16
|
+
* lazily. Defaults to JSDOM's own console handling.
|
|
17
|
+
*/
|
|
18
|
+
virtualConsole?: () => VirtualConsole;
|
|
19
|
+
/**
|
|
20
|
+
* Resolves the logger for JSDOM diagnostics. Called only when there is something to log, so a crawler can hand
|
|
21
|
+
* over its own logger.
|
|
22
|
+
*/
|
|
23
|
+
log?: () => CrawleeLogger;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* A {@link DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a
|
|
27
|
+
* {@link DOMCrawler} to get the crawling context {@link JSDOMCrawler} provides.
|
|
28
|
+
*/
|
|
29
|
+
export declare function jsdomParser(options?: JsdomParserOptions): DOMParser<JSDOMParseResult>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { tryAbsoluteURL } from '@crawlee/utils/internal';
|
|
2
|
+
import { JSDOM, ResourceLoader } from 'jsdom';
|
|
3
|
+
import { addTimeoutToPromise } from '@apify/timeout';
|
|
4
|
+
const resources = new ResourceLoader({
|
|
5
|
+
// Copy from /packages/browser-pool/src/abstract-classes/browser-plugin.ts:17
|
|
6
|
+
// in order not to include the entire package here
|
|
7
|
+
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
|
|
8
|
+
});
|
|
9
|
+
/**
|
|
10
|
+
* A {@link DOMParser} backed by [jsdom](https://www.npmjs.com/package/jsdom). Pass it to a
|
|
11
|
+
* {@link DOMCrawler} to get the crawling context {@link JSDOMCrawler} provides.
|
|
12
|
+
*/
|
|
13
|
+
export function jsdomParser(options = {}) {
|
|
14
|
+
const { runScripts = false, virtualConsole, log } = options;
|
|
15
|
+
return {
|
|
16
|
+
placeholderMembers: { window: true, document: true, body: true },
|
|
17
|
+
mutable: runScripts,
|
|
18
|
+
async parse(context) {
|
|
19
|
+
const isXml = context.contentType.type.includes('xml');
|
|
20
|
+
// TODO handle non-string
|
|
21
|
+
const { window } = new JSDOM(context.body.toString(), {
|
|
22
|
+
url: context.response.url,
|
|
23
|
+
contentType: isXml ? 'text/xml' : 'text/html',
|
|
24
|
+
runScripts: runScripts ? 'dangerously' : undefined,
|
|
25
|
+
resources,
|
|
26
|
+
virtualConsole: virtualConsole?.(),
|
|
27
|
+
pretendToBeVisual: true,
|
|
28
|
+
});
|
|
29
|
+
// add some stubs in place of missing API so processing won't fail
|
|
30
|
+
Object.defineProperty(window, 'matchMedia', {
|
|
31
|
+
writable: true,
|
|
32
|
+
value: (query) => ({
|
|
33
|
+
matches: false,
|
|
34
|
+
media: query,
|
|
35
|
+
onchange: null,
|
|
36
|
+
addListener: () => { },
|
|
37
|
+
removeListener: () => { },
|
|
38
|
+
addEventListener: () => { },
|
|
39
|
+
removeEventListener: () => { },
|
|
40
|
+
dispatchEvent: () => { },
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
window.document.createRange = () => {
|
|
44
|
+
const range = new window.Range();
|
|
45
|
+
range.getBoundingClientRect = () => ({});
|
|
46
|
+
range.getClientRects = () => ({ item: () => null, length: 0 });
|
|
47
|
+
return range;
|
|
48
|
+
};
|
|
49
|
+
if (runScripts) {
|
|
50
|
+
try {
|
|
51
|
+
await addTimeoutToPromise(async () => {
|
|
52
|
+
return new Promise((resolve) => {
|
|
53
|
+
window.addEventListener('load', () => {
|
|
54
|
+
resolve();
|
|
55
|
+
}, false);
|
|
56
|
+
}).catch();
|
|
57
|
+
}, 10_000, 'Window.load event not fired after 10 seconds.').catch();
|
|
58
|
+
}
|
|
59
|
+
catch (e) {
|
|
60
|
+
log?.().debug(e.message);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
window,
|
|
65
|
+
get body() {
|
|
66
|
+
return window.document.documentElement.outerHTML;
|
|
67
|
+
},
|
|
68
|
+
get document() {
|
|
69
|
+
return window.document;
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
},
|
|
73
|
+
extractLinks: ({ window }, selector, baseUrl) => extractUrlsFromWindow(window, selector, baseUrl),
|
|
74
|
+
select: ({ document }, selector) => document.querySelectorAll(selector),
|
|
75
|
+
cleanup: ({ window }) => window.close(),
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Extracts URLs from a given Window object.
|
|
80
|
+
* @ignore
|
|
81
|
+
*/
|
|
82
|
+
function extractUrlsFromWindow(window, selector, baseUrl) {
|
|
83
|
+
return Array.from(window.document.querySelectorAll(selector))
|
|
84
|
+
.map((e) => e.href)
|
|
85
|
+
.filter((href) => href !== undefined && href !== '')
|
|
86
|
+
.map((href) => {
|
|
87
|
+
if (href === undefined) {
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
90
|
+
return tryAbsoluteURL(href, baseUrl);
|
|
91
|
+
})
|
|
92
|
+
.filter((href) => href !== undefined && href !== '');
|
|
93
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/jsdom",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.170",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"homepage": "https://crawlee.dev",
|
|
40
40
|
"scripts": {
|
|
41
|
-
"build": "
|
|
41
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
42
42
|
"clean": "rimraf ./dist",
|
|
43
43
|
"compile": "tsc -p tsconfig.build.json",
|
|
44
44
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -47,16 +47,16 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@apify/timeout": "^0.
|
|
51
|
-
"@apify/utilities": "^
|
|
52
|
-
"@crawlee/http": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
50
|
+
"@apify/timeout": "^1.0.1",
|
|
51
|
+
"@apify/utilities": "^3.0.1",
|
|
52
|
+
"@crawlee/http": "4.0.0-beta.170",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.170",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.170",
|
|
55
55
|
"@types/jsdom": "^21.1.7",
|
|
56
56
|
"cheerio": "^1.0.0",
|
|
57
57
|
"jsdom": "^26.1.0",
|
|
58
|
-
"
|
|
59
|
-
"
|
|
58
|
+
"tslib": "^2.8.1",
|
|
59
|
+
"zod": "^4.5.4"
|
|
60
60
|
},
|
|
61
61
|
"lerna": {
|
|
62
62
|
"command": {
|
|
@@ -65,5 +65,5 @@
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
|
-
"gitHead": "
|
|
68
|
+
"gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
|
|
69
69
|
}
|
package/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,8BAA8B,CAAC"}
|
package/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,8BAA8B,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"jsdom-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,oBAAoB,EACpB,aAAa,EACb,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,sBAAsB,EACzB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,WAAW,EAId,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAS,MAAM,gBAAgB,CAAC;AAE7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACvC,OAAO,EAAyB,cAAc,EAAE,MAAM,OAAO,CAAC;AAK9D,MAAM,MAAM,iBAAiB,CACzB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE3D,MAAM,WAAW,mBAAmB,CAChC,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,GAAG,gBAAgB,EACtF,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,kBAAkB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,eAAe,CAAC;IACrG;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,MAAM,SAAS,CACjB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE/D,MAAM,WAAW,oBAAoB,CACjC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACrD,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,QAAQ,CAAC;IAEnB,IAAI,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAkF7D,qBAAa,YAAY,CACrB,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,oBAAoB,GAAG,oBAAoB,GAAG,gBAAgB,CACxF,SAAQ,WAAW,CAAC,oBAAoB,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC1E,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAIpC;IAEF,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACvC,SAAS,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAQ;gBAE3C,OAAO,GAAE,mBAAmB,CAAC,gBAAgB,EAAE,eAAe,CAAM,EAAE,MAAM,CAAC,EAAE,aAAa;IAwBxG;;;;;;;;;;;;;OAaG;IACH,iBAAiB;IAgBjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAuE;YAE3F,YAAY;YAmEZ,UAAU;CAqC3B;AAED,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,UAAU,gCAAgC;IACtC,YAAY,EAAE,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AASD,gBAAgB;AAChB,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,gCAAgC,oBAoCnH;AAmBD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAC7B,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,EAC3D,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,kDAEzC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"jsdom-crawler.js","sourceRoot":"","sources":["../../src/internals/jsdom-crawler.ts"],"names":[],"mappings":"AAcA,OAAO,EACH,YAAY,EACZ,WAAW,EACX,sCAAsC,EACtC,MAAM,EACN,cAAc,GACjB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAwC,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC9D,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAwErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;AACH,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC;IACjC,6EAA6E;IAC7E,kDAAkD;IAClD,SAAS,EACL,uHAAuH;CAC9H,CAAC,CAAC;AAEH,MAAM,OAAO,YAGX,SAAQ,WAAoE;IAChE,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,WAAW,CAAC,YAAY;QAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC/B,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KAC3C,CAAC;IAEQ,UAAU,CAAU;IACpB,mBAAmB,CAAU;IAC7B,cAAc,GAA0B,IAAI,CAAC;IAEvD,YAAY,UAAkE,EAAE,EAAE,MAAsB;QACpG,MAAM,EAAE,UAAU,GAAG,KAAK,EAAE,mBAAmB,GAAG,KAAK,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAEpF,KAAK,CACD;YACI,GAAG,WAAW;YACd,sBAAsB,EAAE,GAAG,EAAE,CACzB,IAAI,CAAC,oBAAoB,EAAE;iBACtB,OAAO,CAAC;gBACL,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;gBAC3D,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,IAAI,CAAC,iBAAiB,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;oBACnE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;gBAC5B,CAAC;aACJ,CAAC;iBACD,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;SAClF,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,iBAAiB;QACb,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,cAAc,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;QAE3C,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAE7D,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAEgB,iBAAiB,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;IAEjG,KAAK,CAAC,YAAY,CAAC,eAA4C;QACnE,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAE/D,yBAAyB;QACzB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAC1D,GAAG,EAAE,eAAe,CAAC,QAAQ,CAAC,GAAG;YACjC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;YAC7C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;YACvD,SAAS;YACT,cAAc,EAAE,IAAI,CAAC,iBAAiB,EAAE;YACxC,iBAAiB,EAAE,IAAI;SAC1B,CAAC,CAAC;QAEH,kEAAkE;QAClE,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE;YACxC,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,CAAC,KAAc,EAAO,EAAE,CAAC,CAAC;gBAC7B,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,IAAI;gBACd,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC;gBACrB,cAAc,EAAE,GAAG,EAAE,GAAE,CAAC;gBACxB,gBAAgB,EAAE,GAAG,EAAE,GAAE,CAAC;gBAC1B,mBAAmB,EAAE,GAAG,EAAE,GAAE,CAAC;gBAC7B,aAAa,EAAE,GAAG,EAAE,GAAE,CAAC;aAC1B,CAAC;SACL,CAAC,CAAC;QACH,MAAM,CAAC,QAAQ,CAAC,WAAW,GAAG,GAAG,EAAE;YAC/B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjC,KAAK,CAAC,qBAAqB,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,CAAQ,CAAC;YAChD,KAAK,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,IAAW,EAAE,MAAM,EAAE,CAAC,EAAE,CAAQ,CAAC;YAC7E,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,CAAC;gBACD,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE;oBACP,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBACjC,MAAM,CAAC,gBAAgB,CACnB,MAAM,EACN,GAAG,EAAE;4BACD,OAAO,EAAE,CAAC;wBACd,CAAC,EACD,KAAK,CACR,CAAC;oBACN,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBACf,CAAC,EACD,MAAM,EACN,+CAA+C,CAClD,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QAED,OAAO;YACH,MAAM;YACN,IAAI,IAAI;gBACJ,OAAO,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC;YACrD,CAAC;YACD,IAAI,QAAQ;gBACR,OAAO,MAAM,CAAC,QAAQ,CAAC;YAC3B,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,eAAkF;QACvG,OAAO;YACH,YAAY,EAAE,KAAK,EAAE,cAAoC,EAAE,EAAE;gBACzD,OAAO,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,6BAA6B,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE;oBAChG,MAAM,EAAE,eAAe,CAAC,MAAM;oBAC9B,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,oBAAoB;oBAC3C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;iBACrD,CAAC,CAAC;YACP,CAAC;YACD,KAAK,CAAC,eAAe,CAAC,QAAgB,EAAE,SAAS,GAAG,KAAK;gBACrD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7C,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,IAAI,SAAS,EAAE,CAAC;wBACZ,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;wBAChB,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;wBAClE,OAAO;oBACX,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC;YACD,KAAK,CAAC,gBAAgB,CAAC,QAAiB,EAAE,UAAU,GAAG,KAAK;gBACxD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAE7C,IAAI,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC;SACJ,CAAC;IACN,CAAC;;AAqBL,gBAAgB;AAChB,SAAS,oBAAoB,CACzB,OAAuE;IAEvE,OAAO,CAAC,CAAE,OAA4C,CAAC,YAAY,CAAC;AACxE,CAAC;AAED,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,OAAuE;IAChH,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC;IAE9F,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,mBAAmB,EAAE,QAAQ;QAC9C,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,mBAAmB,EAAE,OAAO;KACpD,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,qBAAqB,CAC9B,MAAM,EACN,mBAAmB,EAAE,QAAQ,IAAI,GAAG,EACpC,mBAAmB,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CACxE,CAAC;IAEF,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,YAAY,CAAC;YACxB,IAAI;YACJ,OAAO;YACP,GAAG,mBAAmB;SACzB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,YAAY,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,IAAI;QACJ,OAAO;QACP,GAAG,mBAAmB;KACzB,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,MAAiB,EAAE,QAAgB,EAAE,OAAe;IAC/E,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACvB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC;SACnD,GAAG,CAAC,CAAC,IAAwB,EAAE,EAAE;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAa,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,iBAAiB,CAG/B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|