@crawlee/jsdom 4.0.0-beta.8 → 4.0.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/README.md +17 -13
  2. package/index.d.ts +0 -1
  3. package/index.js +0 -1
  4. package/internals/jsdom-crawler.d.ts +60 -39
  5. package/internals/jsdom-crawler.js +151 -128
  6. package/package.json +8 -8
  7. package/index.d.ts.map +0 -1
  8. package/index.js.map +0 -1
  9. package/internals/jsdom-crawler.d.ts.map +0 -1
  10. package/internals/jsdom-crawler.js.map +0 -1
  11. package/tsconfig.build.tsbuildinfo +0 -1
package/README.md CHANGED
@@ -1,19 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
14
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
15
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
16
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
+ </p>
15
+
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
17
21
  </p>
18
22
 
19
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
24
28
 
25
29
  > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
26
30
 
27
- > Crawlee for Python is open for early adopters. 🐍 [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
31
+ > Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
28
32
 
29
33
  ## Installation
30
34
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
85
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
86
90
 
87
91
  ```bash
88
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
89
93
  ```
90
94
 
91
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
94
98
  {
95
99
  "overrides": {
96
100
  "apify": {
97
- "@crawlee/core": "3.12.3-beta.13",
98
- "@crawlee/types": "3.12.3-beta.13",
99
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
100
104
  }
101
105
  }
102
106
  }
package/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.js.map
package/internals/jsdom-crawler.d.ts CHANGED
@@ -1,14 +1,14 @@
1
- import type { IncomingMessage } from 'node:http';
2
- import type { Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
1
+ import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
3
2
  import { HttpCrawler } from '@crawlee/http';
4
3
  import type { Dictionary } from '@crawlee/types';
5
4
  import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
6
5
  import type { DOMWindow } from 'jsdom';
7
6
  import { VirtualConsole } from 'jsdom';
8
7
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
- JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
10
- export interface JSDOMCrawlerOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
11
- JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>> {
8
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
+ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
10
+ export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
11
+ JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
12
12
  /**
13
13
  * Download and run scripts.
14
14
  */
@@ -21,9 +21,10 @@ JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingConte
21
21
  export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
22
22
  JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
23
23
  export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
24
- JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {
24
+ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
25
25
  window: DOMWindow;
26
26
  document: Document;
27
+ body: string;
27
28
  /**
28
29
  * Wait for an element matching the selector to appear.
29
30
  * Timeout defaults to 5s.
@@ -54,7 +55,7 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
54
55
  }
55
56
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
56
57
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
57
- export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
58
+ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
58
59
  protected static optionsShape: {
59
60
  // @ts-ignore optional peer dependency or compatibility with es2022
60
61
  runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
@@ -71,17 +72,15 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
71
72
  // @ts-ignore optional peer dependency or compatibility with es2022
72
73
  forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
73
74
  // @ts-ignore optional peer dependency or compatibility with es2022
74
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
79
- // @ts-ignore optional peer dependency or compatibility with es2022
80
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
75
+ saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
81
76
  // @ts-ignore optional peer dependency or compatibility with es2022
82
77
  preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
83
78
  // @ts-ignore optional peer dependency or compatibility with es2022
84
79
  postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
80
+ // @ts-ignore optional peer dependency or compatibility with es2022
81
+ contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
82
+ // @ts-ignore optional peer dependency or compatibility with es2022
83
+ extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
85
84
  // @ts-ignore optional peer dependency or compatibility with es2022
86
85
  requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
87
86
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -98,28 +97,42 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
98
97
  maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
99
98
  // @ts-ignore optional peer dependency or compatibility with es2022
100
99
  sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
101
- // @ts-ignore optional peer dependency or compatibility with es2022
102
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
103
100
  // @ts-ignore optional peer dependency or compatibility with es2022
104
101
  maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
102
+ // @ts-ignore optional peer dependency or compatibility with es2022
103
+ maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
105
104
  // @ts-ignore optional peer dependency or compatibility with es2022
106
105
  autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
107
106
  // @ts-ignore optional peer dependency or compatibility with es2022
108
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
107
+ sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
109
108
  // @ts-ignore optional peer dependency or compatibility with es2022
110
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
109
+ proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
111
110
  // @ts-ignore optional peer dependency or compatibility with es2022
112
111
  statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
113
112
  // @ts-ignore optional peer dependency or compatibility with es2022
114
113
  statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
114
+ // @ts-ignore optional peer dependency or compatibility with es2022
115
+ additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
116
+ // @ts-ignore optional peer dependency or compatibility with es2022
117
+ ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
118
+ // @ts-ignore optional peer dependency or compatibility with es2022
119
+ blockedStatusCodes: import("ow").ArrayPredicate<number>;
115
120
  // @ts-ignore optional peer dependency or compatibility with es2022
116
121
  retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
117
122
  // @ts-ignore optional peer dependency or compatibility with es2022
118
- respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
123
+ respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
119
124
  // @ts-ignore optional peer dependency or compatibility with es2022
120
125
  onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
121
126
  // @ts-ignore optional peer dependency or compatibility with es2022
122
127
  httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
128
+ // @ts-ignore optional peer dependency or compatibility with es2022
129
+ configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
130
+ // @ts-ignore optional peer dependency or compatibility with es2022
131
+ storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
132
+ // @ts-ignore optional peer dependency or compatibility with es2022
133
+ eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
134
+ // @ts-ignore optional peer dependency or compatibility with es2022
135
+ logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
123
136
  // @ts-ignore optional peer dependency or compatibility with es2022
124
137
  minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
125
138
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -128,17 +141,26 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
128
141
  maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
129
142
  // @ts-ignore optional peer dependency or compatibility with es2022
130
143
  keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
131
- // @ts-ignore optional peer dependency or compatibility with es2022
132
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
133
- // @ts-ignore optional peer dependency or compatibility with es2022
134
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
135
144
  // @ts-ignore optional peer dependency or compatibility with es2022
136
145
  statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
146
+ // @ts-ignore optional peer dependency or compatibility with es2022
147
+ id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
137
148
  };
138
149
  protected runScripts: boolean;
139
150
  protected hideInternalConsole: boolean;
140
151
  protected virtualConsole: VirtualConsole | null;
141
- constructor(options?: JSDOMCrawlerOptions, config?: Configuration);
152
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
153
+ // @ts-ignore optional peer dependency or compatibility with es2022
154
+ protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
155
+ readonly window: DOMWindow;
156
+ readonly body: string;
157
+ readonly document: Document;
158
+ } & {
159
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
160
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
161
+ // @ts-ignore optional peer dependency or compatibility with es2022
162
+ parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
163
+ }>;
142
164
  /**
143
165
  * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
144
166
  *
@@ -155,28 +177,27 @@ export declare class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
155
177
  */
156
178
  getVirtualConsole(): VirtualConsole;
157
179
  private readonly jsdomErrorHandler;
158
- protected _cleanupContext(context: JSDOMCrawlingContext): Promise<void>;
159
- protected _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: JSDOMCrawlingContext): Promise<{
160
- window: DOMWindow;
161
- readonly body: string;
162
- readonly document: Document;
163
- // @ts-ignore optional peer dependency or compatibility with es2022
164
- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
165
- }>;
166
- _runRequestHandler(context: JSDOMCrawlingContext): Promise<void>;
180
+ private parseContent;
181
+ private addHelpers;
167
182
  }
168
183
  interface EnqueueLinksInternalOptions {
169
184
  options?: EnqueueLinksOptions;
170
185
  window: DOMWindow | null;
171
- requestQueue: RequestProvider;
186
+ requestManager: IRequestManager;
172
187
  robotsTxtFile?: RobotsTxtFile;
173
188
  onSkippedRequest?: SkippedRequestCallback;
174
189
  originalRequestUrl: string;
175
190
  finalRequestUrl?: string;
176
191
  }
192
+ interface BoundEnqueueLinksInternalOptions {
193
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
194
+ options?: EnqueueLinksOptions;
195
+ window: DOMWindow | null;
196
+ originalRequestUrl: string;
197
+ finalRequestUrl?: string;
198
+ }
177
199
  /** @internal */
178
- // @ts-ignore optional peer dependency or compatibility with es2022
179
- export declare function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
200
+ export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
180
201
  /**
181
202
  * Creates new {@link Router} instance that works based on request labels.
182
203
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
@@ -201,7 +222,7 @@ export declare function domCrawlerEnqueueLinks({ options, window, requestQueue,
201
222
  * await crawler.run();
202
223
  * ```
203
224
  */
204
- // @ts-ignore optional peer dependency or compatibility with es2022
205
- export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/http").RouterHandler<Context>;
225
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
226
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
227
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
206
228
  export {};
207
- //# sourceMappingURL=jsdom-crawler.d.ts.map
package/internals/jsdom-crawler.js CHANGED
@@ -1,10 +1,8 @@
1
- import { enqueueLinks, HttpCrawler, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
1
+ import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
2
2
  import { sleep } from '@crawlee/utils';
3
- import * as cheerio from 'cheerio';
4
3
  import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
5
4
  import ow from 'ow';
6
5
  import { addTimeoutToPromise } from '@apify/timeout';
7
- import { concatStreamToBuffer } from '@apify/utilities';
8
6
  /**
9
7
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
10
8
  * [jsdom](https://www.npmjs.com/package/jsdom) JSDOM implementation.
@@ -27,28 +25,30 @@ import { concatStreamToBuffer } from '@apify/utilities';
27
25
  * and then invokes the user-provided {@link JSDOMCrawlerOptions.requestHandler} to extract page data
28
26
  * using the `window` object.
29
27
  *
30
- * The source URLs are represented using {@link Request} objects that are fed from
31
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link JSDOMCrawlerOptions.requestList}
32
- * or {@link JSDOMCrawlerOptions.requestQueue} constructor options, respectively.
28
+ * The source URLs are represented using {@link Request} objects that are fed from the
29
+ * {@link IRequestManager|request manager} provided via the {@link JSDOMCrawlerOptions.requestManager|`requestManager`}
30
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
31
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
32
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
33
+ * result as `requestManager`.
33
34
  *
34
- * If both {@link JSDOMCrawlerOptions.requestList} and {@link JSDOMCrawlerOptions.requestQueue} are used,
35
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
36
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
35
+ * > The {@link JSDOMCrawlerOptions.requestList|`requestList`} and {@link JSDOMCrawlerOptions.requestQueue|`requestQueue`}
36
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
37
37
  *
38
38
  * The crawler finishes when there are no more {@link Request} objects to crawl.
39
39
  *
40
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
40
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
41
41
  *
42
42
  * ```
43
43
  * preNavigationHooks: [
44
- * (crawlingContext, gotOptions) => {
44
+ * (crawlingContext) => {
45
45
  * // ...
46
46
  * },
47
47
  * ]
48
48
  * ```
49
49
  *
50
- * By default, `JSDOMCrawler` only processes web pages with the `text/html`
51
- * and `application/xhtml+xml` MIME content types (as reported by the `Content-Type` HTTP header),
50
+ * By default, `JSDOMCrawler` only processes web pages with the `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
51
+ * and `application/json` MIME content types (as reported by the `Content-Type` HTTP header),
52
52
  * and skips pages with other content types. If you want the crawler to process other content types,
53
53
  * use the {@link JSDOMCrawlerOptions.additionalMimeTypes} constructor option.
54
54
  * Beware that the parsing behavior differs for HTML, XML, JSON and other types of content.
@@ -92,12 +92,27 @@ export class JSDOMCrawler extends HttpCrawler {
92
92
  runScripts;
93
93
  hideInternalConsole;
94
94
  virtualConsole = null;
95
- constructor(options = {}, config) {
96
- const { runScripts = false, hideInternalConsole = false, ...httpOptions } = options;
97
- super(httpOptions, config);
95
+ constructor(options = {}) {
96
+ const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
97
+ super({
98
+ ...httpOptions,
99
+ contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
100
+ });
98
101
  this.runScripts = runScripts;
99
102
  this.hideInternalConsole = hideInternalConsole;
100
103
  }
104
+ buildContextPipeline() {
105
+ return super
106
+ .buildContextPipeline()
107
+ .compose({
108
+ action: async (context) => await this.parseContent(context),
109
+ cleanup: async (context) => {
110
+ this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
111
+ context.window?.close();
112
+ },
113
+ })
114
+ .compose({ action: async (context) => await this.addHelpers(context) });
115
+ }
101
116
  /**
102
117
  * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
103
118
  *
@@ -123,117 +138,150 @@ export class JSDOMCrawler extends HttpCrawler {
123
138
  this.virtualConsole.on('jsdomError', this.jsdomErrorHandler);
124
139
  return this.virtualConsole;
125
140
  }
126
- jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', error);
127
- async _cleanupContext(context) {
128
- this.getVirtualConsole().off('jsdomError', this.jsdomErrorHandler);
129
- context.window?.close();
130
- }
131
- async _parseHTML(response, isXml, crawlingContext) {
132
- const body = await concatStreamToBuffer(response);
133
- const { window } = new JSDOM(body, {
134
- url: response.url,
135
- contentType: isXml ? 'text/xml' : 'text/html',
136
- runScripts: this.runScripts ? 'dangerously' : undefined,
137
- resources,
138
- virtualConsole: this.getVirtualConsole(),
139
- pretendToBeVisual: true,
140
- });
141
- // add some stubs in place of missing API so processing won't fail
142
- Object.defineProperty(window, 'matchMedia', {
143
- writable: true,
144
- value: (query) => ({
145
- matches: false,
146
- media: query,
147
- onchange: null,
148
- addListener: () => { },
149
- removeListener: () => { },
150
- addEventListener: () => { },
151
- removeEventListener: () => { },
152
- dispatchEvent: () => { },
153
- }),
154
- });
155
- window.document.createRange = () => {
156
- const range = new window.Range();
157
- range.getBoundingClientRect = () => ({});
158
- range.getClientRects = () => ({ item: () => null, length: 0 });
159
- return range;
160
- };
161
- if (this.runScripts) {
162
- try {
163
- await addTimeoutToPromise(async () => {
164
- return new Promise((resolve) => {
165
- window.addEventListener('load', () => {
166
- resolve();
167
- }, false);
168
- }).catch();
169
- }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
141
+ jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
142
+ async parseContent(crawlingContext) {
143
+ try {
144
+ const isXml = crawlingContext.contentType.type.includes('xml');
145
+ // TODO handle non-string
146
+ const { window } = new JSDOM(crawlingContext.body.toString(), {
147
+ url: crawlingContext.response.url,
148
+ contentType: isXml ? 'text/xml' : 'text/html',
149
+ runScripts: this.runScripts ? 'dangerously' : undefined,
150
+ resources,
151
+ virtualConsole: this.getVirtualConsole(),
152
+ pretendToBeVisual: true,
153
+ });
154
+ // add some stubs in place of missing API so processing won't fail
155
+ Object.defineProperty(window, 'matchMedia', {
156
+ writable: true,
157
+ value: (query) => ({
158
+ matches: false,
159
+ media: query,
160
+ onchange: null,
161
+ addListener: () => { },
162
+ removeListener: () => { },
163
+ addEventListener: () => { },
164
+ removeEventListener: () => { },
165
+ dispatchEvent: () => { },
166
+ }),
167
+ });
168
+ window.document.createRange = () => {
169
+ const range = new window.Range();
170
+ range.getBoundingClientRect = () => ({});
171
+ range.getClientRects = () => ({ item: () => null, length: 0 });
172
+ return range;
173
+ };
174
+ if (this.runScripts) {
175
+ try {
176
+ await addTimeoutToPromise(async () => {
177
+ return new Promise((resolve) => {
178
+ window.addEventListener('load', () => {
179
+ resolve();
180
+ }, false);
181
+ }).catch();
182
+ }, 10_000, 'Window.load event not fired after 10 seconds.').catch();
183
+ }
184
+ catch (e) {
185
+ this.log.debug(e.message);
186
+ }
170
187
  }
171
- catch (e) {
172
- this.log.debug(e.message);
188
+ return {
189
+ window,
190
+ get body() {
191
+ return window.document.documentElement.outerHTML;
192
+ },
193
+ get document() {
194
+ return window.document;
195
+ },
196
+ };
197
+ }
198
+ catch (err) {
199
+ if (err instanceof NavigationSkippedError) {
200
+ return {
201
+ get window() {
202
+ throw new NavigationSkippedError('The `window` property is not available - `skipNavigation` was used', { cause: err });
203
+ },
204
+ get body() {
205
+ throw new NavigationSkippedError('The `body` property is not available - `skipNavigation` was used', { cause: err });
206
+ },
207
+ get document() {
208
+ throw new NavigationSkippedError('The `document` property is not available - `skipNavigation` was used', { cause: err });
209
+ },
210
+ };
173
211
  }
212
+ throw err;
174
213
  }
214
+ }
215
+ async addHelpers(crawlingContext) {
175
216
  return {
176
- window,
177
- get body() {
178
- return window.document.documentElement.outerHTML;
179
- },
180
- get document() {
181
- return window.document;
182
- },
183
217
  enqueueLinks: async (enqueueOptions) => {
184
218
  return domCrawlerEnqueueLinks({
185
- options: enqueueOptions,
186
- window,
187
- requestQueue: await this.getRequestQueue(),
219
+ options: {
220
+ ...enqueueOptions,
221
+ limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
222
+ },
223
+ window: crawlingContext.window,
224
+ requestManager: await this.getRequestManager(),
188
225
  robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
189
- onSkippedRequest: this.onSkippedRequest,
226
+ onSkippedRequest: this.handleSkippedRequest,
190
227
  originalRequestUrl: crawlingContext.request.url,
191
228
  finalRequestUrl: crawlingContext.request.loadedUrl,
192
229
  });
193
230
  },
194
- };
195
- }
196
- async _runRequestHandler(context) {
197
- context.waitForSelector = async (selector, timeoutMs = 5_000) => {
198
- const $ = cheerio.load(context.body);
199
- if ($(selector).get().length === 0) {
200
- if (timeoutMs) {
201
- await sleep(50);
202
- await context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
203
- return;
231
+ async waitForSelector(selector, timeoutMs = 5_000) {
232
+ const cheerio = await import('cheerio');
233
+ const $ = cheerio.load(crawlingContext.body);
234
+ if ($(selector).get().length === 0) {
235
+ if (timeoutMs) {
236
+ await sleep(50);
237
+ await this.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
238
+ return;
239
+ }
240
+ throw new Error(`Selector '${selector}' not found.`);
204
241
  }
205
- throw new Error(`Selector '${selector}' not found.`);
206
- }
207
- };
208
- context.parseWithCheerio = async (selector, _timeoutMs = 5_000) => {
209
- const $ = cheerio.load(context.body);
210
- if (selector && $(selector).get().length === 0) {
211
- throw new Error(`Selector '${selector}' not found.`);
212
- }
213
- return $;
242
+ },
243
+ async parseWithCheerio(selector, _timeoutMs = 5_000) {
244
+ const cheerio = await import('cheerio');
245
+ const $ = cheerio.load(crawlingContext.body);
246
+ if (selector && $(selector).get().length === 0) {
247
+ throw new Error(`Selector '${selector}' not found.`);
248
+ }
249
+ return $;
250
+ },
214
251
  };
215
- await super._runRequestHandler(context);
216
252
  }
217
253
  }
218
254
  /** @internal */
219
- export async function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
255
+ function containsEnqueueLinks(options) {
256
+ return !!options.enqueueLinks;
257
+ }
258
+ /** @internal */
259
+ export async function domCrawlerEnqueueLinks(options) {
260
+ const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
220
261
  if (!window) {
221
262
  throw new Error('Cannot enqueue links because the JSDOM is not available.');
222
263
  }
223
264
  const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
224
- enqueueStrategy: options?.strategy,
265
+ enqueueStrategy: enqueueLinksOptions?.strategy,
225
266
  finalRequestUrl,
226
267
  originalRequestUrl,
227
- userProvidedBaseUrl: options?.baseUrl,
268
+ userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
228
269
  });
229
- const urls = extractUrlsFromWindow(window, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
270
+ const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
271
+ if (containsEnqueueLinks(options)) {
272
+ return options.enqueueLinks({
273
+ urls,
274
+ baseUrl,
275
+ ...enqueueLinksOptions,
276
+ });
277
+ }
230
278
  return enqueueLinks({
231
- requestQueue,
232
- robotsTxtFile,
233
- onSkippedRequest,
279
+ requestManager: options.requestManager,
280
+ robotsTxtFile: options.robotsTxtFile,
281
+ onSkippedRequest: options.onSkippedRequest,
234
282
  urls,
235
283
  baseUrl,
236
- ...options,
284
+ ...enqueueLinksOptions,
237
285
  });
238
286
  }
239
287
  /**
@@ -252,31 +300,6 @@ function extractUrlsFromWindow(window, selector, baseUrl) {
252
300
  })
253
301
  .filter((href) => href !== undefined && href !== '');
254
302
  }
255
- /**
256
- * Creates new {@link Router} instance that works based on request labels.
257
- * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
258
- * Defaults to the {@link JSDOMCrawlingContext}.
259
- *
260
- * > Serves as a shortcut for using `Router.create<JSDOMCrawlingContext>()`.
261
- *
262
- * ```ts
263
- * import { JSDOMCrawler, createJSDOMRouter } from 'crawlee';
264
- *
265
- * const router = createJSDOMRouter();
266
- * router.addHandler('label-a', async (ctx) => {
267
- * ctx.log.info('...');
268
- * });
269
- * router.addDefaultHandler(async (ctx) => {
270
- * ctx.log.info('...');
271
- * });
272
- *
273
- * const crawler = new JSDOMCrawler({
274
- * requestHandler: router,
275
- * });
276
- * await crawler.run();
277
- * ```
278
- */
279
- export function createJSDOMRouter(routes) {
280
- return Router.create(routes);
303
+ export function createJSDOMRouter(routesOrSchemas) {
304
+ return Router.create(routesOrSchemas);
281
305
  }
282
- //# sourceMappingURL=jsdom-crawler.js.map