@crawlee/jsdom 4.0.0-beta.12 → 4.0.0-beta.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/internals/jsdom-crawler.d.ts +67 -33
- package/internals/jsdom-crawler.js +147 -132
- package/package.json +8 -8
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/jsdom-crawler.d.ts.map +0 -1
- package/internals/jsdom-crawler.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
17
21
|
</p>
|
|
18
22
|
|
|
19
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
85
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
86
90
|
|
|
87
91
|
```bash
|
|
88
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
89
93
|
```
|
|
90
94
|
|
|
91
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
94
98
|
{
|
|
95
99
|
"overrides": {
|
|
96
100
|
"apify": {
|
|
97
|
-
"@crawlee/core": "
|
|
98
|
-
"@crawlee/types": "
|
|
99
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
100
104
|
}
|
|
101
105
|
}
|
|
102
106
|
}
|
package/index.d.ts
CHANGED
package/index.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
|
|
2
2
|
import { HttpCrawler } from '@crawlee/http';
|
|
3
|
-
import type { Dictionary } from '@crawlee/types';
|
|
4
|
-
import { type CheerioRoot
|
|
3
|
+
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
4
|
+
import { type CheerioRoot } from '@crawlee/utils/internal';
|
|
5
|
+
import { type RobotsTxtFile } from '@crawlee/utils';
|
|
5
6
|
import type { DOMWindow } from 'jsdom';
|
|
6
7
|
import { VirtualConsole } from 'jsdom';
|
|
7
8
|
export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
8
|
-
JSONData extends Dictionary = any
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
10
|
+
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
|
|
11
|
+
export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
12
|
+
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
13
|
+
Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
|
|
11
14
|
/**
|
|
12
15
|
* Download and run scripts.
|
|
13
16
|
*/
|
|
@@ -51,10 +54,15 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
51
54
|
* ```
|
|
52
55
|
*/
|
|
53
56
|
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
|
57
|
+
/**
|
|
58
|
+
* Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
|
|
59
|
+
*/
|
|
60
|
+
enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
54
61
|
}
|
|
55
62
|
export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
56
63
|
JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
|
|
57
|
-
export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<JSDOMCrawlingContext, ExtendedContext> {
|
|
64
|
+
export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
65
|
+
#private;
|
|
58
66
|
protected static optionsShape: {
|
|
59
67
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
60
68
|
runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
@@ -63,7 +71,7 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
63
71
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
64
72
|
navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
65
73
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
66
|
-
|
|
74
|
+
ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
67
75
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
68
76
|
additionalMimeTypes: import("ow").ArrayPredicate<string>;
|
|
69
77
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -71,11 +79,7 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
71
79
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
72
80
|
forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
73
81
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
74
|
-
|
|
75
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
|
-
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
77
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
78
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
82
|
+
saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
79
83
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
84
|
preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
|
|
81
85
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -88,6 +92,8 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
88
92
|
requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
89
93
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
90
94
|
requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
95
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
96
|
+
requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
91
97
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
92
98
|
requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
93
99
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -100,30 +106,46 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
100
106
|
maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
101
107
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
102
108
|
sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
103
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
104
|
-
maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
105
109
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
106
110
|
maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
107
111
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
108
|
-
|
|
112
|
+
maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
113
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
114
|
+
taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
109
115
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
110
|
-
|
|
116
|
+
concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
111
117
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
112
|
-
|
|
118
|
+
sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
113
119
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
114
120
|
proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
115
121
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
116
122
|
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
117
123
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
118
124
|
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
125
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
126
|
+
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
127
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
128
|
+
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
129
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
130
|
+
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
119
131
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
120
132
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
121
133
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
122
|
-
respectRobotsTxtFile: import("ow").
|
|
134
|
+
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
135
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
136
|
+
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/http").StorageWritePolicy> | undefined>;
|
|
123
137
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
124
138
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
125
139
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
126
140
|
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
141
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
142
|
+
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
143
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
144
|
+
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
145
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
146
|
+
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
147
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
148
|
+
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
127
149
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
128
150
|
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
129
151
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -133,16 +155,22 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
133
155
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
134
156
|
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
135
157
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
136
|
-
|
|
137
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
138
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
158
|
+
statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
139
159
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
140
|
-
|
|
160
|
+
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
141
161
|
};
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
protected
|
|
145
|
-
|
|
162
|
+
constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
|
|
163
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
164
|
+
protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
|
|
165
|
+
readonly window: DOMWindow;
|
|
166
|
+
readonly body: string;
|
|
167
|
+
readonly document: Document;
|
|
168
|
+
} & {
|
|
169
|
+
enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
|
|
170
|
+
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
171
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
172
|
+
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
|
|
173
|
+
}>;
|
|
146
174
|
/**
|
|
147
175
|
* Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
|
|
148
176
|
*
|
|
@@ -165,15 +193,21 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
|
|
|
165
193
|
interface EnqueueLinksInternalOptions {
|
|
166
194
|
options?: EnqueueLinksOptions;
|
|
167
195
|
window: DOMWindow | null;
|
|
168
|
-
|
|
196
|
+
requestManager: IRequestManager;
|
|
169
197
|
robotsTxtFile?: RobotsTxtFile;
|
|
170
198
|
onSkippedRequest?: SkippedRequestCallback;
|
|
171
199
|
originalRequestUrl: string;
|
|
172
200
|
finalRequestUrl?: string;
|
|
173
201
|
}
|
|
202
|
+
interface BoundEnqueueLinksInternalOptions {
|
|
203
|
+
enqueueLinks: BasicCrawlingContext['enqueueLinks'];
|
|
204
|
+
options?: EnqueueLinksOptions;
|
|
205
|
+
window: DOMWindow | null;
|
|
206
|
+
originalRequestUrl: string;
|
|
207
|
+
finalRequestUrl?: string;
|
|
208
|
+
}
|
|
174
209
|
/** @internal */
|
|
175
|
-
|
|
176
|
-
export declare function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
|
|
210
|
+
export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
|
177
211
|
/**
|
|
178
212
|
* Creates new {@link Router} instance that works based on request labels.
|
|
179
213
|
* This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
|
|
@@ -198,7 +232,7 @@ export declare function domCrawlerEnqueueLinks({ options, window, requestQueue,
|
|
|
198
232
|
* await crawler.run();
|
|
199
233
|
* ```
|
|
200
234
|
*/
|
|
201
|
-
|
|
202
|
-
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData
|
|
235
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
|
|
236
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
|
|
237
|
+
export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
|
|
203
238
|
export {};
|
|
204
|
-
//# sourceMappingURL=jsdom-crawler.d.ts.map
|