@crawlee/jsdom 4.0.0-beta.12 → 4.0.0-beta.121

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,19 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
14
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
15
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
16
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
+ </p>
15
+
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
17
21
  </p>
18
22
 
19
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
24
28
 
25
29
  > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
26
30
 
27
- > Crawlee for Python is open for early adopters. 🐍 [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
31
+ > Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
28
32
 
29
33
  ## Installation
30
34
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
85
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
86
90
 
87
91
  ```bash
88
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
89
93
  ```
90
94
 
91
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
94
98
  {
95
99
  "overrides": {
96
100
  "apify": {
97
- "@crawlee/core": "3.12.3-beta.13",
98
- "@crawlee/types": "3.12.3-beta.13",
99
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
100
104
  }
101
105
  }
102
106
  }
package/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.js.map
@@ -1,13 +1,16 @@
1
- import type { Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
1
+ import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
2
2
  import { HttpCrawler } from '@crawlee/http';
3
- import type { Dictionary } from '@crawlee/types';
4
- import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
3
+ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
4
+ import { type CheerioRoot } from '@crawlee/utils/internal';
5
+ import { type RobotsTxtFile } from '@crawlee/utils';
5
6
  import type { DOMWindow } from 'jsdom';
6
7
  import { VirtualConsole } from 'jsdom';
7
8
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
8
- JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
9
- export interface JSDOMCrawlerOptions<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
- JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {
9
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
+ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
11
+ export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
12
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
13
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
11
14
  /**
12
15
  * Download and run scripts.
13
16
  */
@@ -51,10 +54,15 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
51
54
  * ```
52
55
  */
53
56
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
57
+ /**
58
+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
59
+ */
60
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
54
61
  }
55
62
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
56
63
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
57
- export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<JSDOMCrawlingContext, ExtendedContext> {
64
+ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
65
+ #private;
58
66
  protected static optionsShape: {
59
67
  // @ts-ignore optional peer dependency or compatibility with es2022
60
68
  runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
@@ -63,7 +71,7 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
63
71
  // @ts-ignore optional peer dependency or compatibility with es2022
64
72
  navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
65
73
  // @ts-ignore optional peer dependency or compatibility with es2022
66
- ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
74
+ ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
67
75
  // @ts-ignore optional peer dependency or compatibility with es2022
68
76
  additionalMimeTypes: import("ow").ArrayPredicate<string>;
69
77
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -71,11 +79,7 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
71
79
  // @ts-ignore optional peer dependency or compatibility with es2022
72
80
  forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
73
81
  // @ts-ignore optional peer dependency or compatibility with es2022
74
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
82
+ saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
79
83
  // @ts-ignore optional peer dependency or compatibility with es2022
80
84
  preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
81
85
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -88,6 +92,8 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
88
92
  requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
89
93
  // @ts-ignore optional peer dependency or compatibility with es2022
90
94
  requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
95
+ // @ts-ignore optional peer dependency or compatibility with es2022
96
+ requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
91
97
  // @ts-ignore optional peer dependency or compatibility with es2022
92
98
  requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
93
99
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -100,30 +106,46 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
100
106
  maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
101
107
  // @ts-ignore optional peer dependency or compatibility with es2022
102
108
  sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
103
- // @ts-ignore optional peer dependency or compatibility with es2022
104
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
105
109
  // @ts-ignore optional peer dependency or compatibility with es2022
106
110
  maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
107
111
  // @ts-ignore optional peer dependency or compatibility with es2022
108
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
112
+ maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
113
+ // @ts-ignore optional peer dependency or compatibility with es2022
114
+ taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
109
115
  // @ts-ignore optional peer dependency or compatibility with es2022
110
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
116
+ concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
111
117
  // @ts-ignore optional peer dependency or compatibility with es2022
112
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
118
+ sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
113
119
  // @ts-ignore optional peer dependency or compatibility with es2022
114
120
  proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
115
121
  // @ts-ignore optional peer dependency or compatibility with es2022
116
122
  statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
117
123
  // @ts-ignore optional peer dependency or compatibility with es2022
118
124
  statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
125
+ // @ts-ignore optional peer dependency or compatibility with es2022
126
+ additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
127
+ // @ts-ignore optional peer dependency or compatibility with es2022
128
+ ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
129
+ // @ts-ignore optional peer dependency or compatibility with es2022
130
+ blockedStatusCodes: import("ow").ArrayPredicate<number>;
119
131
  // @ts-ignore optional peer dependency or compatibility with es2022
120
132
  retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
121
133
  // @ts-ignore optional peer dependency or compatibility with es2022
122
- respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
134
+ respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
135
+ // @ts-ignore optional peer dependency or compatibility with es2022
136
+ transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/http").StorageWritePolicy> | undefined>;
123
137
  // @ts-ignore optional peer dependency or compatibility with es2022
124
138
  onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
125
139
  // @ts-ignore optional peer dependency or compatibility with es2022
126
140
  httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
141
+ // @ts-ignore optional peer dependency or compatibility with es2022
142
+ configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
143
+ // @ts-ignore optional peer dependency or compatibility with es2022
144
+ storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
145
+ // @ts-ignore optional peer dependency or compatibility with es2022
146
+ eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
147
+ // @ts-ignore optional peer dependency or compatibility with es2022
148
+ logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
127
149
  // @ts-ignore optional peer dependency or compatibility with es2022
128
150
  minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
129
151
  // @ts-ignore optional peer dependency or compatibility with es2022
@@ -133,16 +155,22 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
133
155
  // @ts-ignore optional peer dependency or compatibility with es2022
134
156
  keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
135
157
  // @ts-ignore optional peer dependency or compatibility with es2022
136
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
137
- // @ts-ignore optional peer dependency or compatibility with es2022
138
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
158
+ statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
139
159
  // @ts-ignore optional peer dependency or compatibility with es2022
140
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
160
+ id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
141
161
  };
142
- protected runScripts: boolean;
143
- protected hideInternalConsole: boolean;
144
- protected virtualConsole: VirtualConsole | null;
145
- constructor(options?: JSDOMCrawlerOptions<ExtendedContext>, config?: Configuration);
162
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
163
+ // @ts-ignore optional peer dependency or compatibility with es2022
164
+ protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
165
+ readonly window: DOMWindow;
166
+ readonly body: string;
167
+ readonly document: Document;
168
+ } & {
169
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
170
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
171
+ // @ts-ignore optional peer dependency or compatibility with es2022
172
+ parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
173
+ }>;
146
174
  /**
147
175
  * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
148
176
  *
@@ -165,15 +193,21 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
165
193
  interface EnqueueLinksInternalOptions {
166
194
  options?: EnqueueLinksOptions;
167
195
  window: DOMWindow | null;
168
- requestQueue: RequestProvider;
196
+ requestManager: IRequestManager;
169
197
  robotsTxtFile?: RobotsTxtFile;
170
198
  onSkippedRequest?: SkippedRequestCallback;
171
199
  originalRequestUrl: string;
172
200
  finalRequestUrl?: string;
173
201
  }
202
+ interface BoundEnqueueLinksInternalOptions {
203
+ enqueueLinks: BasicCrawlingContext['enqueueLinks'];
204
+ options?: EnqueueLinksOptions;
205
+ window: DOMWindow | null;
206
+ originalRequestUrl: string;
207
+ finalRequestUrl?: string;
208
+ }
174
209
  /** @internal */
175
- // @ts-ignore optional peer dependency or compatibility with es2022
176
- export declare function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
210
+ export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
177
211
  /**
178
212
  * Creates new {@link Router} instance that works based on request labels.
179
213
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
@@ -198,7 +232,7 @@ export declare function domCrawlerEnqueueLinks({ options, window, requestQueue,
198
232
  * await crawler.run();
199
233
  * ```
200
234
  */
201
- // @ts-ignore optional peer dependency or compatibility with es2022
202
- export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/http").RouterHandler<Context>;
235
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
236
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
237
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
203
238
  export {};
204
- //# sourceMappingURL=jsdom-crawler.d.ts.map