@crawlee/jsdom 4.0.0-beta.13 → 4.0.0-beta.130

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/README.md +17 -13
  2. package/index.d.ts +0 -1
  3. package/index.js +0 -1
  4. package/internals/jsdom-crawler.d.ts +146 -108
  5. package/internals/jsdom-crawler.js +142 -151
  6. package/package.json +10 -10
  7. package/index.d.ts.map +0 -1
  8. package/index.js.map +0 -1
  9. package/internals/jsdom-crawler.d.ts.map +0 -1
  10. package/internals/jsdom-crawler.js.map +0 -1
  11. package/tsconfig.build.tsbuildinfo +0 -1
package/README.md CHANGED
@@ -1,19 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
14
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
15
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
16
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
+ </p>
15
+
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
17
21
  </p>
18
22
 
19
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
24
28
 
25
29
  > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
26
30
 
27
- > Crawlee for Python is open for early adopters. 🐍 [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
31
+ > Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
28
32
 
29
33
  ## Installation
30
34
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
85
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
86
90
 
87
91
  ```bash
88
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
89
93
  ```
90
94
 
91
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
94
98
  {
95
99
  "overrides": {
96
100
  "apify": {
97
- "@crawlee/core": "3.12.3-beta.13",
98
- "@crawlee/types": "3.12.3-beta.13",
99
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
100
104
  }
101
105
  }
102
106
  }
package/index.d.ts CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  export * from '@crawlee/http';
2
2
  export * from './internals/jsdom-crawler.js';
3
- //# sourceMappingURL=index.js.map
package/internals/jsdom-crawler.d.ts CHANGED
@@ -1,13 +1,16 @@
1
- import type { Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
1
+ import type { AddRequestsBatchedResult, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
2
2
  import { HttpCrawler } from '@crawlee/http';
3
3
  import type { Dictionary } from '@crawlee/types';
4
- import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
4
+ import { type CheerioRoot } from '@crawlee/utils/internal';
5
5
  import type { DOMWindow } from 'jsdom';
6
6
  import { VirtualConsole } from 'jsdom';
7
+ import { z } from 'zod';
7
8
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
8
- JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
9
- export interface JSDOMCrawlerOptions<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
- JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ExtendedContext> {
9
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
+ ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
11
+ export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
12
+ JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
13
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
11
14
  /**
12
15
  * Download and run scripts.
13
16
  */
@@ -51,98 +54,146 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
51
54
  * ```
52
55
  */
53
56
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
57
+ /**
58
+ * Extracts URLs from the parsed DOM, without adding them to the request queue.
59
+ */
60
+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
61
+ /**
62
+ * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
63
+ */
64
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
54
65
  }
55
66
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
56
67
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
57
- export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext> extends HttpCrawler<JSDOMCrawlingContext, ExtendedContext> {
68
+ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
69
+ #private;
58
70
  protected static optionsShape: {
59
- // @ts-ignore optional peer dependency or compatibility with es2022
60
- runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
61
- // @ts-ignore optional peer dependency or compatibility with es2022
62
- hideInternalConsole: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
63
- // @ts-ignore optional peer dependency or compatibility with es2022
64
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
65
- // @ts-ignore optional peer dependency or compatibility with es2022
66
- ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
67
- // @ts-ignore optional peer dependency or compatibility with es2022
68
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
69
- // @ts-ignore optional peer dependency or compatibility with es2022
70
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
71
- // @ts-ignore optional peer dependency or compatibility with es2022
72
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
73
- // @ts-ignore optional peer dependency or compatibility with es2022
74
- persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
79
- // @ts-ignore optional peer dependency or compatibility with es2022
80
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
97
- // @ts-ignore optional peer dependency or compatibility with es2022
98
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
99
- // @ts-ignore optional peer dependency or compatibility with es2022
100
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
101
- // @ts-ignore optional peer dependency or compatibility with es2022
102
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
103
- // @ts-ignore optional peer dependency or compatibility with es2022
104
- maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
105
- // @ts-ignore optional peer dependency or compatibility with es2022
106
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
107
- // @ts-ignore optional peer dependency or compatibility with es2022
108
- autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
109
- // @ts-ignore optional peer dependency or compatibility with es2022
110
- sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
111
- // @ts-ignore optional peer dependency or compatibility with es2022
112
- useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
113
- // @ts-ignore optional peer dependency or compatibility with es2022
114
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
115
- // @ts-ignore optional peer dependency or compatibility with es2022
116
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
117
- // @ts-ignore optional peer dependency or compatibility with es2022
118
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
119
- // @ts-ignore optional peer dependency or compatibility with es2022
120
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
121
- // @ts-ignore optional peer dependency or compatibility with es2022
122
- respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
123
- // @ts-ignore optional peer dependency or compatibility with es2022
124
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
125
- // @ts-ignore optional peer dependency or compatibility with es2022
126
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
127
- // @ts-ignore optional peer dependency or compatibility with es2022
128
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
129
- // @ts-ignore optional peer dependency or compatibility with es2022
130
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
131
- // @ts-ignore optional peer dependency or compatibility with es2022
132
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
133
- // @ts-ignore optional peer dependency or compatibility with es2022
134
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
135
- // @ts-ignore optional peer dependency or compatibility with es2022
136
- log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
137
- // @ts-ignore optional peer dependency or compatibility with es2022
138
- experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
139
- // @ts-ignore optional peer dependency or compatibility with es2022
140
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
71
+ runScripts: z.ZodOptional<z.ZodBoolean>;
72
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
73
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
74
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
75
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
76
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
77
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
78
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
79
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
80
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
81
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
82
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
83
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
84
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
85
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
86
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
87
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
88
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
89
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
90
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
91
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
92
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
93
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
94
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
95
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
96
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
97
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
98
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
99
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
100
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
101
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
102
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
103
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
104
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
105
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
106
+ requestQueue: z.ZodOptional<z.ZodEnum<{
107
+ deferred: "deferred";
108
+ writeThrough: "writeThrough";
109
+ }>>;
110
+ }, z.core.$strict>]>>;
111
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
112
+ // @ts-ignore optional peer dependency or compatibility with es2022
113
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
114
+ // @ts-ignore optional peer dependency or compatibility with es2022
115
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
116
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
117
+ // @ts-ignore optional peer dependency or compatibility with es2022
118
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
119
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
120
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
121
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
122
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
123
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
124
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
125
+ id: z.ZodOptional<z.ZodString>;
141
126
  };
142
- protected runScripts: boolean;
143
- protected hideInternalConsole: boolean;
144
- protected virtualConsole: VirtualConsole | null;
145
- constructor(options?: JSDOMCrawlerOptions<ExtendedContext>, config?: Configuration);
127
+ protected static optionsSchema: z.ZodObject<{
128
+ runScripts: z.ZodOptional<z.ZodBoolean>;
129
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
130
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
131
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
132
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
133
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
134
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
135
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
136
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
137
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
138
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
139
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
140
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
141
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
142
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
143
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
144
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
145
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
146
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
147
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
148
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
149
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
150
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
151
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
152
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
153
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
154
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
155
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
156
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
157
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
158
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
159
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
160
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
161
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
162
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
163
+ requestQueue: z.ZodOptional<z.ZodEnum<{
164
+ deferred: "deferred";
165
+ writeThrough: "writeThrough";
166
+ }>>;
167
+ }, z.core.$strict>]>>;
168
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
169
+ // @ts-ignore optional peer dependency or compatibility with es2022
170
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
171
+ // @ts-ignore optional peer dependency or compatibility with es2022
172
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
173
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
174
+ // @ts-ignore optional peer dependency or compatibility with es2022
175
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
176
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
177
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
178
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
179
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
180
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
181
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
182
+ id: z.ZodOptional<z.ZodString>;
183
+ }, z.core.$strict>;
184
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
185
+ // @ts-ignore optional peer dependency or compatibility with es2022
186
+ protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
187
+ readonly window: DOMWindow;
188
+ readonly body: string;
189
+ readonly document: Document;
190
+ } & {
191
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
192
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
193
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
194
+ // @ts-ignore optional peer dependency or compatibility with es2022
195
+ parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
196
+ }>;
146
197
  /**
147
198
  * Returns the currently used `VirtualConsole` instance. Can be used to listen for the JSDOM's internal console messages.
148
199
  *
@@ -162,18 +213,6 @@ export declare class JSDOMCrawler<ExtendedContext extends JSDOMCrawlingContext =
162
213
  private parseContent;
163
214
  private addHelpers;
164
215
  }
165
- interface EnqueueLinksInternalOptions {
166
- options?: EnqueueLinksOptions;
167
- window: DOMWindow | null;
168
- requestQueue: RequestProvider;
169
- robotsTxtFile?: RobotsTxtFile;
170
- onSkippedRequest?: SkippedRequestCallback;
171
- originalRequestUrl: string;
172
- finalRequestUrl?: string;
173
- }
174
- /** @internal */
175
- // @ts-ignore optional peer dependency or compatibility with es2022
176
- export declare function domCrawlerEnqueueLinks({ options, window, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<import("@crawlee/types").BatchAddRequestsResult>;
177
216
  /**
178
217
  * Creates new {@link Router} instance that works based on request labels.
179
218
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
@@ -198,7 +237,6 @@ export declare function domCrawlerEnqueueLinks({ options, window, requestQueue,
198
237
  * await crawler.run();
199
238
  * ```
200
239
  */
201
- // @ts-ignore optional peer dependency or compatibility with es2022
202
- export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/http").RouterHandler<Context>;
203
- export {};
204
- //# sourceMappingURL=jsdom-crawler.d.ts.map
240
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
241
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
242
+ export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;