@crawlee/jsdom 4.0.0-beta.99 → 4.0.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/internals/jsdom-crawler.d.ts +129 -118
  2. package/internals/jsdom-crawler.js +39 -63
  3. package/package.json +7 -7
package/internals/jsdom-crawler.d.ts CHANGED
@@ -1,15 +1,16 @@
1
- import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
1
+ import type { AddRequestsBatchedResult, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
2
2
  import { HttpCrawler } from '@crawlee/http';
3
3
  import type { Dictionary } from '@crawlee/types';
4
- import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
4
+ import { type CheerioRoot } from '@crawlee/utils/internal';
5
5
  import type { DOMWindow } from 'jsdom';
6
6
  import { VirtualConsole } from 'jsdom';
7
+ import { z } from 'zod';
7
8
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
8
9
  JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
10
  ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
10
11
  export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
11
12
  JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
12
- Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes> {
13
+ Routes extends Record<keyof Routes, Dictionary> = Record<string, UserData>, StatisticStateExtension extends object = {}> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
13
14
  /**
14
15
  * Download and run scripts.
15
16
  */
@@ -53,113 +54,142 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
53
54
  * ```
54
55
  */
55
56
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
57
+ /**
58
+ * Extracts URLs from the parsed DOM, without adding them to the request queue.
59
+ */
60
+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
61
+ /**
62
+ * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
63
+ */
64
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
56
65
  }
57
66
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
58
67
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
59
- export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
68
+ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
69
+ #private;
60
70
  protected static optionsShape: {
61
- // @ts-ignore optional peer dependency or compatibility with es2022
62
- runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
63
- // @ts-ignore optional peer dependency or compatibility with es2022
64
- hideInternalConsole: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
65
- // @ts-ignore optional peer dependency or compatibility with es2022
66
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
67
- // @ts-ignore optional peer dependency or compatibility with es2022
68
- ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
69
- // @ts-ignore optional peer dependency or compatibility with es2022
70
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
71
- // @ts-ignore optional peer dependency or compatibility with es2022
72
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
73
- // @ts-ignore optional peer dependency or compatibility with es2022
74
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
79
- // @ts-ignore optional peer dependency or compatibility with es2022
80
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
97
- // @ts-ignore optional peer dependency or compatibility with es2022
98
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
99
- // @ts-ignore optional peer dependency or compatibility with es2022
100
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
101
- // @ts-ignore optional peer dependency or compatibility with es2022
102
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
103
- // @ts-ignore optional peer dependency or compatibility with es2022
104
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
105
- // @ts-ignore optional peer dependency or compatibility with es2022
106
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
107
- // @ts-ignore optional peer dependency or compatibility with es2022
108
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
109
- // @ts-ignore optional peer dependency or compatibility with es2022
110
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
111
- // @ts-ignore optional peer dependency or compatibility with es2022
112
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
113
- // @ts-ignore optional peer dependency or compatibility with es2022
114
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
115
- // @ts-ignore optional peer dependency or compatibility with es2022
116
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
117
- // @ts-ignore optional peer dependency or compatibility with es2022
118
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
119
- // @ts-ignore optional peer dependency or compatibility with es2022
120
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
121
- // @ts-ignore optional peer dependency or compatibility with es2022
122
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
123
- // @ts-ignore optional peer dependency or compatibility with es2022
124
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
125
- // @ts-ignore optional peer dependency or compatibility with es2022
126
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
127
- // @ts-ignore optional peer dependency or compatibility with es2022
128
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
129
- // @ts-ignore optional peer dependency or compatibility with es2022
130
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
131
- // @ts-ignore optional peer dependency or compatibility with es2022
132
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
133
- // @ts-ignore optional peer dependency or compatibility with es2022
134
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
135
- // @ts-ignore optional peer dependency or compatibility with es2022
136
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
137
- // @ts-ignore optional peer dependency or compatibility with es2022
138
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
139
- // @ts-ignore optional peer dependency or compatibility with es2022
140
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
141
- // @ts-ignore optional peer dependency or compatibility with es2022
142
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
143
- // @ts-ignore optional peer dependency or compatibility with es2022
144
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
145
- // @ts-ignore optional peer dependency or compatibility with es2022
146
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
147
- // @ts-ignore optional peer dependency or compatibility with es2022
148
- statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
149
- // @ts-ignore optional peer dependency or compatibility with es2022
150
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
71
+ runScripts: z.ZodOptional<z.ZodBoolean>;
72
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
73
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
74
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
75
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
76
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
77
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
78
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
79
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
80
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
81
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
82
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
83
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
84
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
85
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
86
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
87
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
88
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
89
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
90
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
91
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
92
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
93
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
94
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
95
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
96
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
97
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
98
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
99
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
100
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
101
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
102
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
103
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
104
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
105
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
106
+ requestQueue: z.ZodOptional<z.ZodEnum<{
107
+ deferred: "deferred";
108
+ writeThrough: "writeThrough";
109
+ }>>;
110
+ }, z.core.$strict>]>>;
111
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
112
+ // @ts-ignore optional peer dependency or compatibility with es2022
113
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
114
+ // @ts-ignore optional peer dependency or compatibility with es2022
115
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
116
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
117
+ // @ts-ignore optional peer dependency or compatibility with es2022
118
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
119
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
120
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
121
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
122
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
123
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
124
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
125
+ id: z.ZodOptional<z.ZodString>;
151
126
  };
152
- private runScripts;
153
- private hideInternalConsole;
154
- private virtualConsole;
155
- constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
127
+ protected static optionsSchema: z.ZodObject<{
128
+ runScripts: z.ZodOptional<z.ZodBoolean>;
129
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
130
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
131
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
132
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
133
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
134
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
135
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
136
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
137
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
138
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
139
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
140
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
141
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
142
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
143
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
144
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
145
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
146
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
147
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
148
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
149
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
150
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
151
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
152
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
153
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
154
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
155
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
156
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
157
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
158
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
159
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
160
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
161
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
162
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
163
+ requestQueue: z.ZodOptional<z.ZodEnum<{
164
+ deferred: "deferred";
165
+ writeThrough: "writeThrough";
166
+ }>>;
167
+ }, z.core.$strict>]>>;
168
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
169
+ // @ts-ignore optional peer dependency or compatibility with es2022
170
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
171
+ // @ts-ignore optional peer dependency or compatibility with es2022
172
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
173
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
174
+ // @ts-ignore optional peer dependency or compatibility with es2022
175
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
176
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
177
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
178
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
179
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
180
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
181
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
182
+ id: z.ZodOptional<z.ZodString>;
183
+ }, z.core.$strict>;
184
+ constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes, StatisticStateExtension>);
156
185
  // @ts-ignore optional peer dependency or compatibility with es2022
157
186
  protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
158
187
  readonly window: DOMWindow;
159
188
  readonly body: string;
160
189
  readonly document: Document;
161
190
  } & {
162
- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
191
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
192
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
163
193
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
164
194
  // @ts-ignore optional peer dependency or compatibility with es2022
165
195
  parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
@@ -183,24 +213,6 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
183
213
  private parseContent;
184
214
  private addHelpers;
185
215
  }
186
- interface EnqueueLinksInternalOptions {
187
- options?: EnqueueLinksOptions;
188
- window: DOMWindow | null;
189
- requestManager: IRequestManager;
190
- robotsTxtFile?: RobotsTxtFile;
191
- onSkippedRequest?: SkippedRequestCallback;
192
- originalRequestUrl: string;
193
- finalRequestUrl?: string;
194
- }
195
- interface BoundEnqueueLinksInternalOptions {
196
- enqueueLinks: BasicCrawlingContext['enqueueLinks'];
197
- options?: EnqueueLinksOptions;
198
- window: DOMWindow | null;
199
- originalRequestUrl: string;
200
- finalRequestUrl?: string;
201
- }
202
- /** @internal */
203
- export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
204
216
  /**
205
217
  * Creates new {@link Router} instance that works based on request labels.
206
218
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
@@ -228,4 +240,3 @@ export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOpti
228
240
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
229
241
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
230
242
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
231
- export {};
package/internals/jsdom-crawler.js CHANGED
@@ -1,7 +1,7 @@
1
- import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
1
+ import { EnqueueStrategy, HttpCrawler, NavigationSkippedError, parseArgument, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
2
2
  import { sleep } from '@crawlee/utils';
3
3
  import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
4
- import ow from 'ow';
4
+ import { z } from 'zod';
5
5
  import { addTimeoutToPromise } from '@apify/timeout';
6
6
  /**
7
7
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
@@ -86,20 +86,21 @@ const resources = new ResourceLoader({
86
86
  export class JSDOMCrawler extends HttpCrawler {
87
87
  static optionsShape = {
88
88
  ...HttpCrawler.optionsShape,
89
- runScripts: ow.optional.boolean,
90
- hideInternalConsole: ow.optional.boolean,
89
+ runScripts: z.boolean().optional(),
90
+ hideInternalConsole: z.boolean().optional(),
91
91
  };
92
- runScripts;
93
- hideInternalConsole;
94
- virtualConsole = null;
92
+ static optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);
93
+ #runScripts;
94
+ #hideInternalConsole;
95
+ #virtualConsole = null;
95
96
  constructor(options = {}) {
96
- const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
97
+ const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');
97
98
  super({
98
99
  ...httpOptions,
99
100
  contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
100
101
  });
101
- this.runScripts = runScripts;
102
- this.hideInternalConsole = hideInternalConsole;
102
+ this.#runScripts = runScripts;
103
+ this.#hideInternalConsole = hideInternalConsole;
103
104
  }
104
105
  buildContextPipeline() {
105
106
  return super
@@ -128,15 +129,15 @@ export class JSDOMCrawler extends HttpCrawler {
128
129
  * ```
129
130
  */
130
131
  getVirtualConsole() {
131
- if (this.virtualConsole) {
132
- return this.virtualConsole;
132
+ if (this.#virtualConsole) {
133
+ return this.#virtualConsole;
133
134
  }
134
- this.virtualConsole = new VirtualConsole();
135
- if (!this.hideInternalConsole) {
136
- this.virtualConsole.sendTo(console, { omitJSDOMErrors: true });
135
+ this.#virtualConsole = new VirtualConsole();
136
+ if (!this.#hideInternalConsole) {
137
+ this.#virtualConsole.sendTo(console, { omitJSDOMErrors: true });
137
138
  }
138
- this.virtualConsole.on('jsdomError', this.jsdomErrorHandler);
139
- return this.virtualConsole;
139
+ this.#virtualConsole.on('jsdomError', this.jsdomErrorHandler);
140
+ return this.#virtualConsole;
140
141
  }
141
142
  jsdomErrorHandler = (error) => this.log.debug('JSDOM error from console', { error });
142
143
  async parseContent(crawlingContext) {
@@ -146,7 +147,7 @@ export class JSDOMCrawler extends HttpCrawler {
146
147
  const { window } = new JSDOM(crawlingContext.body.toString(), {
147
148
  url: crawlingContext.response.url,
148
149
  contentType: isXml ? 'text/xml' : 'text/html',
149
- runScripts: this.runScripts ? 'dangerously' : undefined,
150
+ runScripts: this.#runScripts ? 'dangerously' : undefined,
150
151
  resources,
151
152
  virtualConsole: this.getVirtualConsole(),
152
153
  pretendToBeVisual: true,
@@ -171,7 +172,7 @@ export class JSDOMCrawler extends HttpCrawler {
171
172
  range.getClientRects = () => ({ item: () => null, length: 0 });
172
173
  return range;
173
174
  };
174
- if (this.runScripts) {
175
+ if (this.#runScripts) {
175
176
  try {
176
177
  await addTimeoutToPromise(async () => {
177
178
  return new Promise((resolve) => {
@@ -213,19 +214,27 @@ export class JSDOMCrawler extends HttpCrawler {
213
214
  }
214
215
  }
215
216
  async addHelpers(crawlingContext) {
217
+ const addRequests = crawlingContext.addRequests;
218
+ const extractLinks = async (options) => {
219
+ if (!crawlingContext.window) {
220
+ throw new Error('Cannot extract links because the JSDOM is not available.');
221
+ }
222
+ return extractUrlsFromWindow(crawlingContext.window, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
223
+ };
216
224
  return {
217
- enqueueLinks: async (enqueueOptions) => {
218
- return domCrawlerEnqueueLinks({
219
- options: {
220
- ...enqueueOptions,
221
- limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
222
- },
223
- window: crawlingContext.window,
224
- requestManager: await this.getRequestManager(),
225
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
226
- onSkippedRequest: this.handleSkippedRequest,
227
- originalRequestUrl: crawlingContext.request.url,
225
+ extractLinks,
226
+ enqueueLinks: async (options = {}) => {
227
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
228
+ enqueueStrategy: options.strategy,
228
229
  finalRequestUrl: crawlingContext.request.loadedUrl,
230
+ originalRequestUrl: crawlingContext.request.url,
231
+ userProvidedBaseUrl: options.baseUrl,
232
+ });
233
+ const urls = await extractLinks(options);
234
+ return addRequests(urls, {
235
+ ...options,
236
+ baseUrl,
237
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
229
238
  });
230
239
  },
231
240
  async waitForSelector(selector, timeoutMs = 5_000) {
@@ -251,39 +260,6 @@ export class JSDOMCrawler extends HttpCrawler {
251
260
  };
252
261
  }
253
262
  }
254
- /** @internal */
255
- function containsEnqueueLinks(options) {
256
- return !!options.enqueueLinks;
257
- }
258
- /** @internal */
259
- export async function domCrawlerEnqueueLinks(options) {
260
- const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
261
- if (!window) {
262
- throw new Error('Cannot enqueue links because the JSDOM is not available.');
263
- }
264
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
265
- enqueueStrategy: enqueueLinksOptions?.strategy,
266
- finalRequestUrl,
267
- originalRequestUrl,
268
- userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
269
- });
270
- const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
271
- if (containsEnqueueLinks(options)) {
272
- return options.enqueueLinks({
273
- urls,
274
- baseUrl,
275
- ...enqueueLinksOptions,
276
- });
277
- }
278
- return enqueueLinks({
279
- requestManager: options.requestManager,
280
- robotsTxtFile: options.robotsTxtFile,
281
- onSkippedRequest: options.onSkippedRequest,
282
- urls,
283
- baseUrl,
284
- ...enqueueLinksOptions,
285
- });
286
- }
287
263
  /**
288
264
  * Extracts URLs from a given Window object.
289
265
  * @ignore
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/jsdom",
3
- "version": "4.0.0-beta.99",
3
+ "version": "4.0.0-rc.0",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,14 +49,14 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
51
  "@apify/utilities": "^2.7.10",
52
- "@crawlee/http": "4.0.0-beta.99",
53
- "@crawlee/types": "4.0.0-beta.99",
54
- "@crawlee/utils": "4.0.0-beta.99",
52
+ "@crawlee/http": "4.0.0-rc.0",
53
+ "@crawlee/types": "4.0.0-rc.0",
54
+ "@crawlee/utils": "4.0.0-rc.0",
55
55
  "@types/jsdom": "^21.1.7",
56
56
  "cheerio": "^1.0.0",
57
57
  "jsdom": "^26.1.0",
58
- "ow": "^2.0.0",
59
- "tslib": "^2.8.1"
58
+ "tslib": "^2.8.1",
59
+ "zod": "^4.4.3"
60
60
  },
61
61
  "lerna": {
62
62
  "command": {
@@ -65,5 +65,5 @@
65
65
  }
66
66
  }
67
67
  },
68
- "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
68
+ "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
69
69
  }