@crawlee/jsdom 4.0.0-beta.121 → 4.0.0-beta.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/internals/jsdom-crawler.d.ts +123 -119
  2. package/internals/jsdom-crawler.js +26 -50
  3. package/package.json +7 -7
package/internals/jsdom-crawler.d.ts CHANGED
@@ -1,10 +1,10 @@
1
- import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
1
+ import type { AddRequestsBatchedResult, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/http';
2
2
  import { HttpCrawler } from '@crawlee/http';
3
- import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
+ import type { Dictionary } from '@crawlee/types';
4
4
  import { type CheerioRoot } from '@crawlee/utils/internal';
5
- import { type RobotsTxtFile } from '@crawlee/utils';
6
5
  import type { DOMWindow } from 'jsdom';
7
6
  import { VirtualConsole } from 'jsdom';
7
+ import { z } from 'zod';
8
8
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
9
  JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
10
  ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
@@ -55,110 +55,132 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
55
55
  */
56
56
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
57
57
  /**
58
- * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
58
+ * Extracts URLs from the parsed DOM, without adding them to the request queue.
59
59
  */
60
- enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
60
+ extractLinks(options?: ExtractLinksOptions): Promise<string[]>;
61
+ /**
62
+ * Helper function for extracting URLs from the parsed DOM and adding them to the request queue.
63
+ */
64
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<AddRequestsBatchedResult>;
61
65
  }
62
66
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
63
67
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
64
68
  export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
65
69
  #private;
66
70
  protected static optionsShape: {
67
- // @ts-ignore optional peer dependency or compatibility with es2022
68
- runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
69
- // @ts-ignore optional peer dependency or compatibility with es2022
70
- hideInternalConsole: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
71
- // @ts-ignore optional peer dependency or compatibility with es2022
72
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
73
- // @ts-ignore optional peer dependency or compatibility with es2022
74
- ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
79
- // @ts-ignore optional peer dependency or compatibility with es2022
80
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
97
- // @ts-ignore optional peer dependency or compatibility with es2022
98
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
99
- // @ts-ignore optional peer dependency or compatibility with es2022
100
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
101
- // @ts-ignore optional peer dependency or compatibility with es2022
102
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
103
- // @ts-ignore optional peer dependency or compatibility with es2022
104
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
105
- // @ts-ignore optional peer dependency or compatibility with es2022
106
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
107
- // @ts-ignore optional peer dependency or compatibility with es2022
108
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
109
- // @ts-ignore optional peer dependency or compatibility with es2022
110
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
111
- // @ts-ignore optional peer dependency or compatibility with es2022
112
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
113
- // @ts-ignore optional peer dependency or compatibility with es2022
114
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
115
- // @ts-ignore optional peer dependency or compatibility with es2022
116
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
117
- // @ts-ignore optional peer dependency or compatibility with es2022
118
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
119
- // @ts-ignore optional peer dependency or compatibility with es2022
120
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
121
- // @ts-ignore optional peer dependency or compatibility with es2022
122
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
123
- // @ts-ignore optional peer dependency or compatibility with es2022
124
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
125
- // @ts-ignore optional peer dependency or compatibility with es2022
126
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
127
- // @ts-ignore optional peer dependency or compatibility with es2022
128
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
129
- // @ts-ignore optional peer dependency or compatibility with es2022
130
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
131
- // @ts-ignore optional peer dependency or compatibility with es2022
132
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
133
- // @ts-ignore optional peer dependency or compatibility with es2022
134
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
135
- // @ts-ignore optional peer dependency or compatibility with es2022
136
- transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/http").StorageWritePolicy> | undefined>;
137
- // @ts-ignore optional peer dependency or compatibility with es2022
138
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
139
- // @ts-ignore optional peer dependency or compatibility with es2022
140
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
141
- // @ts-ignore optional peer dependency or compatibility with es2022
142
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
143
- // @ts-ignore optional peer dependency or compatibility with es2022
144
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
145
- // @ts-ignore optional peer dependency or compatibility with es2022
146
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
147
- // @ts-ignore optional peer dependency or compatibility with es2022
148
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
149
- // @ts-ignore optional peer dependency or compatibility with es2022
150
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
151
- // @ts-ignore optional peer dependency or compatibility with es2022
152
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
153
- // @ts-ignore optional peer dependency or compatibility with es2022
154
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
155
- // @ts-ignore optional peer dependency or compatibility with es2022
156
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
157
- // @ts-ignore optional peer dependency or compatibility with es2022
158
- statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
159
- // @ts-ignore optional peer dependency or compatibility with es2022
160
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
71
+ runScripts: z.ZodOptional<z.ZodBoolean>;
72
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
73
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
74
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
75
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
76
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
77
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
78
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
79
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
80
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
81
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
82
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
83
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
84
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
85
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
86
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
87
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
88
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
89
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
90
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
91
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
92
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
93
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
94
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
95
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
96
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
97
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
98
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
99
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
100
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
101
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
102
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
103
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
104
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
105
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
106
+ requestQueue: z.ZodOptional<z.ZodEnum<{
107
+ deferred: "deferred";
108
+ writeThrough: "writeThrough";
109
+ }>>;
110
+ }, z.core.$strict>]>>;
111
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
112
+ // @ts-ignore optional peer dependency or compatibility with es2022
113
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
114
+ // @ts-ignore optional peer dependency or compatibility with es2022
115
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
116
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
117
+ // @ts-ignore optional peer dependency or compatibility with es2022
118
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
119
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
120
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
121
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
122
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
123
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
124
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
125
+ id: z.ZodOptional<z.ZodString>;
161
126
  };
127
+ protected static optionsSchema: z.ZodObject<{
128
+ runScripts: z.ZodOptional<z.ZodBoolean>;
129
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
130
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
131
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
132
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
133
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
134
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
135
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
136
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
137
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
138
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
139
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
140
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
141
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
142
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
143
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
144
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
145
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
146
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
147
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
148
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
149
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
150
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
151
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
152
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
153
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
154
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
155
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
156
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
157
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
158
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
159
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
160
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
161
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
162
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
163
+ requestQueue: z.ZodOptional<z.ZodEnum<{
164
+ deferred: "deferred";
165
+ writeThrough: "writeThrough";
166
+ }>>;
167
+ }, z.core.$strict>]>>;
168
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
169
+ // @ts-ignore optional peer dependency or compatibility with es2022
170
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
171
+ // @ts-ignore optional peer dependency or compatibility with es2022
172
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
173
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
174
+ // @ts-ignore optional peer dependency or compatibility with es2022
175
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
176
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
177
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
178
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
179
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
180
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
181
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
182
+ id: z.ZodOptional<z.ZodString>;
183
+ }, z.core.$strict>;
162
184
  constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
163
185
  // @ts-ignore optional peer dependency or compatibility with es2022
164
186
  protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
@@ -166,7 +188,8 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
166
188
  readonly body: string;
167
189
  readonly document: Document;
168
190
  } & {
169
- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
191
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
192
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
170
193
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
171
194
  // @ts-ignore optional peer dependency or compatibility with es2022
172
195
  parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
@@ -190,24 +213,6 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
190
213
  private parseContent;
191
214
  private addHelpers;
192
215
  }
193
- interface EnqueueLinksInternalOptions {
194
- options?: EnqueueLinksOptions;
195
- window: DOMWindow | null;
196
- requestManager: IRequestManager;
197
- robotsTxtFile?: RobotsTxtFile;
198
- onSkippedRequest?: SkippedRequestCallback;
199
- originalRequestUrl: string;
200
- finalRequestUrl?: string;
201
- }
202
- interface BoundEnqueueLinksInternalOptions {
203
- enqueueLinks: BasicCrawlingContext['enqueueLinks'];
204
- options?: EnqueueLinksOptions;
205
- window: DOMWindow | null;
206
- originalRequestUrl: string;
207
- finalRequestUrl?: string;
208
- }
209
- /** @internal */
210
- export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
211
216
  /**
212
217
  * Creates new {@link Router} instance that works based on request labels.
213
218
  * This instance can then serve as a `requestHandler` of your {@link JSDOMCrawler}.
@@ -235,4 +240,3 @@ export declare function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOpti
235
240
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
236
241
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
237
242
  export declare function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
238
- export {};
package/internals/jsdom-crawler.js CHANGED
@@ -1,7 +1,7 @@
1
- import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
1
+ import { EnqueueStrategy, HttpCrawler, NavigationSkippedError, parseArgument, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
2
2
  import { sleep } from '@crawlee/utils';
3
3
  import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
4
- import ow from 'ow';
4
+ import { z } from 'zod';
5
5
  import { addTimeoutToPromise } from '@apify/timeout';
6
6
  /**
7
7
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
@@ -86,14 +86,15 @@ const resources = new ResourceLoader({
86
86
  export class JSDOMCrawler extends HttpCrawler {
87
87
  static optionsShape = {
88
88
  ...HttpCrawler.optionsShape,
89
- runScripts: ow.optional.boolean,
90
- hideInternalConsole: ow.optional.boolean,
89
+ runScripts: z.boolean().optional(),
90
+ hideInternalConsole: z.boolean().optional(),
91
91
  };
92
+ static optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);
92
93
  #runScripts;
93
94
  #hideInternalConsole;
94
95
  #virtualConsole = null;
95
96
  constructor(options = {}) {
96
- const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
97
+ const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');
97
98
  super({
98
99
  ...httpOptions,
99
100
  contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
@@ -213,20 +214,28 @@ export class JSDOMCrawler extends HttpCrawler {
213
214
  }
214
215
  }
215
216
  async addHelpers(crawlingContext) {
217
+ const addRequests = crawlingContext.addRequests;
218
+ const extractLinks = async (options) => {
219
+ if (!crawlingContext.window) {
220
+ throw new Error('Cannot extract links because the JSDOM is not available.');
221
+ }
222
+ return extractUrlsFromWindow(crawlingContext.window, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
223
+ };
216
224
  return {
217
- enqueueLinks: async (enqueueOptions) => {
218
- return (await domCrawlerEnqueueLinks({
219
- options: {
220
- ...enqueueOptions,
221
- limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
222
- },
223
- window: crawlingContext.window,
224
- requestManager: await this.getRequestManager(),
225
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
226
- onSkippedRequest: this.handleSkippedRequest,
227
- originalRequestUrl: crawlingContext.request.url,
225
+ extractLinks,
226
+ enqueueLinks: async (options = {}) => {
227
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
228
+ enqueueStrategy: options.strategy,
228
229
  finalRequestUrl: crawlingContext.request.loadedUrl,
229
- })); // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
230
+ originalRequestUrl: crawlingContext.request.url,
231
+ userProvidedBaseUrl: options.baseUrl,
232
+ });
233
+ const urls = await extractLinks(options);
234
+ return addRequests(urls, {
235
+ ...options,
236
+ baseUrl,
237
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
238
+ });
230
239
  },
231
240
  async waitForSelector(selector, timeoutMs = 5_000) {
232
241
  const cheerio = await import('cheerio');
@@ -251,39 +260,6 @@ export class JSDOMCrawler extends HttpCrawler {
251
260
  };
252
261
  }
253
262
  }
254
- /** @internal */
255
- function containsEnqueueLinks(options) {
256
- return !!options.enqueueLinks;
257
- }
258
- /** @internal */
259
- export async function domCrawlerEnqueueLinks(options) {
260
- const { options: enqueueLinksOptions, window, originalRequestUrl, finalRequestUrl } = options;
261
- if (!window) {
262
- throw new Error('Cannot enqueue links because the JSDOM is not available.');
263
- }
264
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
265
- enqueueStrategy: enqueueLinksOptions?.strategy,
266
- finalRequestUrl,
267
- originalRequestUrl,
268
- userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
269
- });
270
- const urls = extractUrlsFromWindow(window, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
271
- if (containsEnqueueLinks(options)) {
272
- return options.enqueueLinks({
273
- urls,
274
- baseUrl,
275
- ...enqueueLinksOptions,
276
- });
277
- }
278
- return enqueueLinks({
279
- requestManager: options.requestManager,
280
- robotsTxtFile: options.robotsTxtFile,
281
- onSkippedRequest: options.onSkippedRequest,
282
- urls,
283
- baseUrl,
284
- ...enqueueLinksOptions,
285
- });
286
- }
287
263
  /**
288
264
  * Extracts URLs from a given Window object.
289
265
  * @ignore
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/jsdom",
3
- "version": "4.0.0-beta.121",
3
+ "version": "4.0.0-beta.123",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,14 +49,14 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
51
  "@apify/utilities": "^2.7.10",
52
- "@crawlee/http": "4.0.0-beta.121",
53
- "@crawlee/types": "4.0.0-beta.121",
54
- "@crawlee/utils": "4.0.0-beta.121",
52
+ "@crawlee/http": "4.0.0-beta.123",
53
+ "@crawlee/types": "4.0.0-beta.123",
54
+ "@crawlee/utils": "4.0.0-beta.123",
55
55
  "@types/jsdom": "^21.1.7",
56
56
  "cheerio": "^1.0.0",
57
57
  "jsdom": "^26.1.0",
58
- "ow": "^2.0.0",
59
- "tslib": "^2.8.1"
58
+ "tslib": "^2.8.1",
59
+ "zod": "^4.4.3"
60
60
  },
61
61
  "lerna": {
62
62
  "command": {
@@ -65,5 +65,5 @@
65
65
  }
66
66
  }
67
67
  },
68
- "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
68
+ "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
69
69
  }