@crawlee/jsdom 4.0.0-beta.120 → 4.0.0-beta.122

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/internals/jsdom-crawler.d.ts +119 -96
  2. package/internals/jsdom-crawler.js +8 -7
  3. package/package.json +7 -7
package/internals/jsdom-crawler.d.ts CHANGED
@@ -1,10 +1,11 @@
1
1
  import type { BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, IRequestManager, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
2
2
  import { HttpCrawler } from '@crawlee/http';
3
- import type { Dictionary } from '@crawlee/types';
3
+ import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
4
4
  import { type CheerioRoot } from '@crawlee/utils/internal';
5
5
  import { type RobotsTxtFile } from '@crawlee/utils';
6
6
  import type { DOMWindow } from 'jsdom';
7
7
  import { VirtualConsole } from 'jsdom';
8
+ import { z } from 'zod';
8
9
  export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
9
10
  JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
10
11
  ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, JSDOMCrawlingContext<UserData, JSONData> & ContextExtension>;
@@ -54,107 +55,129 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
54
55
  * ```
55
56
  */
56
57
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
58
+ /**
59
+ * Helper function for extracting URLs from the parsed HTML and adding them to the request queue.
60
+ */
61
+ enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
57
62
  }
58
63
  export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
59
64
  JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
60
65
  export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<JSDOMCrawlingContext['request']>>> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext, Routes> {
61
66
  #private;
62
67
  protected static optionsShape: {
63
- // @ts-ignore optional peer dependency or compatibility with es2022
64
- runScripts: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
65
- // @ts-ignore optional peer dependency or compatibility with es2022
66
- hideInternalConsole: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
67
- // @ts-ignore optional peer dependency or compatibility with es2022
68
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
69
- // @ts-ignore optional peer dependency or compatibility with es2022
70
- ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
71
- // @ts-ignore optional peer dependency or compatibility with es2022
72
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
73
- // @ts-ignore optional peer dependency or compatibility with es2022
74
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
75
- // @ts-ignore optional peer dependency or compatibility with es2022
76
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
77
- // @ts-ignore optional peer dependency or compatibility with es2022
78
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
79
- // @ts-ignore optional peer dependency or compatibility with es2022
80
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
97
- // @ts-ignore optional peer dependency or compatibility with es2022
98
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
99
- // @ts-ignore optional peer dependency or compatibility with es2022
100
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
101
- // @ts-ignore optional peer dependency or compatibility with es2022
102
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
103
- // @ts-ignore optional peer dependency or compatibility with es2022
104
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
105
- // @ts-ignore optional peer dependency or compatibility with es2022
106
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
107
- // @ts-ignore optional peer dependency or compatibility with es2022
108
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
109
- // @ts-ignore optional peer dependency or compatibility with es2022
110
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
111
- // @ts-ignore optional peer dependency or compatibility with es2022
112
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
113
- // @ts-ignore optional peer dependency or compatibility with es2022
114
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
115
- // @ts-ignore optional peer dependency or compatibility with es2022
116
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
117
- // @ts-ignore optional peer dependency or compatibility with es2022
118
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
119
- // @ts-ignore optional peer dependency or compatibility with es2022
120
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
121
- // @ts-ignore optional peer dependency or compatibility with es2022
122
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
123
- // @ts-ignore optional peer dependency or compatibility with es2022
124
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
125
- // @ts-ignore optional peer dependency or compatibility with es2022
126
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
127
- // @ts-ignore optional peer dependency or compatibility with es2022
128
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
129
- // @ts-ignore optional peer dependency or compatibility with es2022
130
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
131
- // @ts-ignore optional peer dependency or compatibility with es2022
132
- transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/http").StorageWritePolicy> | undefined>;
133
- // @ts-ignore optional peer dependency or compatibility with es2022
134
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
135
- // @ts-ignore optional peer dependency or compatibility with es2022
136
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
137
- // @ts-ignore optional peer dependency or compatibility with es2022
138
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
139
- // @ts-ignore optional peer dependency or compatibility with es2022
140
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
141
- // @ts-ignore optional peer dependency or compatibility with es2022
142
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
143
- // @ts-ignore optional peer dependency or compatibility with es2022
144
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
145
- // @ts-ignore optional peer dependency or compatibility with es2022
146
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
147
- // @ts-ignore optional peer dependency or compatibility with es2022
148
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
149
- // @ts-ignore optional peer dependency or compatibility with es2022
150
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
151
- // @ts-ignore optional peer dependency or compatibility with es2022
152
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
153
- // @ts-ignore optional peer dependency or compatibility with es2022
154
- statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
155
- // @ts-ignore optional peer dependency or compatibility with es2022
156
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
68
+ runScripts: z.ZodOptional<z.ZodBoolean>;
69
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
70
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
71
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
72
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
73
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
74
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
75
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
76
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
77
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
78
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
79
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
80
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
81
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
82
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
83
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
84
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
85
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
86
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
87
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
88
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
89
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
90
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
91
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
92
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
93
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
94
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
95
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
96
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
97
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
98
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
99
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
100
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
101
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
102
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
103
+ requestQueue: z.ZodOptional<z.ZodEnum<{
104
+ deferred: "deferred";
105
+ writeThrough: "writeThrough";
106
+ }>>;
107
+ }, z.core.$strict>]>>;
108
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
109
+ // @ts-ignore optional peer dependency or compatibility with es2022
110
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
111
+ // @ts-ignore optional peer dependency or compatibility with es2022
112
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
113
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
114
+ // @ts-ignore optional peer dependency or compatibility with es2022
115
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
116
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
117
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
118
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
119
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
120
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
121
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
122
+ id: z.ZodOptional<z.ZodString>;
157
123
  };
124
+ protected static optionsSchema: z.ZodObject<{
125
+ runScripts: z.ZodOptional<z.ZodBoolean>;
126
+ hideInternalConsole: z.ZodOptional<z.ZodBoolean>;
127
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
128
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
129
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
130
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
131
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
132
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
133
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
134
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
135
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
136
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
137
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
138
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
139
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
140
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
141
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
142
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
143
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
144
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
145
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
146
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
147
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
148
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
149
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
150
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
151
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
152
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
153
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
154
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
155
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
156
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
157
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
158
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
159
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
160
+ requestQueue: z.ZodOptional<z.ZodEnum<{
161
+ deferred: "deferred";
162
+ writeThrough: "writeThrough";
163
+ }>>;
164
+ }, z.core.$strict>]>>;
165
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
166
+ // @ts-ignore optional peer dependency or compatibility with es2022
167
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
168
+ // @ts-ignore optional peer dependency or compatibility with es2022
169
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/http").Configuration, import("@crawlee/http").Configuration>>;
170
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
171
+ // @ts-ignore optional peer dependency or compatibility with es2022
172
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/http").EventManager, import("@crawlee/http").EventManager>>;
173
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
174
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
175
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
176
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
177
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
178
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
179
+ id: z.ZodOptional<z.ZodString>;
180
+ }, z.core.$strict>;
158
181
  constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext, any, any, Routes>);
159
182
  // @ts-ignore optional peer dependency or compatibility with es2022
160
183
  protected buildContextPipeline(): import("@crawlee/http").ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
@@ -162,7 +185,7 @@ export declare class JSDOMCrawler<ContextExtension = Dictionary<never>, Extended
162
185
  readonly body: string;
163
186
  readonly document: Document;
164
187
  } & {
165
- enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
188
+ enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
166
189
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
167
190
  // @ts-ignore optional peer dependency or compatibility with es2022
168
191
  parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<import("cheerio").CheerioAPI>;
package/internals/jsdom-crawler.js CHANGED
@@ -1,7 +1,7 @@
1
- import { enqueueLinks, HttpCrawler, NavigationSkippedError, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
1
+ import { enqueueLinks, HttpCrawler, NavigationSkippedError, parseArgument, resolveBaseUrlForEnqueueLinksFiltering, Router, tryAbsoluteURL, } from '@crawlee/http';
2
2
  import { sleep } from '@crawlee/utils';
3
3
  import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
4
- import ow from 'ow';
4
+ import { z } from 'zod';
5
5
  import { addTimeoutToPromise } from '@apify/timeout';
6
6
  /**
7
7
  * Provides a framework for the parallel crawling of web pages using plain HTTP requests and
@@ -86,14 +86,15 @@ const resources = new ResourceLoader({
86
86
  export class JSDOMCrawler extends HttpCrawler {
87
87
  static optionsShape = {
88
88
  ...HttpCrawler.optionsShape,
89
- runScripts: ow.optional.boolean,
90
- hideInternalConsole: ow.optional.boolean,
89
+ runScripts: z.boolean().optional(),
90
+ hideInternalConsole: z.boolean().optional(),
91
91
  };
92
+ static optionsSchema = z.strictObject(JSDOMCrawler.optionsShape);
92
93
  #runScripts;
93
94
  #hideInternalConsole;
94
95
  #virtualConsole = null;
95
96
  constructor(options = {}) {
96
- const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = options;
97
+ const { runScripts = false, hideInternalConsole = false, contextPipelineBuilder, ...httpOptions } = parseArgument(options, JSDOMCrawler.optionsSchema, 'JSDOMCrawlerOptions');
97
98
  super({
98
99
  ...httpOptions,
99
100
  contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
@@ -215,7 +216,7 @@ export class JSDOMCrawler extends HttpCrawler {
215
216
  async addHelpers(crawlingContext) {
216
217
  return {
217
218
  enqueueLinks: async (enqueueOptions) => {
218
- return domCrawlerEnqueueLinks({
219
+ return (await domCrawlerEnqueueLinks({
219
220
  options: {
220
221
  ...enqueueOptions,
221
222
  limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
@@ -226,7 +227,7 @@ export class JSDOMCrawler extends HttpCrawler {
226
227
  onSkippedRequest: this.handleSkippedRequest,
227
228
  originalRequestUrl: crawlingContext.request.url,
228
229
  finalRequestUrl: crawlingContext.request.loadedUrl,
229
- });
230
+ })); // TODO make this type safe, see https://github.com/apify/crawlee/issues/4024
230
231
  },
231
232
  async waitForSelector(selector, timeoutMs = 5_000) {
232
233
  const cheerio = await import('cheerio');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/jsdom",
3
- "version": "4.0.0-beta.120",
3
+ "version": "4.0.0-beta.122",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,14 +49,14 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
51
  "@apify/utilities": "^2.7.10",
52
- "@crawlee/http": "4.0.0-beta.120",
53
- "@crawlee/types": "4.0.0-beta.120",
54
- "@crawlee/utils": "4.0.0-beta.120",
52
+ "@crawlee/http": "4.0.0-beta.122",
53
+ "@crawlee/types": "4.0.0-beta.122",
54
+ "@crawlee/utils": "4.0.0-beta.122",
55
55
  "@types/jsdom": "^21.1.7",
56
56
  "cheerio": "^1.0.0",
57
57
  "jsdom": "^26.1.0",
58
- "ow": "^2.0.0",
59
- "tslib": "^2.8.1"
58
+ "tslib": "^2.8.1",
59
+ "zod": "^4.4.3"
60
60
  },
61
61
  "lerna": {
62
62
  "command": {
@@ -65,5 +65,5 @@
65
65
  }
66
66
  }
67
67
  },
68
- "gitHead": "da5d427c4c1c9dcaea95b151e9c4c7310100885d"
68
+ "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
69
69
  }