@crawlee/browser 4.0.0-beta.120 → 4.0.0-beta.122

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/internals/browser-crawler.d.ts +115 -92
  2. package/internals/browser-crawler.js +18 -14
  3. package/internals/browser-launcher.d.ts +19 -16
  4. package/internals/browser-launcher.js +11 -10
  5. package/package.json +8 -8
package/internals/browser-crawler.d.ts CHANGED
@@ -4,6 +4,7 @@ import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOpt
4
4
  import type { Awaitable, BatchAddRequestsResult, Dictionary, IBrowserPool } from '@crawlee/types';
5
5
  import type { RobotsTxtFile } from '@crawlee/utils';
6
6
  import type { ReadonlyDeep } from 'type-fest';
7
+ import { z } from 'zod';
7
8
  import type { BrowserLaunchContext } from './browser-launcher.js';
8
9
  interface BaseResponse {
9
10
  status(): number;
@@ -239,99 +240,121 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
239
240
  protected readonly ignoreShadowRoots: boolean;
240
241
  protected readonly ignoreIframes: boolean;
241
242
  protected static optionsShape: {
242
- // @ts-ignore optional peer dependency or compatibility with es2022
243
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
244
- // @ts-ignore optional peer dependency or compatibility with es2022
245
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
246
- // @ts-ignore optional peer dependency or compatibility with es2022
247
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
248
- // @ts-ignore optional peer dependency or compatibility with es2022
249
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
250
- // @ts-ignore optional peer dependency or compatibility with es2022
251
- headless: import("ow").AnyPredicate<string | boolean>;
252
- // @ts-ignore optional peer dependency or compatibility with es2022
253
- browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
254
- // @ts-ignore optional peer dependency or compatibility with es2022
255
- remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
258
- // @ts-ignore optional peer dependency or compatibility with es2022
259
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
260
- // @ts-ignore optional peer dependency or compatibility with es2022
261
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
262
- // @ts-ignore optional peer dependency or compatibility with es2022
263
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
266
- // @ts-ignore optional peer dependency or compatibility with es2022
267
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
268
- // @ts-ignore optional peer dependency or compatibility with es2022
269
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
270
- // @ts-ignore optional peer dependency or compatibility with es2022
271
- requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
272
- // @ts-ignore optional peer dependency or compatibility with es2022
273
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
274
- // @ts-ignore optional peer dependency or compatibility with es2022
275
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
276
- // @ts-ignore optional peer dependency or compatibility with es2022
277
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
278
- // @ts-ignore optional peer dependency or compatibility with es2022
279
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
280
- // @ts-ignore optional peer dependency or compatibility with es2022
281
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
282
- // @ts-ignore optional peer dependency or compatibility with es2022
283
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
284
- // @ts-ignore optional peer dependency or compatibility with es2022
285
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
286
- // @ts-ignore optional peer dependency or compatibility with es2022
287
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
288
- // @ts-ignore optional peer dependency or compatibility with es2022
289
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
290
- // @ts-ignore optional peer dependency or compatibility with es2022
291
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
292
- // @ts-ignore optional peer dependency or compatibility with es2022
293
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
294
- // @ts-ignore optional peer dependency or compatibility with es2022
295
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
296
- // @ts-ignore optional peer dependency or compatibility with es2022
297
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
298
- // @ts-ignore optional peer dependency or compatibility with es2022
299
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
300
- // @ts-ignore optional peer dependency or compatibility with es2022
301
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
302
- // @ts-ignore optional peer dependency or compatibility with es2022
303
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
304
- // @ts-ignore optional peer dependency or compatibility with es2022
305
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
306
- // @ts-ignore optional peer dependency or compatibility with es2022
307
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
308
- // @ts-ignore optional peer dependency or compatibility with es2022
309
- transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/basic").StorageWritePolicy> | undefined>;
310
- // @ts-ignore optional peer dependency or compatibility with es2022
311
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
312
- // @ts-ignore optional peer dependency or compatibility with es2022
313
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
314
- // @ts-ignore optional peer dependency or compatibility with es2022
315
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
316
- // @ts-ignore optional peer dependency or compatibility with es2022
317
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
318
- // @ts-ignore optional peer dependency or compatibility with es2022
319
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
320
- // @ts-ignore optional peer dependency or compatibility with es2022
321
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
322
- // @ts-ignore optional peer dependency or compatibility with es2022
323
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
324
- // @ts-ignore optional peer dependency or compatibility with es2022
325
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
326
- // @ts-ignore optional peer dependency or compatibility with es2022
327
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
328
- // @ts-ignore optional peer dependency or compatibility with es2022
329
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
330
- // @ts-ignore optional peer dependency or compatibility with es2022
331
- statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
332
- // @ts-ignore optional peer dependency or compatibility with es2022
333
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
243
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
244
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
245
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
246
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
247
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
248
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
249
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
250
+ browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
251
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
252
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
253
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
254
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
255
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
256
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
257
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
258
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
259
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
260
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
261
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
262
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
263
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
264
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
265
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
266
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
267
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
268
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
269
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
270
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
271
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
272
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
273
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
274
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
275
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
276
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
277
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
278
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
279
+ requestQueue: z.ZodOptional<z.ZodEnum<{
280
+ deferred: "deferred";
281
+ writeThrough: "writeThrough";
282
+ }>>;
283
+ }, z.core.$strict>]>>;
284
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
285
+ // @ts-ignore optional peer dependency or compatibility with es2022
286
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
287
+ // @ts-ignore optional peer dependency or compatibility with es2022
288
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
289
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
290
+ // @ts-ignore optional peer dependency or compatibility with es2022
291
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
292
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
293
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
294
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
295
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
296
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
297
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
298
+ id: z.ZodOptional<z.ZodString>;
334
299
  };
300
+ protected static optionsSchema: z.ZodObject<{
301
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
302
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
303
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
304
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
305
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
306
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
307
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
308
+ browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
309
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
310
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
311
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
312
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
313
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
314
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
315
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
316
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
317
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
318
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
319
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
320
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
321
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
322
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
323
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
324
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
325
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
326
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
327
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
328
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
329
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
330
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
331
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
332
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
333
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
334
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
335
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
336
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
337
+ requestQueue: z.ZodOptional<z.ZodEnum<{
338
+ deferred: "deferred";
339
+ writeThrough: "writeThrough";
340
+ }>>;
341
+ }, z.core.$strict>]>>;
342
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
343
+ // @ts-ignore optional peer dependency or compatibility with es2022
344
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
345
+ // @ts-ignore optional peer dependency or compatibility with es2022
346
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
347
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
348
+ // @ts-ignore optional peer dependency or compatibility with es2022
349
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
350
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
351
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
352
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
353
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
354
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
355
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
356
+ id: z.ZodOptional<z.ZodString>;
357
+ }, z.core.$strict>;
335
358
  /**
336
359
  * All `BrowserCrawler` parameters are passed via an options object.
337
360
  */
package/internals/browser-crawler.js CHANGED
@@ -1,8 +1,8 @@
1
- import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
2
  import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
3
3
  import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
4
4
  import { sleep } from '@crawlee/utils';
5
- import ow from 'ow';
5
+ import { z } from 'zod';
6
6
  import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
7
7
  const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
8
8
  const readContextField = (ctx, key) => ctx[key];
@@ -76,23 +76,27 @@ export class BrowserCrawler extends BasicCrawler {
76
76
  #saveResponseCookies;
77
77
  static optionsShape = {
78
78
  ...BasicCrawler.optionsShape,
79
- navigationTimeoutSecs: ow.optional.number.greaterThan(0),
80
- preNavigationHooks: ow.optional.array,
81
- postNavigationHooks: ow.optional.array,
82
- launchContext: ow.optional.object,
83
- headless: ow.optional.any(ow.boolean, ow.string),
84
- browserPool: ow.optional.object.validate(validators.browserPool),
85
- remoteBrowser: ow.optional.object,
86
- browserPoolOptions: ow.optional.object,
87
- saveResponseCookies: ow.optional.boolean,
88
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
79
+ navigationTimeoutSecs: schemas.anyNumber
80
+ .refine((value) => value > 0, 'Expected a number greater than 0')
81
+ .default(60),
82
+ preNavigationHooks: schemas.anyArray.default(() => []),
83
+ postNavigationHooks: schemas.anyArray.default(() => []),
84
+ launchContext: schemas.anyObject.default(() => ({})),
85
+ headless: z.union([z.boolean(), z.string()]).optional(),
86
+ browserPool: validators.browserPool.optional(),
87
+ remoteBrowser: schemas.anyObject.optional(),
88
+ browserPoolOptions: schemas.anyObject.optional(),
89
+ saveResponseCookies: z.boolean().default(true),
90
+ proxyConfiguration: validators.proxyConfiguration.optional(),
91
+ ignoreIframes: z.boolean().default(false),
92
+ ignoreShadowRoots: z.boolean().default(false),
89
93
  };
94
+ static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
90
95
  /**
91
96
  * All `BrowserCrawler` parameters are passed via an options object.
92
97
  */
93
98
  constructor(options) {
94
- ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
95
- const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
99
+ const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks, postNavigationHooks, headless, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
96
100
  const skipGuard = (action) => ({
97
101
  action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
98
102
  });
package/internals/browser-launcher.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { Configuration } from '@crawlee/basic';
2
2
  import type { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';
3
3
  import type { Constructor, Dictionary } from '@crawlee/types';
4
+ import { z } from 'zod';
4
5
  export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
5
6
  /**
6
7
  * URL to an HTTP proxy server. It must define the port number,
@@ -78,23 +79,25 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
78
79
  Plugin: T;
79
80
  userAgent?: string;
80
81
  protected static optionsShape: {
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
82
+ proxyUrl: z.ZodOptional<z.ZodURL>;
83
+ useChrome: z.ZodOptional<z.ZodBoolean>;
84
+ useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
85
+ browserPerProxy: z.ZodOptional<z.ZodBoolean>;
86
+ ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
87
+ userDataDir: z.ZodOptional<z.ZodString>;
88
+ launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
89
+ userAgent: z.ZodOptional<z.ZodString>;
97
90
  };
91
+ protected static optionsSchema: z.ZodObject<{
92
+ proxyUrl: z.ZodOptional<z.ZodURL>;
93
+ useChrome: z.ZodOptional<z.ZodBoolean>;
94
+ useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
95
+ browserPerProxy: z.ZodOptional<z.ZodBoolean>;
96
+ ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
97
+ userDataDir: z.ZodOptional<z.ZodString>;
98
+ launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
99
+ userAgent: z.ZodOptional<z.ZodString>;
100
+ }, z.core.$strict>;
98
101
  static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T;
99
102
  /**
100
103
  * All `BrowserLauncher` parameters are passed via an launchContext object.
package/internals/browser-launcher.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import fs from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import os from 'node:os';
4
- import { Configuration } from '@crawlee/basic';
5
- import ow from 'ow';
4
+ import { Configuration, schemas } from '@crawlee/basic';
5
+ import { z } from 'zod';
6
6
  const DEFAULT_VIEWPORT = {
7
7
  width: 1366,
8
8
  height: 768,
@@ -23,15 +23,16 @@ export class BrowserLauncher {
23
23
  Plugin;
24
24
  userAgent;
25
25
  static optionsShape = {
26
- proxyUrl: ow.optional.string.url,
27
- useChrome: ow.optional.boolean,
28
- useIncognitoPages: ow.optional.boolean,
29
- browserPerProxy: ow.optional.boolean,
30
- ignoreProxyCertificate: ow.optional.boolean,
31
- userDataDir: ow.optional.string,
32
- launchOptions: ow.optional.object,
33
- userAgent: ow.optional.string,
26
+ proxyUrl: z.url().optional(),
27
+ useChrome: z.boolean().optional(),
28
+ useIncognitoPages: z.boolean().optional(),
29
+ browserPerProxy: z.boolean().optional(),
30
+ ignoreProxyCertificate: z.boolean().optional(),
31
+ userDataDir: z.string().optional(),
32
+ launchOptions: schemas.anyObject.optional(),
33
+ userAgent: z.string().optional(),
34
34
  };
35
+ static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
35
36
  static requireLauncherOrThrow(launcher, apifyImageName) {
36
37
  try {
37
38
  return require(launcher); // eslint-disable-line
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-beta.120",
3
+ "version": "4.0.0-beta.122",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -48,13 +48,13 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
- "@crawlee/basic": "4.0.0-beta.120",
52
- "@crawlee/browser-pool": "4.0.0-beta.120",
53
- "@crawlee/types": "4.0.0-beta.120",
54
- "@crawlee/utils": "4.0.0-beta.120",
55
- "ow": "^2.0.0",
51
+ "@crawlee/basic": "4.0.0-beta.122",
52
+ "@crawlee/browser-pool": "4.0.0-beta.122",
53
+ "@crawlee/types": "4.0.0-beta.122",
54
+ "@crawlee/utils": "4.0.0-beta.122",
56
55
  "tslib": "^2.8.1",
57
- "type-fest": "^4.41.0"
56
+ "type-fest": "^4.41.0",
57
+ "zod": "^4.4.3"
58
58
  },
59
59
  "peerDependencies": {
60
60
  "playwright": "*",
@@ -75,5 +75,5 @@
75
75
  }
76
76
  }
77
77
  },
78
- "gitHead": "da5d427c4c1c9dcaea95b151e9c4c7310100885d"
78
+ "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
79
79
  }