@crawlee/browser 4.0.0-beta.121 → 4.0.0-beta.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/internals/browser-crawler.d.ts +122 -117
  2. package/internals/browser-crawler.js +35 -58
  3. package/internals/browser-launcher.d.ts +19 -16
  4. package/internals/browser-launcher.js +11 -10
  5. package/package.json +8 -8
package/internals/browser-crawler.d.ts CHANGED
@@ -1,9 +1,8 @@
1
- import type { BasicCrawlerOptions, BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, IRequestManager, LoadedRequest, Request, RequestHandler, RouterHandler, SkippedRequestCallback } from '@crawlee/basic';
1
+ import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, Request, RequestHandler, RouterHandler } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
3
  import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, CrawlerRemoteBrowserOptions, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
4
- import type { Awaitable, BatchAddRequestsResult, Dictionary, IBrowserPool } from '@crawlee/types';
5
- import type { RobotsTxtFile } from '@crawlee/utils';
6
- import type { ReadonlyDeep } from 'type-fest';
4
+ import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
+ import { z } from 'zod';
7
6
  import type { BrowserLaunchContext } from './browser-launcher.js';
8
7
  interface BaseResponse {
9
8
  status(): number;
@@ -29,10 +28,14 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
29
28
  * object (or return `{ gotoOptions: ... }`) to influence the navigation.
30
29
  */
31
30
  gotoOptions: GoToOptions;
31
+ /**
32
+ * Extracts URLs from the current page, without adding them to the request queue.
33
+ */
34
+ extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
32
35
  /**
33
36
  * Helper function for extracting URLs from the current page and adding them to the request queue.
34
37
  */
35
- enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
38
+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
36
39
  }
37
40
  export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
38
41
  export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
@@ -239,99 +242,121 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
239
242
  protected readonly ignoreShadowRoots: boolean;
240
243
  protected readonly ignoreIframes: boolean;
241
244
  protected static optionsShape: {
242
- // @ts-ignore optional peer dependency or compatibility with es2022
243
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
244
- // @ts-ignore optional peer dependency or compatibility with es2022
245
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
246
- // @ts-ignore optional peer dependency or compatibility with es2022
247
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
248
- // @ts-ignore optional peer dependency or compatibility with es2022
249
- launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
250
- // @ts-ignore optional peer dependency or compatibility with es2022
251
- headless: import("ow").AnyPredicate<string | boolean>;
252
- // @ts-ignore optional peer dependency or compatibility with es2022
253
- browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
254
- // @ts-ignore optional peer dependency or compatibility with es2022
255
- remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
258
- // @ts-ignore optional peer dependency or compatibility with es2022
259
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
260
- // @ts-ignore optional peer dependency or compatibility with es2022
261
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
262
- // @ts-ignore optional peer dependency or compatibility with es2022
263
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
266
- // @ts-ignore optional peer dependency or compatibility with es2022
267
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
268
- // @ts-ignore optional peer dependency or compatibility with es2022
269
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
270
- // @ts-ignore optional peer dependency or compatibility with es2022
271
- requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
272
- // @ts-ignore optional peer dependency or compatibility with es2022
273
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
274
- // @ts-ignore optional peer dependency or compatibility with es2022
275
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
276
- // @ts-ignore optional peer dependency or compatibility with es2022
277
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
278
- // @ts-ignore optional peer dependency or compatibility with es2022
279
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
280
- // @ts-ignore optional peer dependency or compatibility with es2022
281
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
282
- // @ts-ignore optional peer dependency or compatibility with es2022
283
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
284
- // @ts-ignore optional peer dependency or compatibility with es2022
285
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
286
- // @ts-ignore optional peer dependency or compatibility with es2022
287
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
288
- // @ts-ignore optional peer dependency or compatibility with es2022
289
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
290
- // @ts-ignore optional peer dependency or compatibility with es2022
291
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
292
- // @ts-ignore optional peer dependency or compatibility with es2022
293
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
294
- // @ts-ignore optional peer dependency or compatibility with es2022
295
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
296
- // @ts-ignore optional peer dependency or compatibility with es2022
297
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
298
- // @ts-ignore optional peer dependency or compatibility with es2022
299
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
300
- // @ts-ignore optional peer dependency or compatibility with es2022
301
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
302
- // @ts-ignore optional peer dependency or compatibility with es2022
303
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
304
- // @ts-ignore optional peer dependency or compatibility with es2022
305
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
306
- // @ts-ignore optional peer dependency or compatibility with es2022
307
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
308
- // @ts-ignore optional peer dependency or compatibility with es2022
309
- transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/basic").StorageWritePolicy> | undefined>;
310
- // @ts-ignore optional peer dependency or compatibility with es2022
311
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
312
- // @ts-ignore optional peer dependency or compatibility with es2022
313
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
314
- // @ts-ignore optional peer dependency or compatibility with es2022
315
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
316
- // @ts-ignore optional peer dependency or compatibility with es2022
317
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
318
- // @ts-ignore optional peer dependency or compatibility with es2022
319
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
320
- // @ts-ignore optional peer dependency or compatibility with es2022
321
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
322
- // @ts-ignore optional peer dependency or compatibility with es2022
323
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
324
- // @ts-ignore optional peer dependency or compatibility with es2022
325
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
326
- // @ts-ignore optional peer dependency or compatibility with es2022
327
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
328
- // @ts-ignore optional peer dependency or compatibility with es2022
329
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
330
- // @ts-ignore optional peer dependency or compatibility with es2022
331
- statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
332
- // @ts-ignore optional peer dependency or compatibility with es2022
333
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
245
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
246
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
247
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
248
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
249
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
250
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
251
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
252
+ browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
253
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
254
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
255
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
256
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
257
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
258
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
259
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
260
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
261
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
262
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
263
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
264
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
265
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
266
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
267
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
268
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
269
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
270
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
271
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
272
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
273
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
274
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
275
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
276
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
277
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
278
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
279
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
280
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
281
+ requestQueue: z.ZodOptional<z.ZodEnum<{
282
+ deferred: "deferred";
283
+ writeThrough: "writeThrough";
284
+ }>>;
285
+ }, z.core.$strict>]>>;
286
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
287
+ // @ts-ignore optional peer dependency or compatibility with es2022
288
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
289
+ // @ts-ignore optional peer dependency or compatibility with es2022
290
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
291
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
292
+ // @ts-ignore optional peer dependency or compatibility with es2022
293
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
294
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
295
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
296
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
297
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
298
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
299
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
300
+ id: z.ZodOptional<z.ZodString>;
334
301
  };
302
+ protected static optionsSchema: z.ZodObject<{
303
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
304
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
305
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
306
+ launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
307
+ headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
308
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
309
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
310
+ browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
311
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
312
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
313
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
314
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
315
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
316
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
317
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
318
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
319
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
320
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
321
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
322
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
323
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
324
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
325
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
326
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
327
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
328
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
329
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
330
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
331
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
332
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
333
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
334
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
335
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
336
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
337
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
338
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
339
+ requestQueue: z.ZodOptional<z.ZodEnum<{
340
+ deferred: "deferred";
341
+ writeThrough: "writeThrough";
342
+ }>>;
343
+ }, z.core.$strict>]>>;
344
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
345
+ // @ts-ignore optional peer dependency or compatibility with es2022
346
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
347
+ // @ts-ignore optional peer dependency or compatibility with es2022
348
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
349
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
350
+ // @ts-ignore optional peer dependency or compatibility with es2022
351
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
352
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
353
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
354
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
355
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
356
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
357
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
358
+ id: z.ZodOptional<z.ZodString>;
359
+ }, z.core.$strict>;
335
360
  /**
336
361
  * All `BrowserCrawler` parameters are passed via an options object.
337
362
  */
@@ -374,26 +399,6 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
374
399
  */
375
400
  teardown(): Promise<void>;
376
401
  }
377
- /** @internal */
378
- interface EnqueueLinksInternalOptions {
379
- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
380
- page: CommonPage;
381
- requestManager: IRequestManager;
382
- robotsTxtFile?: RobotsTxtFile;
383
- onSkippedRequest?: SkippedRequestCallback;
384
- originalRequestUrl: string;
385
- finalRequestUrl?: string;
386
- }
387
- /** @internal */
388
- interface BoundEnqueueLinksInternalOptions {
389
- enqueueLinks: BasicCrawlingContext['enqueueLinks'];
390
- options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
391
- originalRequestUrl: string;
392
- finalRequestUrl?: string;
393
- page: CommonPage;
394
- }
395
- /** @internal */
396
- export declare function browserCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
397
402
  /**
398
403
  * Extracts URLs from a given page.
399
404
  * @ignore
package/internals/browser-crawler.js CHANGED
@@ -1,8 +1,8 @@
1
- import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
2
  import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
3
3
  import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
4
4
  import { sleep } from '@crawlee/utils';
5
- import ow from 'ow';
5
+ import { z } from 'zod';
6
6
  import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
7
7
  const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
8
8
  const readContextField = (ctx, key) => ctx[key];
@@ -76,23 +76,27 @@ export class BrowserCrawler extends BasicCrawler {
76
76
  #saveResponseCookies;
77
77
  static optionsShape = {
78
78
  ...BasicCrawler.optionsShape,
79
- navigationTimeoutSecs: ow.optional.number.greaterThan(0),
80
- preNavigationHooks: ow.optional.array,
81
- postNavigationHooks: ow.optional.array,
82
- launchContext: ow.optional.object,
83
- headless: ow.optional.any(ow.boolean, ow.string),
84
- browserPool: ow.optional.object.validate(validators.browserPool),
85
- remoteBrowser: ow.optional.object,
86
- browserPoolOptions: ow.optional.object,
87
- saveResponseCookies: ow.optional.boolean,
88
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
79
+ navigationTimeoutSecs: schemas.anyNumber
80
+ .refine((value) => value > 0, 'Expected a number greater than 0')
81
+ .default(60),
82
+ preNavigationHooks: schemas.anyArray.default(() => []),
83
+ postNavigationHooks: schemas.anyArray.default(() => []),
84
+ launchContext: schemas.anyObject.default(() => ({})),
85
+ headless: z.union([z.boolean(), z.string()]).optional(),
86
+ browserPool: validators.browserPool.optional(),
87
+ remoteBrowser: schemas.anyObject.optional(),
88
+ browserPoolOptions: schemas.anyObject.optional(),
89
+ saveResponseCookies: z.boolean().default(true),
90
+ proxyConfiguration: validators.proxyConfiguration.optional(),
91
+ ignoreIframes: z.boolean().default(false),
92
+ ignoreShadowRoots: z.boolean().default(false),
89
93
  };
94
+ static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
90
95
  /**
91
96
  * All `BrowserCrawler` parameters are passed via an options object.
92
97
  */
93
98
  constructor(options) {
94
- ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
95
- const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
99
+ const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks, postNavigationHooks, headless, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
96
100
  const skipGuard = (action) => ({
97
101
  action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
98
102
  });
@@ -218,7 +222,10 @@ export class BrowserCrawler extends BasicCrawler {
218
222
  session: crawlingContext.session,
219
223
  });
220
224
  tryCancel();
221
- const contextEnqueueLinks = crawlingContext.enqueueLinks;
225
+ const addRequests = crawlingContext.addRequests;
226
+ const extractLinks = async (options) => {
227
+ return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
228
+ };
222
229
  return {
223
230
  page,
224
231
  get response() {
@@ -227,20 +234,20 @@ export class BrowserCrawler extends BasicCrawler {
227
234
  get gotoOptions() {
228
235
  throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
229
236
  },
230
- enqueueLinks: async (enqueueOptions = {}) => {
231
- return (await browserCrawlerEnqueueLinks({
232
- options: {
233
- ...enqueueOptions,
234
- limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
235
- },
236
- page,
237
- requestManager: await this.getRequestManager(),
238
- robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
239
- onSkippedRequest: this.handleSkippedRequest,
240
- originalRequestUrl: crawlingContext.request.url,
237
+ extractLinks,
238
+ enqueueLinks: async (options = {}) => {
239
+ const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
240
+ enqueueStrategy: options.strategy,
241
241
  finalRequestUrl: crawlingContext.request.loadedUrl,
242
- enqueueLinks: contextEnqueueLinks,
243
- })); // TODO make this type safe
242
+ originalRequestUrl: crawlingContext.request.url,
243
+ userProvidedBaseUrl: options.baseUrl,
244
+ });
245
+ const urls = await extractLinks(options);
246
+ return addRequests(urls, {
247
+ ...options,
248
+ baseUrl,
249
+ strategy: options.strategy ?? EnqueueStrategy.SameHostname,
250
+ });
244
251
  },
245
252
  };
246
253
  }
@@ -444,36 +451,6 @@ export class BrowserCrawler extends BasicCrawler {
444
451
  await super.teardown();
445
452
  }
446
453
  }
447
- /** @internal */
448
- function containsEnqueueLinks(options) {
449
- return !!options.enqueueLinks;
450
- }
451
- /** @internal */
452
- export async function browserCrawlerEnqueueLinks(options) {
453
- const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
454
- const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
455
- enqueueStrategy: enqueueLinksOptions?.strategy,
456
- finalRequestUrl,
457
- originalRequestUrl,
458
- userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
459
- });
460
- const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
461
- if (containsEnqueueLinks(options)) {
462
- return options.enqueueLinks({
463
- urls,
464
- baseUrl,
465
- ...enqueueLinksOptions,
466
- });
467
- }
468
- return enqueueLinks({
469
- requestManager: options.requestManager,
470
- robotsTxtFile: options.robotsTxtFile,
471
- onSkippedRequest: options.onSkippedRequest,
472
- urls,
473
- baseUrl,
474
- ...enqueueLinksOptions,
475
- });
476
- }
477
454
  /**
478
455
  * Extracts URLs from a given page.
479
456
  * @ignore
package/internals/browser-launcher.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import { Configuration } from '@crawlee/basic';
2
2
  import type { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';
3
3
  import type { Constructor, Dictionary } from '@crawlee/types';
4
+ import { z } from 'zod';
4
5
  export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
5
6
  /**
6
7
  * URL to an HTTP proxy server. It must define the port number,
@@ -78,23 +79,25 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
78
79
  Plugin: T;
79
80
  userAgent?: string;
80
81
  protected static optionsShape: {
81
- // @ts-ignore optional peer dependency or compatibility with es2022
82
- proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
83
- // @ts-ignore optional peer dependency or compatibility with es2022
84
- useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
85
- // @ts-ignore optional peer dependency or compatibility with es2022
86
- useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
87
- // @ts-ignore optional peer dependency or compatibility with es2022
88
- browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
89
- // @ts-ignore optional peer dependency or compatibility with es2022
90
- ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
91
- // @ts-ignore optional peer dependency or compatibility with es2022
92
- userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
93
- // @ts-ignore optional peer dependency or compatibility with es2022
94
- launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
95
- // @ts-ignore optional peer dependency or compatibility with es2022
96
- userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
82
+ proxyUrl: z.ZodOptional<z.ZodURL>;
83
+ useChrome: z.ZodOptional<z.ZodBoolean>;
84
+ useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
85
+ browserPerProxy: z.ZodOptional<z.ZodBoolean>;
86
+ ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
87
+ userDataDir: z.ZodOptional<z.ZodString>;
88
+ launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
89
+ userAgent: z.ZodOptional<z.ZodString>;
97
90
  };
91
+ protected static optionsSchema: z.ZodObject<{
92
+ proxyUrl: z.ZodOptional<z.ZodURL>;
93
+ useChrome: z.ZodOptional<z.ZodBoolean>;
94
+ useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
95
+ browserPerProxy: z.ZodOptional<z.ZodBoolean>;
96
+ ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
97
+ userDataDir: z.ZodOptional<z.ZodString>;
98
+ launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
99
+ userAgent: z.ZodOptional<z.ZodString>;
100
+ }, z.core.$strict>;
98
101
  static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T;
99
102
  /**
100
103
  * All `BrowserLauncher` parameters are passed via an launchContext object.
package/internals/browser-launcher.js CHANGED
@@ -1,8 +1,8 @@
1
1
  import fs from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import os from 'node:os';
4
- import { Configuration } from '@crawlee/basic';
5
- import ow from 'ow';
4
+ import { Configuration, schemas } from '@crawlee/basic';
5
+ import { z } from 'zod';
6
6
  const DEFAULT_VIEWPORT = {
7
7
  width: 1366,
8
8
  height: 768,
@@ -23,15 +23,16 @@ export class BrowserLauncher {
23
23
  Plugin;
24
24
  userAgent;
25
25
  static optionsShape = {
26
- proxyUrl: ow.optional.string.url,
27
- useChrome: ow.optional.boolean,
28
- useIncognitoPages: ow.optional.boolean,
29
- browserPerProxy: ow.optional.boolean,
30
- ignoreProxyCertificate: ow.optional.boolean,
31
- userDataDir: ow.optional.string,
32
- launchOptions: ow.optional.object,
33
- userAgent: ow.optional.string,
26
+ proxyUrl: z.url().optional(),
27
+ useChrome: z.boolean().optional(),
28
+ useIncognitoPages: z.boolean().optional(),
29
+ browserPerProxy: z.boolean().optional(),
30
+ ignoreProxyCertificate: z.boolean().optional(),
31
+ userDataDir: z.string().optional(),
32
+ launchOptions: schemas.anyObject.optional(),
33
+ userAgent: z.string().optional(),
34
34
  };
35
+ static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
35
36
  static requireLauncherOrThrow(launcher, apifyImageName) {
36
37
  try {
37
38
  return require(launcher); // eslint-disable-line
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-beta.121",
3
+ "version": "4.0.0-beta.123",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -48,13 +48,13 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
- "@crawlee/basic": "4.0.0-beta.121",
52
- "@crawlee/browser-pool": "4.0.0-beta.121",
53
- "@crawlee/types": "4.0.0-beta.121",
54
- "@crawlee/utils": "4.0.0-beta.121",
55
- "ow": "^2.0.0",
51
+ "@crawlee/basic": "4.0.0-beta.123",
52
+ "@crawlee/browser-pool": "4.0.0-beta.123",
53
+ "@crawlee/types": "4.0.0-beta.123",
54
+ "@crawlee/utils": "4.0.0-beta.123",
56
55
  "tslib": "^2.8.1",
57
- "type-fest": "^4.41.0"
56
+ "type-fest": "^4.41.0",
57
+ "zod": "^4.4.3"
58
58
  },
59
59
  "peerDependencies": {
60
60
  "playwright": "*",
@@ -75,5 +75,5 @@
75
75
  }
76
76
  }
77
77
  },
78
- "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
78
+ "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
79
79
  }