@crawlee/basic 4.0.0-beta.121 → 4.0.0-beta.122

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/internals/basic-crawler.d.ts +91 -77
  2. package/internals/basic-crawler.js +62 -53
  3. package/internals/send-request.d.ts +2 -1
  4. package/package.json +9 -9
package/internals/basic-crawler.d.ts CHANGED
@@ -1,9 +1,10 @@
1
- import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
2
- import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue } from '@crawlee/core';
3
- import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
1
+ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
2
+ import { ConcurrencySystem, Configuration, ContextPipeline, Dataset, EventManager, RequestQueue } from '@crawlee/core';
3
+ import { BaseHttpClient } from '@crawlee/http-client';
4
+ import type { Awaitable, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
4
5
  import { RobotsTxtFile } from '@crawlee/utils';
5
- import { type BasePredicate } from 'ow';
6
6
  import type { ReadonlyDeep, SetRequired } from 'type-fest';
7
+ import { z } from 'zod';
7
8
  import { TimeoutError } from '@apify/timeout';
8
9
  export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
9
10
  }
@@ -514,80 +515,93 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
514
515
  protected readonly onSkippedRequest?: SkippedRequestCallback;
515
516
  protected readonly identity: CrawlerIdentity;
516
517
  protected static optionsShape: {
517
- // @ts-ignore optional peer dependency or compatibility with es2022
518
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
519
- // @ts-ignore optional peer dependency or compatibility with es2022
520
- extendContext: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
521
- // @ts-ignore optional peer dependency or compatibility with es2022
522
- requestList: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
523
- // @ts-ignore optional peer dependency or compatibility with es2022
524
- requestQueue: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
525
- // @ts-ignore optional peer dependency or compatibility with es2022
526
- requestManager: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
527
- // @ts-ignore optional peer dependency or compatibility with es2022
528
- requestHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
529
- // @ts-ignore optional peer dependency or compatibility with es2022
530
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & BasePredicate<number | undefined>;
531
- // @ts-ignore optional peer dependency or compatibility with es2022
532
- errorHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
533
- // @ts-ignore optional peer dependency or compatibility with es2022
534
- failedRequestHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
535
- // @ts-ignore optional peer dependency or compatibility with es2022
536
- maxRequestRetries: import("ow").NumberPredicate & BasePredicate<number | undefined>;
537
- // @ts-ignore optional peer dependency or compatibility with es2022
538
- sameDomainDelaySecs: import("ow").NumberPredicate & BasePredicate<number | undefined>;
539
- // @ts-ignore optional peer dependency or compatibility with es2022
540
- maxRequestsPerCrawl: import("ow").NumberPredicate & BasePredicate<number | undefined>;
541
- // @ts-ignore optional peer dependency or compatibility with es2022
542
- maxCrawlDepth: import("ow").NumberPredicate & BasePredicate<number | undefined>;
543
- // @ts-ignore optional peer dependency or compatibility with es2022
544
- taskLoopOptions: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
545
- // @ts-ignore optional peer dependency or compatibility with es2022
546
- concurrencySystem: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
547
- // @ts-ignore optional peer dependency or compatibility with es2022
548
- sessionPool: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
549
- // @ts-ignore optional peer dependency or compatibility with es2022
550
- proxyConfiguration: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
551
- // @ts-ignore optional peer dependency or compatibility with es2022
552
- statusMessageLoggingInterval: import("ow").NumberPredicate & BasePredicate<number | undefined>;
553
- // @ts-ignore optional peer dependency or compatibility with es2022
554
- statusMessageCallback: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
555
- // @ts-ignore optional peer dependency or compatibility with es2022
556
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
557
- // @ts-ignore optional peer dependency or compatibility with es2022
558
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
559
- // @ts-ignore optional peer dependency or compatibility with es2022
560
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
561
- // @ts-ignore optional peer dependency or compatibility with es2022
562
- retryOnBlocked: import("ow").BooleanPredicate & BasePredicate<boolean | undefined>;
563
- // @ts-ignore optional peer dependency or compatibility with es2022
564
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
565
- transactionalStorage: BasePredicate<boolean | Partial<StorageWritePolicy> | undefined>;
566
- // @ts-ignore optional peer dependency or compatibility with es2022
567
- onSkippedRequest: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
568
- // @ts-ignore optional peer dependency or compatibility with es2022
569
- httpClient: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
570
- // @ts-ignore optional peer dependency or compatibility with es2022
571
- configuration: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
572
- // @ts-ignore optional peer dependency or compatibility with es2022
573
- storageBackend: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
574
- // @ts-ignore optional peer dependency or compatibility with es2022
575
- eventManager: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
576
- // @ts-ignore optional peer dependency or compatibility with es2022
577
- logger: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
578
- // @ts-ignore optional peer dependency or compatibility with es2022
579
- minConcurrency: import("ow").NumberPredicate & BasePredicate<number | undefined>;
580
- // @ts-ignore optional peer dependency or compatibility with es2022
581
- maxConcurrency: import("ow").NumberPredicate & BasePredicate<number | undefined>;
582
- // @ts-ignore optional peer dependency or compatibility with es2022
583
- maxRequestsPerMinute: import("ow").NumberPredicate & BasePredicate<number | undefined>;
584
- // @ts-ignore optional peer dependency or compatibility with es2022
585
- keepAlive: import("ow").BooleanPredicate & BasePredicate<boolean | undefined>;
586
- // @ts-ignore optional peer dependency or compatibility with es2022
587
- statistics: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
588
- // @ts-ignore optional peer dependency or compatibility with es2022
589
- id: import("ow").StringPredicate & BasePredicate<string | undefined>;
518
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
519
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
520
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
521
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
522
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
523
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
524
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
525
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
526
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
527
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
528
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
529
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
530
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
531
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
532
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
533
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
534
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
535
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
536
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
537
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
538
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
539
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
540
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
541
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
542
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
543
+ requestQueue: z.ZodOptional<z.ZodEnum<{
544
+ deferred: "deferred";
545
+ writeThrough: "writeThrough";
546
+ }>>;
547
+ }, z.core.$strict>]>>;
548
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
549
+ httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
550
+ configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
551
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
552
+ eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
553
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
554
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
555
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
556
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
557
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
558
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
559
+ id: z.ZodOptional<z.ZodString>;
590
560
  };
561
+ protected static optionsSchema: z.ZodObject<{
562
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
563
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
564
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
565
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
566
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
567
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
568
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
569
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
570
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
571
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
572
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
573
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
574
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
575
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
576
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
577
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
578
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
579
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
580
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
581
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
582
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
583
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
584
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
585
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
586
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
587
+ requestQueue: z.ZodOptional<z.ZodEnum<{
588
+ deferred: "deferred";
589
+ writeThrough: "writeThrough";
590
+ }>>;
591
+ }, z.core.$strict>]>>;
592
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
593
+ httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
594
+ configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
595
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
596
+ eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
597
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
598
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
599
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
600
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
601
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
602
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
603
+ id: z.ZodOptional<z.ZodString>;
604
+ }, z.core.$strict>;
591
605
  /**
592
606
  * All `BasicCrawler` parameters are passed via an options object.
593
607
  */
package/internals/basic-crawler.js CHANGED
@@ -1,19 +1,20 @@
1
1
  import { mkdir, writeFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
- import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, EventType, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
4
- import { FetchHttpClient } from '@crawlee/http-client';
3
+ import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, Configuration, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
4
+ import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
5
5
  import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
6
6
  import { RobotsTxtFile } from '@crawlee/utils';
7
- import ow, { ArgumentError } from 'ow';
8
7
  import { getDomain } from 'tldts';
8
+ import { z } from 'zod';
9
9
  import { LruCache } from '@apify/datastructures';
10
10
  import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
11
11
  import { cryptoRandomObjectId } from '@apify/utilities';
12
12
  import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
13
13
  import { createSendRequest } from './send-request.js';
14
- class LazyDefaultHttpClient {
14
+ class LazyDefaultHttpClient extends BaseHttpClient {
15
15
  #delegatePromise;
16
16
  constructor(options) {
17
+ super(options);
17
18
  this.#delegatePromise = import('@crawlee/impit-client')
18
19
  .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
19
20
  .catch(() => {
@@ -22,6 +23,9 @@ class LazyDefaultHttpClient {
22
23
  return new FetchHttpClient(options);
23
24
  });
24
25
  }
26
+ fetch() {
27
+ throw new Error('LazyDefaultHttpClient delegates `sendRequest` entirely; `fetch` is never called.');
28
+ }
25
29
  async sendRequest(...args) {
26
30
  return (await this.#delegatePromise).sendRequest(...args);
27
31
  }
@@ -200,72 +204,77 @@ export class BasicCrawler {
200
204
  identity;
201
205
  #contextPipelineOptions;
202
206
  static optionsShape = {
203
- contextPipelineBuilder: ow.optional.object,
204
- extendContext: ow.optional.function,
205
- requestList: ow.optional.object.validate(validators.requestList),
206
- requestQueue: ow.optional.object.validate(validators.requestQueue),
207
- requestManager: ow.optional.object,
207
+ contextPipelineBuilder: schemas.anyObject.optional(),
208
+ extendContext: schemas.anyFunction.optional(),
209
+ requestList: validators.requestList.optional(),
210
+ requestQueue: validators.requestQueue.optional(),
211
+ requestManager: validators.requestManager.optional(),
208
212
  // Subclasses override this function instead of passing it
209
213
  // in constructor, so this validation needs to apply only
210
214
  // if the user creates an instance of BasicCrawler directly.
211
- requestHandler: ow.optional.function,
212
- requestHandlerTimeoutSecs: ow.optional.number,
213
- errorHandler: ow.optional.function,
214
- failedRequestHandler: ow.optional.function,
215
- maxRequestRetries: ow.optional.number,
216
- sameDomainDelaySecs: ow.optional.number,
217
- maxRequestsPerCrawl: ow.optional.number,
218
- maxCrawlDepth: ow.optional.number,
219
- taskLoopOptions: ow.optional.object,
220
- concurrencySystem: ow.optional.object,
221
- sessionPool: ow.optional.object.validate(validators.sessionPool),
222
- proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
223
- statusMessageLoggingInterval: ow.optional.number,
224
- statusMessageCallback: ow.optional.function,
225
- additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
226
- ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
227
- blockedStatusCodes: ow.optional.array.ofType(ow.number),
228
- retryOnBlocked: ow.optional.boolean,
229
- respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
230
- transactionalStorage: ow.optional.any(ow.boolean, ow.object.exactShape({
231
- requestQueue: ow.optional.string.oneOf(['deferred', 'writeThrough']),
232
- })),
233
- onSkippedRequest: ow.optional.function,
234
- httpClient: ow.optional.object,
235
- configuration: ow.optional.object,
236
- storageBackend: ow.optional.object,
237
- eventManager: ow.optional.object,
238
- logger: ow.optional.object,
215
+ requestHandler: schemas.anyFunction.optional(),
216
+ requestHandlerTimeoutSecs: schemas.anyNumber.optional(),
217
+ errorHandler: schemas.anyFunction.optional(),
218
+ failedRequestHandler: schemas.anyFunction.optional(),
219
+ maxRequestRetries: schemas.anyNumber.default(3),
220
+ sameDomainDelaySecs: schemas.anyNumber.default(0),
221
+ maxRequestsPerCrawl: schemas.anyNumber.optional(),
222
+ maxCrawlDepth: schemas.anyNumber.optional(),
223
+ // No zod default — subclasses provide their own fallback (e.g. HTTP-optimized pool options).
224
+ taskLoopOptions: schemas.anyObject.optional(),
225
+ concurrencySystem: schemas.anyObject.optional(),
226
+ sessionPool: validators.sessionPool.optional(),
227
+ proxyConfiguration: validators.proxyConfiguration.optional(),
228
+ statusMessageLoggingInterval: schemas.anyNumber.default(10),
229
+ statusMessageCallback: schemas.anyFunction.optional(),
230
+ additionalHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
231
+ ignoreHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
232
+ blockedStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').optional(),
233
+ retryOnBlocked: z.boolean().default(false),
234
+ respectRobotsTxtFile: z.union([z.boolean(), schemas.anyObject]).default(false),
235
+ transactionalStorage: z
236
+ .union([z.boolean(), z.strictObject({ requestQueue: z.enum(['deferred', 'writeThrough']).optional() })])
237
+ .optional(),
238
+ onSkippedRequest: schemas.anyFunction.optional(),
239
+ httpClient: schemas.httpClient.optional(),
240
+ configuration: z.instanceof(Configuration).optional(),
241
+ storageBackend: validators.storageBackend.optional(),
242
+ eventManager: z.instanceof(EventManager).optional(),
243
+ logger: validators.logger.optional(),
239
244
  // AutoscaledPool shorthands
240
- minConcurrency: ow.optional.number,
241
- maxConcurrency: ow.optional.number,
242
- maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
243
- keepAlive: ow.optional.boolean,
244
- statistics: ow.optional.object,
245
- id: ow.optional.string,
245
+ minConcurrency: schemas.anyNumber.optional(),
246
+ maxConcurrency: schemas.anyNumber.optional(),
247
+ maxRequestsPerMinute: schemas.anyNumber
248
+ .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
249
+ .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
250
+ .optional(),
251
+ keepAlive: z.boolean().optional(),
252
+ statistics: schemas.anyObject.optional(),
253
+ id: z.string().optional(),
246
254
  };
255
+ static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
247
256
  /**
248
257
  * All `BasicCrawler` parameters are passed via an options object.
249
258
  */
250
259
  constructor(options = {}) {
251
- ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
260
+ const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');
252
261
  const {
253
262
  // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
254
263
  requestList,
255
264
  // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
256
- requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
265
+ requestQueue, requestManager, maxRequestRetries, sameDomainDelaySecs, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes, ignoreHttpErrorStatusCodes,
257
266
  // Service locator options
258
267
  configuration, storageBackend, eventManager, logger,
259
268
  // AutoscaledPool shorthands
260
- minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statistics, httpClient, id, } = options;
269
+ minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
261
270
  // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
262
271
  // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
263
272
  // hammering a site.
264
273
  if (concurrencySystem !== undefined &&
265
274
  (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
266
- throw new ArgumentError('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
275
+ throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
267
276
  '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
268
- 'replaces. Pass them to the `ConcurrencySystem` constructor instead.', this.constructor);
277
+ 'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
269
278
  }
270
279
  // Create per-crawler service locator if custom services were provided.
271
280
  // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -287,8 +296,8 @@ export class BasicCrawler {
287
296
  try {
288
297
  serviceLocatorScope.enterScope();
289
298
  this.#contextPipelineOptions = {
290
- contextPipelineBuilder: options.contextPipelineBuilder,
291
- extendContext: options.extendContext,
299
+ contextPipelineBuilder: parsedOptions.contextPipelineBuilder,
300
+ extendContext: parsedOptions.extendContext,
292
301
  };
293
302
  this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
294
303
  // Initialize the Configuration instance to avoid lazy loading in the components
@@ -1009,9 +1018,9 @@ export class BasicCrawler {
1009
1018
  const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
1010
1019
  const maxCrawlDepth = this.maxCrawlDepth;
1011
1020
  const validateRequestUserData = this.validateRequestUserData.bind(this);
1012
- ow(requests, ow.object
1013
- .is((value) => isIterable(value) || isAsyncIterable(value))
1014
- .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
1021
+ if (!isIterable(requests) && !isAsyncIterable(requests)) {
1022
+ throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
1023
+ }
1015
1024
  async function* filteredRequests() {
1016
1025
  for await (const request of requests) {
1017
1026
  const url = typeof request === 'string' ? request : request.url;
package/internals/send-request.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type { Request as CrawleeRequest } from '@crawlee/core';
2
- import type { BaseHttpClient, HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';
2
+ import type { BaseHttpClient } from '@crawlee/http-client';
3
+ import type { HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';
3
4
  /**
4
5
  * Prepares a function to be used as the `sendRequest` context helper.
5
6
  *
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.121",
3
+ "version": "4.0.0-beta.122",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -42,18 +42,18 @@
42
42
  "@apify/datastructures": "^2.0.0",
43
43
  "@apify/timeout": "^0.4.4",
44
44
  "@apify/utilities": "^2.15.5",
45
- "@crawlee/core": "4.0.0-beta.121",
46
- "@crawlee/http-client": "4.0.0-beta.121",
47
- "@crawlee/types": "4.0.0-beta.121",
48
- "@crawlee/utils": "4.0.0-beta.121",
45
+ "@crawlee/core": "4.0.0-beta.122",
46
+ "@crawlee/http-client": "4.0.0-beta.122",
47
+ "@crawlee/types": "4.0.0-beta.122",
48
+ "@crawlee/utils": "4.0.0-beta.122",
49
49
  "csv-stringify": "^6.5.2",
50
- "ow": "^2.0.0",
51
50
  "tldts": "^7.0.6",
52
51
  "tslib": "^2.8.1",
53
- "type-fest": "^4.41.0"
52
+ "type-fest": "^4.41.0",
53
+ "zod": "^4.4.3"
54
54
  },
55
55
  "optionalDependencies": {
56
- "@crawlee/impit-client": "^4.0.0-beta.121"
56
+ "@crawlee/impit-client": "^4.0.0-beta.122"
57
57
  },
58
58
  "lerna": {
59
59
  "command": {
@@ -62,5 +62,5 @@
62
62
  }
63
63
  }
64
64
  },
65
- "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
65
+ "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
66
66
  }