@crawlee/core 4.0.0-beta.121 → 4.0.0-beta.122

package/storages/request_queue.js

@@ -1,7 +1,7 @@
 import { inspect } from 'node:util';
 import { isAsyncIterable, isIterable } from '@crawlee/utils/internal';
 import { downloadListOfUrls } from '@crawlee/utils';
-import ow from 'ow';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
@@ -9,6 +9,7 @@ import { getObjectType } from '../debug.js';
 import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { activeStorageTransaction, rejectOperationInTransaction } from './transaction.js';
 import { drainRequestBatches } from './batched_adds.js';
 import { StorageStatsTracker } from './storage_stats.js';
@@ -20,6 +21,43 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @internal
  */
 const MAX_CACHED_REQUESTS = 2_000_000;
+const iterableSchema = z.custom((value) => isIterable(value) || isAsyncIterable(value), {
+    error: (issue) => `Expected an iterable or async iterable, got ${getObjectType(issue.input)}`,
+});
+const operationOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+});
+const addRequestsOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+    cache: z.boolean().default(true),
+});
+const addRequestsBatchedOptionsSchema = z.strictObject({
+    forefront: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().default(false),
+    batchSize: schemas.anyNumber.default(1000),
+    waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
+    maxNewRequests: schemas.anyNumber.optional(),
+});
+const newRequestLikeSchema = z.looseObject({
+    url: z.string(),
+    id: z.undefined().optional(),
+});
+const handledRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+    handledAt: z.string().optional(),
+});
+const reclaimedRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+});
+const uniqueKeySchema = z.string();
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -140,20 +178,14 @@ export class RequestQueue {
      */
     async addRequest(requestLike, options = {}) {
         const transaction = activeStorageTransaction();
-        ow(requestLike, ow.object);
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
+        parseArgument(requestLike, schemas.anyObject);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
         if ('requestsFromUrl' in requestLike) {
             const requests = await this.fetchRequestsFromUrl(requestLike);
             const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
-        ow(requestLike, ow.object.partialShape({
-            url: ow.string,
-            id: ow.undefined,
-        }));
+        parseArgument(requestLike, newRequestLikeSchema);
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
         if (transaction?.policy.requestQueue === 'deferred') {
             return this.addRequestDeferred(transaction, request, forefront);
@@ -346,14 +378,8 @@ export class RequestQueue {
      */
     async addRequests(requestsLike, options = {}) {
         const transaction = activeStorageTransaction();
-        ow(requestsLike, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            cache: ow.optional.boolean,
-        }));
-        const { forefront = false, cache = true } = options;
+        parseArgument(requestsLike, iterableSchema);
+        const { forefront, cache } = parseArgument(options, addRequestsOptionsSchema);
         const uniqueKeyToCacheKey = new Map();
         const getCachedRequestId = (uniqueKey) => {
             const cached = uniqueKeyToCacheKey.get(uniqueKey);
@@ -439,16 +465,8 @@ export class RequestQueue {
      * @param options Options for the request queue
      */
     async addRequestsBatched(requests, options = {}) {
-        ow(requests, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            waitForAllRequestsToBeAdded: ow.optional.boolean,
-            batchSize: ow.optional.number,
-            waitBetweenBatchesMillis: ow.optional.number,
-            maxNewRequests: ow.optional.number,
-        }));
+        parseArgument(requests, iterableSchema);
+        const { forefront, waitForAllRequestsToBeAdded, batchSize, waitBetweenBatchesMillis, maxNewRequests } = parseArgument(options, addRequestsBatchedOptionsSchema);
         const addRequest = this.addRequest.bind(this);
         async function* generateRequests() {
             for await (const opts of requests) {
@@ -467,7 +485,7 @@ export class RequestQueue {
                 }
                 if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
                     // Handle URL lists right away
-                    await addRequest(opts, { forefront: options.forefront });
+                    await addRequest(opts, { forefront });
                 }
                 else {
                     // Yield valid requests
@@ -477,10 +495,10 @@ export class RequestQueue {
         }
         return drainRequestBatches({
             items: generateRequests(),
-            batchSize: options.batchSize ?? 1000,
-            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
-            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
-            maxNewRequests: options.maxNewRequests,
+            batchSize,
+            waitBetweenBatchesMillis,
+            waitForAllRequestsToBeAdded,
+            maxNewRequests,
             /**
              * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
              * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
@@ -489,7 +507,7 @@ export class RequestQueue {
              */
             processChunk: async (chunk, isInitial) => {
                 const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
-                    forefront: options.forefront,
+                    forefront,
                     cache: isInitial,
                 });
                 if (unprocessedRequests.length > 0) {
@@ -514,7 +532,7 @@ export class RequestQueue {
      */
     async getRequest(uniqueKey) {
         const transaction = activeStorageTransaction();
-        ow(uniqueKey, ow.string);
+        parseArgument(uniqueKey, uniqueKeySchema);
         // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
         const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
         if (buffered) {
@@ -561,11 +579,7 @@ export class RequestQueue {
      */
     async markRequestAsHandled(request) {
         rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-            handledAt: ow.optional.string,
-        }));
+        parseArgument(request, handledRequestSchema);
         const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
         this.#statsTracker.add('writeCount');
@@ -594,16 +608,12 @@ export class RequestQueue {
      */
     async reclaimRequest(request, options = {}) {
         rejectOperationInTransaction('RequestQueue.reclaimRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-        }));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
+        parseArgument(request, reclaimedRequestSchema);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
         this.#statsTracker.add('writeCount');
-        const processedRequest = await this.backend.reclaimRequest(request, { forefront });
+        const processedRequest = await this.backend.reclaimRequest(request, {
+            forefront,
+        });
         // The request was not in progress — nothing to reclaim.
         if (!processedRequest) {
             return null;
@@ -837,14 +847,9 @@ export class RequestQueue {
      */
     static async open(identifier, options = {}) {
         tryCancel();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-            proxyConfiguration: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        const configuration = options.configuration ?? serviceLocator.getConfiguration();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        const configuration = parsedOptions.configuration ?? serviceLocator.getConfiguration();
         await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'RequestQueue');
         const queue = await serviceLocator
@@ -854,8 +859,8 @@ export class RequestQueue {
             backendOpener: () => storageBackend.createRequestQueueBackend(resolved),
             backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
         });
-        queue.#proxyConfiguration = options.proxyConfiguration;
-        queue.#httpClient = options.httpClient;
+        queue.#proxyConfiguration = parsedOptions.proxyConfiguration;
+        queue.#httpClient = parsedOptions.httpClient;
         return queue;
     }
 }

package/storages/sitemap_request_loader.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
 import { type ParseSitemapOptions } from '@crawlee/utils';
 import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';

package/storages/sitemap_request_loader.js

@@ -1,13 +1,27 @@
 import { Transform } from 'node:stream';
 import { parseSitemap } from '@crawlee/utils';
 import { minimatch } from 'minimatch';
-import ow from 'ow';
-import { constructUrlPatternObjects } from '../enqueue_links/shared.js';
+import { z } from 'zod';
+import { constructUrlPatternObjects, urlPatternSchema } from '../enqueue_links/shared.js';
 import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas } from '../validators.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
+const sitemapRequestLoaderOptionsSchema = z.strictObject({
+    sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
+    proxyUrl: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    signal: z.unknown().optional(),
+    timeoutMillis: schemas.anyNumber.optional(),
+    maxBufferSize: schemas.anyNumber.default(200),
+    parseSitemapOptions: z.looseObject({}).optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    persistenceOptions: z.looseObject({}).optional(),
+    httpClient: schemas.httpClient.optional(),
+});
 /** @internal */
 const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
 /**
@@ -80,21 +94,7 @@ export class SitemapRequestLoader {
     #persistenceOptions;
     /** @internal */
     constructor(options) {
-        const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
-        ow(options, ow.object.exactShape({
-            sitemapUrls: ow.array.ofType(ow.string),
-            proxyUrl: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            signal: ow.optional.any(),
-            timeoutMillis: ow.optional.number,
-            maxBufferSize: ow.optional.number,
-            parseSitemapOptions: ow.optional.object,
-            include: ow.optional.array.ofType(urlPatternValidator),
-            exclude: ow.optional.array.ofType(urlPatternValidator),
-            persistenceOptions: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        const { include, exclude } = options;
+        const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
         this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
         if (exclude?.length) {
             this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
@@ -102,11 +102,11 @@ export class SitemapRequestLoader {
         if (include?.length) {
             this.#urlPatternObjects.push(...constructUrlPatternObjects(include));
         }
-        this.#persistStateKey = options.persistStateKey;
-        this.#persistenceOptions = { enable: true, ...options.persistenceOptions };
-        this.#proxyUrl = options.proxyUrl;
-        this.#urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
-        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
+        this.#persistStateKey = persistStateKey;
+        this.#persistenceOptions = { enable: true, ...persistenceOptions };
+        this.#proxyUrl = proxyUrl;
+        this.#urlQueueStream = this.createNewStream(maxBufferSize);
+        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
         this.#events = serviceLocator.getEventManager();
         this.persistState = this.persistState.bind(this);
     }

package/storages/throttling_request_manager.js

@@ -1,11 +1,20 @@
 import { URL } from 'node:url';
-import ow from 'ow';
+import { z } from 'zod';
 import { PersistentRateLimitError } from '../errors.js';
 import { asyncifyIterable } from '../iterables.js';
 import { serviceLocator } from '../service_locator.js';
 import { normalizeHostname } from '../url.js';
+import { parseArgument, schemas } from '../validators.js';
 import { drainRequestBatches } from './batched_adds.js';
 import { RequestQueue } from './request_queue.js';
+const throttlingRequestManagerOptionsSchema = z.strictObject({
+    inner: schemas.anyObject,
+    domains: schemas.arrayOf(z.string().nonempty(), 'non-empty strings'),
+    requestManagerOpener: schemas.anyFunction.optional(),
+    baseDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDomainStallSecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+});
 /** Whether `manager` can pace requests per domain. */
 export function supportsDomainThrottling(manager) {
     const candidate = manager;
@@ -75,14 +84,7 @@ export class ThrottlingRequestManager {
     }
     constructor(options, config = serviceLocator.getConfiguration()) {
         this.config = config;
-        ow(options, ow.object.exactShape({
-            inner: ow.object,
-            domains: ow.array.ofType(ow.string.nonEmpty),
-            requestManagerOpener: ow.optional.function,
-            baseDelaySecs: ow.optional.number.positive,
-            maxDelaySecs: ow.optional.number.positive,
-            maxDomainStallSecs: ow.optional.number.positive,
-        }));
+        parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
         this.inner = options.inner;
         this.requestManagerOpener =
             options.requestManagerOpener ??

package/storages/utils.d.ts

@@ -1,4 +1,5 @@
-import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 /**

package/validators.d.ts

@@ -1,28 +1,25 @@
-import type { Dictionary } from '@crawlee/types';
+export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export declare const validators: {
-    browserPage: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    proxyConfiguration: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestList: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestQueue: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    browserPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    sessionPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPage: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    proxyConfiguration: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestList: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestQueue: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    sessionPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestManager: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    storageBackend: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    logger: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    httpClient: import("zod").ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>;
 };

package/validators.js

@@ -1,29 +1,17 @@
-import ow from 'ow';
+import { schemas } from '@crawlee/utils/internal';
+export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export const validators = {
     // Naming it browser page for future proofing with Playwright
-    browserPage: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('goto', 'evaluate', '$', 'on')),
-        message: (label) => `Expected argument '${label}' to be a Puppeteer Page, got something else.`,
-    }),
-    proxyConfiguration: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('newProxyInfo')),
-        message: (label) => `Expected argument '${label}' to implement the IProxyConfiguration interface (missing 'newProxyInfo'), got something else.`,
-    }),
-    requestList: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'persistState')),
-        message: (label) => `Expected argument '${label}' to be a RequestList, got something else.`,
-    }),
-    requestQueue: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'addRequest')),
-        message: (label) => `Expected argument '${label}' to be a RequestQueue, got something else.`,
-    }),
-    browserPool: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('newPage', 'closePage', 'extractPageState', 'injectPageState')),
-        message: (label) => `Expected argument '${label}' to implement the IBrowserPool interface (missing one of 'newPage', 'closePage', 'extractPageState', 'injectPageState'), got something else.`,
-    }),
-    sessionPool: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('getSession')),
-        message: (label) => `Expected argument '${label}' to implement the ISessionPool interface (missing 'getSession'), got something else.`,
-    }),
+    browserPage: schemas.objectWithKeys(['goto', 'evaluate', '$', 'on'], 'Expected a Puppeteer Page, got something else.'),
+    proxyConfiguration: schemas.objectWithKeys(['newProxyInfo'], "Expected an object implementing the IProxyConfiguration interface (missing 'newProxyInfo'), got something else."),
+    requestList: schemas.objectWithKeys(['fetchNextRequest', 'persistState'], 'Expected a RequestList, got something else.'),
+    requestQueue: schemas.objectWithKeys(['fetchNextRequest', 'addRequest'], 'Expected a RequestQueue, got something else.'),
+    browserPool: schemas.objectWithKeys(['newPage', 'closePage', 'extractPageState', 'injectPageState'], "Expected an object implementing the IBrowserPool interface (missing one of 'newPage', 'closePage', 'extractPageState', 'injectPageState'), got something else."),
+    sessionPool: schemas.objectWithKeys(['getSession'], "Expected an object implementing the ISessionPool interface (missing 'getSession'), got something else."),
+    requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequest', 'addRequestsBatched'], "Expected an object implementing the IRequestManager interface (missing one of 'fetchNextRequest', 'addRequest', 'addRequestsBatched'), got something else."),
+    storageBackend: schemas.objectWithKeys(['createDatasetBackend', 'createKeyValueStoreBackend', 'createRequestQueueBackend'], "Expected an object implementing the StorageBackend interface (missing one of 'createDatasetBackend', 'createKeyValueStoreBackend', 'createRequestQueueBackend'), got something else."),
+    logger: schemas.logger,
+    httpClient: schemas.httpClient,
 };