@crawlee/core 4.0.0-beta.121 → 4.0.0-beta.123

package/storages/key_value_store.js

@@ -1,8 +1,9 @@
-import ow, { ArgumentError } from 'ow';
+import { z } from 'zod';
 import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
 import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { activeStorageTransaction, operationRejectedInTransaction, rejectOperationInTransaction, snapshotValue, withDirectStorageAccess, } from './transaction.js';
 import { parseValue, serializeValue } from './key_value_store_codec.js';
 import { StorageStatsTracker } from './storage_stats.js';
@@ -11,6 +12,20 @@ import { createDualIterable, purgeDefaultStorages } from './utils.js';
 import { isBuffer, isStream } from '../byte_utils.js';
 /** @internal */
 const KVS_KEYS_DEFAULT_LIMIT = 1000;
+const keySchema = z.string().nonempty();
+const setValueKeySchema = z.string().nonempty().regex(KEY_VALUE_STORE_KEY_REGEX, {
+    message: `The "key" argument must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
+});
+const recordOptionsSchema = z.strictObject({
+    contentType: z.string().nonempty().optional(),
+});
+const iteratorOptionsSchema = z.strictObject({
+    prefix: z.string().optional(),
+});
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+});
 /**
  * The `KeyValueStore` class represents a key-value store, a simple data storage that is used
  * for saving and reading data records or files. Each data record is
@@ -132,7 +147,7 @@ export class KeyValueStore {
      */
     async getValue(key, defaultValue) {
         tryCancel();
-        ow(key, ow.string.nonEmpty);
+        parseArgument(key, keySchema);
         const record = await this.readRecord(key);
         // A missing record falls back to the default; a record that parses to a falsy value (including
         // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
@@ -216,7 +231,7 @@ export class KeyValueStore {
      */
     async getRecord(key) {
         tryCancel();
-        ow(key, ow.string.nonEmpty);
+        parseArgument(key, keySchema);
         return this.readRecord(key);
     }
     /**
@@ -227,7 +242,7 @@ export class KeyValueStore {
      */
     async recordExists(key) {
         tryCancel();
-        ow(key, ow.string.nonEmpty);
+        parseArgument(key, keySchema);
         const entry = this.bufferedJournalEntries()?.get(key);
         if (entry) {
             return entry.value !== null;
@@ -362,19 +377,12 @@ export class KeyValueStore {
      */
     async setValue(key, value, options = {}) {
         const transaction = activeStorageTransaction();
-        ow(key, 'key', ow.string.nonEmpty);
-        ow(key, ow.string.validate((k) => ({
-            validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
-            message: `The "key" argument "${key}" must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
-        })));
+        parseArgument(key, setValueKeySchema);
         if (options.contentType && !(typeof value === 'string' || isBuffer(value) || isStream(value))) {
-            throw new ArgumentError('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.', this.setValue);
+            throw new Error('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.');
         }
-        ow(options, ow.object.exactShape({
-            contentType: ow.optional.string.nonEmpty,
-        }));
-        // Make copy of options, don't update what user passed.
-        const optionsCopy = { ...options };
+        // The parse result is a fresh copy, so we never update what user passed.
+        const optionsCopy = parseArgument(options, recordOptionsSchema);
         // The whole transaction branch sits *above* the auto-saved cache update below, so a buffered
         // write touches nothing outside the journal. That cache is shared, process-lifetime frontend
         // state, so mutating it here would survive a rollback and later be persisted by `persistState`.
@@ -480,12 +488,10 @@ export class KeyValueStore {
      */
     async forEachKey(iteratee, options = {}) {
         tryCancel();
-        ow(iteratee, ow.function);
-        ow(options, ow.object.exactShape({
-            prefix: ow.optional.string,
-        }));
+        parseArgument(iteratee, schemas.anyFunction);
+        const parsedOptions = parseArgument(options, iteratorOptionsSchema);
         let index = 0;
-        for await (const page of this.fetchKeyPages(options)) {
+        for await (const page of this.fetchKeyPages(parsedOptions)) {
             for (const item of page) {
                 await iteratee(item.key, index++, { size: item.size });
             }
@@ -623,13 +629,10 @@ export class KeyValueStore {
      */
     static async open(identifier, options = {}) {
         tryCancel();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-        }));
-        options.configuration ??= Configuration.getGlobalConfiguration();
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
         return serviceLocator.getStorageInstanceManager().openStorage(this, {
             ...resolved,

package/storages/request_list.d.ts

@@ -1,4 +1,5 @@
-import type { BaseHttpClient, Dictionary } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import { Request, type RequestOptions, type Source } from '../request.js';

package/storages/request_list.js

@@ -1,9 +1,10 @@
 import { downloadListOfUrls } from '@crawlee/utils';
-import ow, { ArgumentError } from 'ow';
+import { z } from 'zod';
 import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
 /** @internal */
@@ -11,6 +12,24 @@ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
 export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
+const requestListOptionsSchema = z.strictObject({
+    sources: schemas.anyArray.optional(), // check only for array and not subtypes to avoid iteration over the whole thing
+    sourcesFunction: schemas.anyFunction.optional(),
+    persistStateKey: z.string().optional(),
+    persistRequestsKey: z.string().optional(),
+    state: z
+        .strictObject({
+        nextIndex: schemas.anyNumber,
+        nextUniqueKey: z.string(),
+        inProgress: schemas.anyObject, // persisted as an array of unique keys
+    })
+        .optional(),
+    keepDuplicateUrls: z.boolean().default(false),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
+const listNameSchema = z.string().nullish();
+const openOptionsSchema = z.looseObject({});
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
@@ -123,24 +142,10 @@ export class RequestList {
      * @internal
      */
     constructor(options = {}) {
-        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls = false, httpClient, } = options;
+        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls, httpClient, } = parseArgument(options, requestListOptionsSchema);
         if (!(sources || sourcesFunction)) {
-            throw new ArgumentError('At least one of "sources" or "sourcesFunction" must be provided.', this.constructor);
-        }
-        ow(options, ow.object.exactShape({
-            sources: ow.optional.array, // check only for array and not subtypes to avoid iteration over the whole thing
-            sourcesFunction: ow.optional.function,
-            persistStateKey: ow.optional.string,
-            persistRequestsKey: ow.optional.string,
-            state: ow.optional.object.exactShape({
-                nextIndex: ow.number,
-                nextUniqueKey: ow.string,
-                inProgress: ow.object,
-            }),
-            keepDuplicateUrls: ow.optional.boolean,
-            proxyConfiguration: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
+            throw new Error('At least one of "sources" or "sourcesFunction" must be provided.');
+        }
         this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
         this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
         this.#initialState = state;
@@ -641,9 +646,9 @@ export class RequestList {
             return rl;
         }
         const listName = listNameOrOptions;
-        ow(listName, ow.optional.any(ow.string, ow.null));
-        ow(sources, ow.array);
-        ow(options, ow.object.is((v) => !Array.isArray(v)));
+        parseArgument(listName, listNameSchema);
+        parseArgument(sources, schemas.anyArray);
+        parseArgument(options, openOptionsSchema);
         const rl = new RequestList({
             ...options,
             persistStateKey: listName ? `${listName}-${STATE_PERSISTENCE_KEY}` : options.persistStateKey,

package/storages/request_queue.js

@@ -1,7 +1,7 @@
 import { inspect } from 'node:util';
 import { isAsyncIterable, isIterable } from '@crawlee/utils/internal';
 import { downloadListOfUrls } from '@crawlee/utils';
-import ow from 'ow';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
@@ -9,6 +9,7 @@ import { getObjectType } from '../debug.js';
 import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { activeStorageTransaction, rejectOperationInTransaction } from './transaction.js';
 import { drainRequestBatches } from './batched_adds.js';
 import { StorageStatsTracker } from './storage_stats.js';
@@ -20,6 +21,43 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @internal
  */
 const MAX_CACHED_REQUESTS = 2_000_000;
+const iterableSchema = z.custom((value) => isIterable(value) || isAsyncIterable(value), {
+    error: (issue) => `Expected an iterable or async iterable, got ${getObjectType(issue.input)}`,
+});
+const operationOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+});
+const addRequestsOptionsSchema = z.strictObject({
+    forefront: z.boolean().default(false),
+    cache: z.boolean().default(true),
+});
+const addRequestsBatchedOptionsSchema = z.strictObject({
+    forefront: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().default(false),
+    batchSize: schemas.anyNumber.default(1000),
+    waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
+    maxNewRequests: schemas.anyNumber.optional(),
+});
+const newRequestLikeSchema = z.looseObject({
+    url: z.string(),
+    id: z.undefined().optional(),
+});
+const handledRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+    handledAt: z.string().optional(),
+});
+const reclaimedRequestSchema = z.looseObject({
+    id: z.string(),
+    uniqueKey: z.string(),
+});
+const uniqueKeySchema = z.string();
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -140,20 +178,14 @@ export class RequestQueue {
      */
     async addRequest(requestLike, options = {}) {
         const transaction = activeStorageTransaction();
-        ow(requestLike, ow.object);
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
+        parseArgument(requestLike, schemas.anyObject);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
         if ('requestsFromUrl' in requestLike) {
             const requests = await this.fetchRequestsFromUrl(requestLike);
             const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
-        ow(requestLike, ow.object.partialShape({
-            url: ow.string,
-            id: ow.undefined,
-        }));
+        parseArgument(requestLike, newRequestLikeSchema);
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
         if (transaction?.policy.requestQueue === 'deferred') {
             return this.addRequestDeferred(transaction, request, forefront);
@@ -346,14 +378,8 @@ export class RequestQueue {
      */
     async addRequests(requestsLike, options = {}) {
         const transaction = activeStorageTransaction();
-        ow(requestsLike, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            cache: ow.optional.boolean,
-        }));
-        const { forefront = false, cache = true } = options;
+        parseArgument(requestsLike, iterableSchema);
+        const { forefront, cache } = parseArgument(options, addRequestsOptionsSchema);
         const uniqueKeyToCacheKey = new Map();
         const getCachedRequestId = (uniqueKey) => {
             const cached = uniqueKeyToCacheKey.get(uniqueKey);
@@ -439,16 +465,8 @@ export class RequestQueue {
      * @param options Options for the request queue
      */
     async addRequestsBatched(requests, options = {}) {
-        ow(requests, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            waitForAllRequestsToBeAdded: ow.optional.boolean,
-            batchSize: ow.optional.number,
-            waitBetweenBatchesMillis: ow.optional.number,
-            maxNewRequests: ow.optional.number,
-        }));
+        parseArgument(requests, iterableSchema);
+        const { forefront, waitForAllRequestsToBeAdded, batchSize, waitBetweenBatchesMillis, maxNewRequests } = parseArgument(options, addRequestsBatchedOptionsSchema);
         const addRequest = this.addRequest.bind(this);
         async function* generateRequests() {
             for await (const opts of requests) {
@@ -467,7 +485,7 @@ export class RequestQueue {
                 }
                 if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
                     // Handle URL lists right away
-                    await addRequest(opts, { forefront: options.forefront });
+                    await addRequest(opts, { forefront });
                 }
                 else {
                     // Yield valid requests
@@ -477,10 +495,10 @@ export class RequestQueue {
         }
         return drainRequestBatches({
             items: generateRequests(),
-            batchSize: options.batchSize ?? 1000,
-            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis ?? 1000,
-            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded ?? false,
-            maxNewRequests: options.maxNewRequests,
+            batchSize,
+            waitBetweenBatchesMillis,
+            waitForAllRequestsToBeAdded,
+            maxNewRequests,
             /**
              * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
              * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
@@ -489,7 +507,7 @@ export class RequestQueue {
              */
             processChunk: async (chunk, isInitial) => {
                 const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
-                    forefront: options.forefront,
+                    forefront,
                     cache: isInitial,
                 });
                 if (unprocessedRequests.length > 0) {
@@ -514,7 +532,7 @@ export class RequestQueue {
      */
     async getRequest(uniqueKey) {
         const transaction = activeStorageTransaction();
-        ow(uniqueKey, ow.string);
+        parseArgument(uniqueKey, uniqueKeySchema);
         // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
         const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
         if (buffered) {
@@ -561,11 +579,7 @@ export class RequestQueue {
      */
     async markRequestAsHandled(request) {
         rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-            handledAt: ow.optional.string,
-        }));
+        parseArgument(request, handledRequestSchema);
         const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
         this.#statsTracker.add('writeCount');
@@ -594,16 +608,12 @@ export class RequestQueue {
      */
     async reclaimRequest(request, options = {}) {
         rejectOperationInTransaction('RequestQueue.reclaimRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-        }));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
+        parseArgument(request, reclaimedRequestSchema);
+        const { forefront } = parseArgument(options, operationOptionsSchema);
         this.#statsTracker.add('writeCount');
-        const processedRequest = await this.backend.reclaimRequest(request, { forefront });
+        const processedRequest = await this.backend.reclaimRequest(request, {
+            forefront,
+        });
         // The request was not in progress — nothing to reclaim.
         if (!processedRequest) {
             return null;
@@ -837,14 +847,9 @@ export class RequestQueue {
      */
     static async open(identifier, options = {}) {
         tryCancel();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-            proxyConfiguration: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        const configuration = options.configuration ?? serviceLocator.getConfiguration();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        const configuration = parsedOptions.configuration ?? serviceLocator.getConfiguration();
         await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'RequestQueue');
         const queue = await serviceLocator
@@ -854,8 +859,8 @@ export class RequestQueue {
             backendOpener: () => storageBackend.createRequestQueueBackend(resolved),
             backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
         });
-        queue.#proxyConfiguration = options.proxyConfiguration;
-        queue.#httpClient = options.httpClient;
+        queue.#proxyConfiguration = parsedOptions.proxyConfiguration;
+        queue.#httpClient = parsedOptions.httpClient;
         return queue;
     }
 }

package/storages/sitemap_request_loader.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
 import { type ParseSitemapOptions } from '@crawlee/utils';
 import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';

package/storages/sitemap_request_loader.js

@@ -1,13 +1,27 @@
 import { Transform } from 'node:stream';
 import { parseSitemap } from '@crawlee/utils';
 import { minimatch } from 'minimatch';
-import ow from 'ow';
-import { constructUrlPatternObjects } from '../enqueue_links/shared.js';
+import { z } from 'zod';
+import { constructUrlPatternObjects, urlPatternSchema } from '../enqueue_links/shared.js';
 import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas } from '../validators.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
+const sitemapRequestLoaderOptionsSchema = z.strictObject({
+    sitemapUrls: schemas.arrayOf(z.string(), 'strings'),
+    proxyUrl: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    signal: z.unknown().optional(),
+    timeoutMillis: schemas.anyNumber.optional(),
+    maxBufferSize: schemas.anyNumber.default(200),
+    parseSitemapOptions: z.looseObject({}).optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    persistenceOptions: z.looseObject({}).optional(),
+    httpClient: schemas.httpClient.optional(),
+});
 /** @internal */
 const STATE_PERSISTENCE_KEY = 'SITEMAP_REQUEST_LOADER_STATE';
 /**
@@ -80,21 +94,7 @@ export class SitemapRequestLoader {
     #persistenceOptions;
     /** @internal */
     constructor(options) {
-        const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
-        ow(options, ow.object.exactShape({
-            sitemapUrls: ow.array.ofType(ow.string),
-            proxyUrl: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            signal: ow.optional.any(),
-            timeoutMillis: ow.optional.number,
-            maxBufferSize: ow.optional.number,
-            parseSitemapOptions: ow.optional.object,
-            include: ow.optional.array.ofType(urlPatternValidator),
-            exclude: ow.optional.array.ofType(urlPatternValidator),
-            persistenceOptions: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        const { include, exclude } = options;
+        const { include, exclude, persistStateKey, persistenceOptions, proxyUrl, maxBufferSize, sitemapUrls } = parseArgument(options, sitemapRequestLoaderOptionsSchema, 'SitemapRequestLoaderOptions');
         this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
         if (exclude?.length) {
             this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
@@ -102,11 +102,11 @@ export class SitemapRequestLoader {
         if (include?.length) {
             this.#urlPatternObjects.push(...constructUrlPatternObjects(include));
         }
-        this.#persistStateKey = options.persistStateKey;
-        this.#persistenceOptions = { enable: true, ...options.persistenceOptions };
-        this.#proxyUrl = options.proxyUrl;
-        this.#urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
-        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
+        this.#persistStateKey = persistStateKey;
+        this.#persistenceOptions = { enable: true, ...persistenceOptions };
+        this.#proxyUrl = proxyUrl;
+        this.#urlQueueStream = this.createNewStream(maxBufferSize);
+        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
         this.#events = serviceLocator.getEventManager();
         this.persistState = this.persistState.bind(this);
     }

package/storages/throttling_request_manager.js

@@ -1,11 +1,20 @@
 import { URL } from 'node:url';
-import ow from 'ow';
+import { z } from 'zod';
 import { PersistentRateLimitError } from '../errors.js';
 import { asyncifyIterable } from '../iterables.js';
 import { serviceLocator } from '../service_locator.js';
 import { normalizeHostname } from '../url.js';
+import { parseArgument, schemas } from '../validators.js';
 import { drainRequestBatches } from './batched_adds.js';
 import { RequestQueue } from './request_queue.js';
+const throttlingRequestManagerOptionsSchema = z.strictObject({
+    inner: schemas.anyObject,
+    domains: schemas.arrayOf(z.string().nonempty(), 'non-empty strings'),
+    requestManagerOpener: schemas.anyFunction.optional(),
+    baseDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDelaySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    maxDomainStallSecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+});
 /** Whether `manager` can pace requests per domain. */
 export function supportsDomainThrottling(manager) {
     const candidate = manager;
@@ -75,14 +84,7 @@ export class ThrottlingRequestManager {
     }
     constructor(options, config = serviceLocator.getConfiguration()) {
         this.config = config;
-        ow(options, ow.object.exactShape({
-            inner: ow.object,
-            domains: ow.array.ofType(ow.string.nonEmpty),
-            requestManagerOpener: ow.optional.function,
-            baseDelaySecs: ow.optional.number.positive,
-            maxDelaySecs: ow.optional.number.positive,
-            maxDomainStallSecs: ow.optional.number.positive,
-        }));
+        parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
         this.inner = options.inner;
         this.requestManagerOpener =
             options.requestManagerOpener ??

package/storages/utils.d.ts

@@ -1,4 +1,5 @@
-import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 /**

package/validators.d.ts

@@ -1,28 +1,25 @@
-import type { Dictionary } from '@crawlee/types';
+export { ArgumentValidationError, parseArgument } from '@crawlee/utils';
+export { schemas } from '@crawlee/utils/internal';
 /** @internal */
 export declare const validators: {
-    browserPage: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    proxyConfiguration: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestList: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    requestQueue: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    browserPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
-    sessionPool: (value: Dictionary) => {
-        validator: boolean;
-        message: (label: string) => string;
-    };
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPage: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    proxyConfiguration: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestList: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestQueue: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    browserPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    sessionPool: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    requestManager: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    storageBackend: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    logger: import("zod").ZodType<import("@crawlee/types").Dictionary, unknown, import("zod/v4/core").$ZodTypeInternals<import("@crawlee/types").Dictionary, unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    httpClient: import("zod").ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>;
 };