@crawlee/core 4.0.0-beta.120 → 4.0.0-beta.122

package/autoscaling/autoscaled_pool.js

@@ -1,8 +1,26 @@
-import ow from 'ow';
+import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { CriticalError } from '../errors.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+// `schemas.anyObject` and `objectWithKeys`-based validators pass values through by reference
+// (object schemas return a pruned plain copy), so class instances like loggers and the
+// concurrency system keep their prototype.
+const autoscaledPoolOptionsSchema = z.strictObject({
+    runTaskFunction: schemas.anyFunction,
+    isFinishedFunction: schemas.anyFunction,
+    isTaskReadyFunction: schemas.anyFunction,
+    maybeRunIntervalSecs: schemas.anyNumber
+        .refine((value) => value > 0, 'Expected a number greater than 0')
+        .default(0.5),
+    taskTimeoutSecs: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .default(0),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    concurrencySystem: schemas.anyObject,
+    consumer: schemas.anyObject.refine((value) => typeof value.id === 'string' && value.id.length > 0, "Expected an object with a non-empty string 'id'"),
+});
 /**
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
  * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
@@ -78,17 +96,7 @@ export class AutoscaledPool {
      */
     #ownConcurrency = 0;
     constructor(options) {
-        ow(options, ow.object.exactShape({
-            runTaskFunction: ow.function,
-            isFinishedFunction: ow.function,
-            isTaskReadyFunction: ow.function,
-            maybeRunIntervalSecs: ow.optional.number.greaterThan(0),
-            taskTimeoutSecs: ow.optional.number.greaterThanOrEqual(0),
-            log: ow.optional.object,
-            concurrencySystem: ow.object,
-            consumer: ow.object.partialShape({ id: ow.string.nonEmpty }),
-        }));
-        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maybeRunIntervalSecs = 0.5, taskTimeoutSecs = 0, log = serviceLocator.getLogger(), concurrencySystem, consumer, } = options;
+        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maybeRunIntervalSecs, taskTimeoutSecs, log, concurrencySystem, consumer, } = parseArgument(options, autoscaledPoolOptionsSchema, 'AutoscaledPoolOptions');
         this.#log = log.child({ prefix: 'AutoscaledPool' });
         // Configurable properties.
         this.#maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;

package/autoscaling/concurrency_system.js

@@ -1,8 +1,34 @@
-import ow from 'ow';
+import { z } from 'zod';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { Snapshotter } from './snapshotter.js';
 import { SystemStatus } from './system_status.js';
+const concurrencySchema = z.number().int().gte(1).optional();
+// `schemas.anyObject` passes values through by reference, so the load signal instances inside
+// `loadSignals` and class instances like loggers keep their prototype.
+const concurrencySystemOptionsSchema = z.strictObject({
+    maxConcurrency: z.number().int().gte(1).default(200),
+    minConcurrency: z.number().int().gte(1).default(1),
+    desiredConcurrency: z.number().int().gte(1).optional(),
+    desiredConcurrencyRatio: z.number().gt(0).lt(1).default(0.9),
+    scaleUpStepRatio: z.number().gt(0).lt(1).default(0.05),
+    scaleDownStepRatio: z.number().gt(0).lt(1).default(0.05),
+    loggingIntervalSecs: schemas.anyNumber
+        .refine((value) => value > 0, 'Expected a number greater than 0')
+        .nullish()
+        .default(60),
+    autoscaleIntervalSecs: schemas.anyNumber
+        .refine((value) => value > 0, 'Expected a number greater than 0')
+        .default(10),
+    loadSignals: schemas.anyObject.default(() => ({})),
+    snapshotHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    currentHistorySecs: schemas.anyNumber.refine((value) => value > 0, 'Expected a number greater than 0').optional(),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    maxTasksPerMinute: z
+        .union([z.number().int().gte(1), z.literal(Number.POSITIVE_INFINITY)])
+        .default(Number.POSITIVE_INFINITY),
+});
 /**
  * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
  * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
@@ -44,22 +70,7 @@ export class ConcurrencySystem {
     /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
     #warnedAboutQueryWhileStopped = false;
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
-            minConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
-            desiredConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
-            desiredConcurrencyRatio: ow.optional.number.greaterThan(0).lessThan(1),
-            scaleUpStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
-            scaleDownStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
-            loggingIntervalSecs: ow.any(ow.number.greaterThan(0), ow.nullOrUndefined),
-            autoscaleIntervalSecs: ow.optional.number.greaterThan(0),
-            loadSignals: ow.optional.object,
-            snapshotHistorySecs: ow.optional.number.greaterThan(0),
-            currentHistorySecs: ow.optional.number.greaterThan(0),
-            log: ow.optional.object,
-            maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
-        }));
-        const { maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, loggingIntervalSecs = 60, autoscaleIntervalSecs = 10, loadSignals = {}, snapshotHistorySecs, currentHistorySecs, log = serviceLocator.getLogger(), maxTasksPerMinute = Infinity, } = options;
+        const { maxConcurrency, minConcurrency, desiredConcurrency, desiredConcurrencyRatio, scaleUpStepRatio, scaleDownStepRatio, loggingIntervalSecs, autoscaleIntervalSecs, loadSignals, snapshotHistorySecs, currentHistorySecs, log, maxTasksPerMinute, } = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');
         this.log = log.child({ prefix: 'ConcurrencySystem' });
         this.desiredConcurrencyRatio = desiredConcurrencyRatio;
         this.scaleUpStepRatio = scaleUpStepRatio;
@@ -99,7 +110,7 @@ export class ConcurrencySystem {
      * If you're not sure, just keep the default value and the concurrency will scale up automatically.
      */
     set minConcurrency(value) {
-        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        parseArgument(value, concurrencySchema);
         this.#minConcurrency = value;
         this.clampDesiredConcurrency();
     }
@@ -116,7 +127,7 @@ export class ConcurrencySystem {
      * limit as they settle).
      */
     set maxConcurrency(value) {
-        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        parseArgument(value, concurrencySchema);
         this.#maxConcurrency = value;
         this.clampDesiredConcurrency();
     }
@@ -132,7 +143,7 @@ export class ConcurrencySystem {
      * in parallel if there's large enough supply of tasks.
      */
     set desiredConcurrency(value) {
-        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        parseArgument(value, concurrencySchema);
         this.#desiredConcurrency = value;
         this.clampDesiredConcurrency();
     }

package/configuration.d.ts

@@ -6,47 +6,47 @@ export interface ConfigField<T extends z.ZodType = z.ZodType> {
 }
 export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
 /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
-export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
-export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
+export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean>;
+export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber>;
 export declare const crawleeConfigFields: {
     /** @default 'default' */
     defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default true */
-    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
     /** @default 'default' */
     defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default 'default' */
     defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default 0.95 */
-    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
     /** @default 0.25 */
-    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
-    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
+    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
     /** @default 60_000 */
-    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
     /**
      * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
      * the request handler timeout (twice it, and never below 5 minutes).
      */
-    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
     /** @default 1_000 */
-    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
     /** @default 'INPUT' */
     inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default true */
-    headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
     /** @default false */
-    xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
     chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
     defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
     /** @default false */
-    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
-    logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
+    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
+    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>>>>;
     /** @default true */
-    persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
     /** @default './storage' */
     storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
-    containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
 };
 export type FieldsInput<F extends Record<string, ConfigField>> = {
     [K in keyof F]?: z.output<F[K]['schema']>;

package/configuration.js

@@ -36,7 +36,7 @@ const logLevelSchema = z.preprocess((val) => {
     if (key in LogLevel)
         return LogLevel[key];
     return val;
-}, z.nativeEnum(LogLevel));
+}, z.enum(LogLevel));
 // --- Crawlee config field definitions ---
 export const crawleeConfigFields = {
     /** @default 'default' */
@@ -196,8 +196,8 @@ export class Configuration {
                 continue;
             }
             // 4. Schema default (by parsing undefined through the schema)
-            const result = fieldDef.schema.safeParse(undefined);
-            values[key] = result.success ? result.data : undefined;
+            const parsed = fieldDef.schema.safeParse(undefined);
+            values[key] = parsed.success ? parsed.data : undefined;
         }
         return values;
     }

package/crawlers/error_snapshotter.d.ts

@@ -46,7 +46,7 @@ export declare class ErrorSnapshotter {
      */
     contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
     /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     * Save the HTML snapshot of the page, and return the key it was stored under.
      */
     saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
     /**

package/crawlers/error_snapshotter.js

@@ -77,12 +77,14 @@ export class ErrorSnapshotter {
         }
     }
     /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     * Save the HTML snapshot of the page, and return the key it was stored under.
      */
     async saveHTMLSnapshot(html, keyValueStore, fileName) {
         try {
             await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
-            return `${fileName}.html`;
+            // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
+            // where local storage put the extension in the key) would break `getPublicUrl`.
+            return fileName;
         }
         catch {
             return undefined;

package/crawlers/statistics.d.ts

@@ -1,5 +1,5 @@
 import type { CrawleeLogger } from '../log.js';
-import type { KeyValueStore } from '../storages/key_value_store.js';
+import { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * Persistence-related options to control how and when crawler's data gets persisted.

package/crawlers/statistics.js

@@ -1,7 +1,8 @@
-import ow from 'ow';
 import { z } from 'zod';
 import { RecoverableState } from '../recoverable_state.js';
 import { serviceLocator } from '../service_locator.js';
+import { KeyValueStore } from '../storages/key_value_store.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * @ignore
@@ -17,6 +18,16 @@ class Job {
         return this.#durationMillis;
     }
 }
+const statisticsOptionsSchema = z.strictObject({
+    logIntervalSecs: schemas.anyNumber.default(60),
+    logMessage: z.string().default('Statistics'),
+    log: validators.logger.optional(),
+    keyValueStore: z.instanceof(KeyValueStore).optional(),
+    // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain copy).
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    saveErrorSnapshots: z.boolean().default(false),
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+});
 const errorTrackerConfig = {
     showErrorCode: true,
     showErrorName: true,
@@ -184,21 +195,10 @@ export class Statistics {
      * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            logIntervalSecs: ow.optional.number,
-            logMessage: ow.optional.string,
-            log: ow.optional.object,
-            keyValueStore: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-            saveErrorSnapshots: ow.optional.boolean,
-            id: ow.optional.any(ow.number, ow.string),
-        }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
-            enable: true,
-        }, saveErrorSnapshots = false, id, } = options;
+        const { logIntervalSecs, logMessage, log, keyValueStore, persistenceOptions, saveErrorSnapshots, id } = parseArgument(options, statisticsOptionsSchema);
         this.id = id ?? String(Statistics.id++);
         this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
-        this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
+        this.log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.#logIntervalMillis = logIntervalSecs * 1000;

package/debug.js

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import ow from 'ow';
+import { parseArgument, schemas } from './validators.js';
 /**
  * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
  *
@@ -12,9 +12,9 @@ import ow from 'ow';
  * @internal
  */
 export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
-    ow(request, ow.object);
-    ow(response, ow.object);
-    ow(additionalFields, ow.object);
+    parseArgument(request, schemas.anyObject);
+    parseArgument(response, schemas.anyObject);
+    parseArgument(additionalFields, schemas.anyObject);
     return {
         requestId: request.id,
         url: request.url,

package/enqueue_links/enqueue_links.js

@@ -1,7 +1,8 @@
-import ow from 'ow';
 import { getDomain } from 'tldts';
+import { z } from 'zod';
 import { Request } from '../request.js';
-import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, } from './shared.js';
+import { parseArgument, schemas } from '../validators.js';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, urlPatternSchema, } from './shared.js';
 /**
  * The different enqueueing strategies available.
  *
@@ -54,6 +55,28 @@ export var EnqueueStrategy;
      */
     EnqueueStrategy["SameOrigin"] = "same-origin";
 })(EnqueueStrategy || (EnqueueStrategy = {}));
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so `userData` keeps its identity for the enqueued requests.
+const enqueueLinksOptionsSchema = z.strictObject({
+    urls: schemas.arrayOf(z.string(), 'strings'),
+    requestManager: schemas.objectWithKeys(['addRequestsBatched']),
+    robotsTxtFile: schemas.objectWithKeys(['isAllowed']).optional(),
+    respectRobotsTxtFile: z.union([z.boolean(), z.strictObject({ userAgent: z.string().optional() })]).optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+    forefront: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    sessionId: z.string().optional(),
+    limit: schemas.anyNumber.optional(),
+    selector: z.string().optional(),
+    baseUrl: z.string().optional(),
+    userData: schemas.anyObject.optional(),
+    label: z.string().optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    strategy: z.enum(EnqueueStrategy).optional(),
+    waitForAllRequestsToBeAdded: z.boolean().optional(),
+});
 /**
  * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
@@ -84,37 +107,17 @@ export async function enqueueLinks(options) {
             'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/js/docs/examples/crawl-relative-links',
         ].join('\n'));
     }
-    const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
-    ow(options, ow.object.exactShape({
-        urls: ow.array.ofType(ow.string),
-        requestManager: ow.object.hasKeys('addRequestsBatched'),
-        robotsTxtFile: ow.optional.object.hasKeys('isAllowed'),
-        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object.exactShape({ userAgent: ow.optional.string })),
-        onSkippedRequest: ow.optional.function,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        sessionId: ow.optional.string,
-        limit: ow.optional.number,
-        selector: ow.optional.string,
-        baseUrl: ow.optional.string,
-        userData: ow.optional.object,
-        label: ow.optional.string,
-        include: ow.optional.array.minLength(1).ofType(urlPatternValidator),
-        exclude: ow.optional.array.ofType(urlPatternValidator),
-        transformRequestFunction: ow.optional.function,
-        strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
-        waitForAllRequestsToBeAdded: ow.optional.boolean,
-    }));
-    const { requestManager, limit, urls, include, exclude, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = options;
+    const parsedOptions = parseArgument(options, enqueueLinksOptionsSchema, 'EnqueueLinksOptions');
+    const { requestManager, limit, urls, include, exclude, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, robotsTxtFile, onSkippedRequest, } = parsedOptions;
     const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
     const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
     // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
     // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
-    options.strategy ??= EnqueueStrategy.SameHostname;
+    parsedOptions.strategy ??= EnqueueStrategy.SameHostname;
     const enqueueStrategyPatterns = [];
-    if (options.baseUrl) {
-        const url = new URL(options.baseUrl);
-        switch (options.strategy) {
+    if (parsedOptions.baseUrl) {
+        const url = new URL(parsedOptions.baseUrl);
+        switch (parsedOptions.strategy) {
             case EnqueueStrategy.SameHostname:
                 // We need to get the origin of the passed in domain in the event someone sets baseUrl
                 // to an url like https://example.com/deep/default/path and one of the found urls is an
@@ -157,9 +160,11 @@ export async function enqueueLinks(options) {
             }));
         }
     }
-    let requestOptions = createRequestOptions(urls, options);
-    if (robotsTxtFile && options.respectRobotsTxtFile !== false) {
-        const robotsUserAgent = typeof options.respectRobotsTxtFile === 'object' ? (options.respectRobotsTxtFile.userAgent ?? '*') : '*';
+    let requestOptions = createRequestOptions(urls, parsedOptions);
+    if (robotsTxtFile && parsedOptions.respectRobotsTxtFile !== false) {
+        const robotsUserAgent = typeof parsedOptions.respectRobotsTxtFile === 'object'
+            ? (parsedOptions.respectRobotsTxtFile.userAgent ?? '*')
+            : '*';
         const skippedRequests = [];
         requestOptions = requestOptions.filter((request) => {
             if (robotsTxtFile.isAllowed(request.url, robotsUserAgent)) {
@@ -175,13 +180,13 @@ export async function enqueueLinks(options) {
         // Step 1: Filter request options by exclude patterns, user include patterns, and strategy patterns.
         let filteredOptions;
         if (urlPatternObjects.length === 0) {
-            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
+            filteredOptions = filterRequestOptionsByPatterns(requestOptions, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, urlExcludePatternObjects, parsedOptions.strategy, (url) => skippedRequests.push(url));
         }
         else {
             // Filter by user patterns first (with exclude)
-            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy, (url) => skippedRequests.push(url));
+            const afterUserPatterns = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects, urlExcludePatternObjects, parsedOptions.strategy, (url) => skippedRequests.push(url));
             // ...then filter by the enqueue links strategy (making this an AND check)
-            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], options.strategy, (url) => skippedRequests.push(url));
+            filteredOptions = filterRequestOptionsByPatterns(afterUserPatterns, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], parsedOptions.strategy, (url) => skippedRequests.push(url));
         }
         await reportSkippedRequests(skippedRequests.map((url) => ({ url })), 'filters');
         // Step 2: Apply transformRequestFunction on request options - it has the highest priority

package/enqueue_links/shared.d.ts

@@ -1,4 +1,5 @@
 import type { Awaitable } from '@crawlee/types';
+import { z } from 'zod';
 import type { RequestOptions } from '../request.js';
 import type { EnqueueLinksOptions } from './enqueue_links.js';
 export { tryAbsoluteURL } from '@crawlee/utils/internal';
@@ -16,6 +17,11 @@ export interface RegExpObject {
 export type RegExpInput = RegExp | RegExpObject;
 /** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
 export type UrlPatternInput = GlobInput | RegExpInput;
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
 export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
 export type SkippedRequestCallback = (args: {
     url: string;

package/enqueue_links/shared.js

@@ -1,5 +1,7 @@
 import { URL } from 'node:url';
 import { Minimatch } from 'minimatch';
+import { z } from 'zod';
+import { schemas } from '../validators.js';
 export { tryAbsoluteURL } from '@crawlee/utils/internal';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
@@ -8,6 +10,16 @@ const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
  * @ignore
  */
 const enqueueLinksPatternCache = new Map();
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
 /**
  * @ignore
  */

package/memory-storage/resource-clients/dataset.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { BaseClient } from './common/base-client.js';
 /**
  * This is what API returns in the x-apify-pagination-limit
@@ -49,13 +49,7 @@ export class DatasetBackend extends BaseClient {
         this.updateTimestamps(true);
     }
     getData(options = {}) {
-        const { desc, limit, offset } = s
-            .object({
-            desc: s.boolean().optional(),
-            limit: s.number().int().optional(),
-            offset: s.number().int().optional(),
-        })
-            .parse(options);
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
         return this.getDataPage({
             desc,
             offset: offset ?? 0,

package/memory-storage/resource-clients/key-value-store.js

@@ -1,9 +1,25 @@
 import { randomUUID } from 'node:crypto';
-import { s } from '@sapphire/shapeshift';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { isStream, toBuffer } from '../utils.js';
 import { BaseClient } from './common/base-client.js';
 import mime from 'mime-types';
 const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
+const keySchema = z.string();
+const inputRecordSchema = z.object({
+    key: z.string().min(1),
+    value: z.union([
+        z.null(),
+        z.string(),
+        z.number(),
+        z.instanceof(Buffer),
+        z.instanceof(ArrayBuffer),
+        schemas.typedArray,
+        // only checks the value is an actual object, not null, nor array
+        schemas.plainObject,
+    ]),
+    contentType: z.string().min(1).optional(),
+});
 /**
  * Key under which a run's input is stored in the default key-value store. Matches Crawlee's default
  * `inputKey` (`CRAWLEE_INPUT_KEY`) and the `INPUT` files `FileSystemStorageBackend` preserves on purge.
@@ -58,13 +74,7 @@ export class KeyValueStoreBackend extends BaseClient {
         this.updateTimestamps(true);
     }
     async listKeys(options = {}) {
-        const { prefix, exclusiveStartKey, limit } = s
-            .object({
-            prefix: s.string().optional(),
-            exclusiveStartKey: s.string().optional(),
-            limit: s.number().int().greaterThan(0).optional(),
-        })
-            .parse(options);
+        const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
         const items = [];
         for (const record of this.#keyValueEntries.values()) {
             const size = Buffer.byteLength(record.value);
@@ -104,7 +114,7 @@ export class KeyValueStoreBackend extends BaseClient {
      * @param key The key of the record to generate the public URL for.
      */
     async getPublicUrl(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return undefined;
     }
     /**
@@ -114,11 +124,11 @@ export class KeyValueStoreBackend extends BaseClient {
      * @returns `true` if the record exists, `false` if it does not.
      */
     async recordExists(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         return this.#keyValueEntries.has(key);
     }
     async getValue(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         const entry = this.#keyValueEntries.get(key);
         if (!entry) {
             return undefined;
@@ -136,20 +146,7 @@ export class KeyValueStoreBackend extends BaseClient {
         return record;
     }
     async setValue(record) {
-        s.object({
-            key: s.string().lengthGreaterThan(0),
-            value: s.union([
-                s.null(),
-                s.string(),
-                s.number(),
-                s.instance(Buffer),
-                s.instance(ArrayBuffer),
-                s.typedArray(),
-                // disabling validation will make shapeshift only check the object given is an actual object, not null, nor array
-                s.object({}).setValidationEnabled(false),
-            ]),
-            contentType: s.string().lengthGreaterThan(0).optional(),
-        }).parse(record);
+        parseArgument(record, inputRecordSchema);
         const { key } = record;
         let { value } = record;
         // The frontend (KeyValueStore codec) serializes the value and resolves its content type
@@ -180,7 +177,7 @@ export class KeyValueStoreBackend extends BaseClient {
         this.updateTimestamps(true);
     }
     async deleteValue(key) {
-        s.string().parse(key);
+        parseArgument(key, keySchema);
         if (this.#keyValueEntries.has(key)) {
             this.#keyValueEntries.delete(key);
             this.updateTimestamps(true);