@crawlee/basic 3.17.1-beta.54 → 3.17.1-beta.55

package/index.mjs

@@ -48,6 +48,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -76,6 +77,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -96,5 +98,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/basic-crawler.d.ts

@@ -545,6 +545,24 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      */
     stop(reason?: string): void;
     getRequestQueue(): Promise<RequestProvider>;
+    /**
+     * The request handler exactly as the user supplied it — a {@link Router} when one is in use, whether it
+     * was passed as `requestHandler` or auto-wired from {@link BasicCrawler.router|`crawler.router`}.
+     *
+     * Router-aware features read per-label metadata off this handler (currently the `userData` schema map), so
+     * it must resolve to the *unwrapped* handler. Subclasses that hand a wrapper to `BasicCrawler` instead of
+     * the user's own function — {@link BrowserCrawler} and its descendants do — have to override this, or
+     * those features silently no-op against the wrapper.
+     */
+    protected get userRequestHandler(): RequestHandler<Context>;
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    protected validateRequestUserData(source: Source | string): Promise<void>;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
     protected get pendingRequestCountApproximation(): number;
     protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;

package/internals/basic-crawler.js

@@ -729,6 +729,44 @@ class BasicCrawler {
         }
         return this.requestQueue;
     }
+    /**
+     * The request handler exactly as the user supplied it — a {@link Router} when one is in use, whether it
+     * was passed as `requestHandler` or auto-wired from {@link BasicCrawler.router|`crawler.router`}.
+     *
+     * Router-aware features read per-label metadata off this handler (currently the `userData` schema map), so
+     * it must resolve to the *unwrapped* handler. Subclasses that hand a wrapper to `BasicCrawler` instead of
+     * the user's own function — {@link BrowserCrawler} and its descendants do — have to override this, or
+     * those features silently no-op against the wrapper.
+     */
+    get userRequestHandler() {
+        return this.requestHandler;
+    }
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    async validateRequestUserData(source) {
+        if (typeof source === 'string') {
+            return;
+        }
+        const getSchema = this.userRequestHandler.getSchema;
+        if (typeof getSchema !== 'function') {
+            return;
+        }
+        // Resolve the label via its public accessors only — the top-level `label` of a `RequestOptions` or the
+        // `Request.label` getter — rather than reaching into `userData`, where the request happens to store it.
+        const target = source;
+        const schema = getSchema(target.label);
+        if (!schema) {
+            return;
+        }
+        // Store the parsed value rather than the raw input, so the queue holds the same coerced `userData` the
+        // handler will see. Assigning through a `Request` instance's setter keeps its internal `__crawlee` meta.
+        target.userData = await (0, core_1.validateUserData)(target.label, schema, target.userData ?? {});
+    }
     async useState(defaultValue = {}) {
         const kvs = await core_1.KeyValueStore.open(null, { config: this.config });
         return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
@@ -777,6 +815,7 @@ class BasicCrawler {
         const skippedBecauseOfMaxCrawlDepth = new Set();
         const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
         const maxCrawlDepth = this.maxCrawlDepth;
+        const validateRequestUserData = this.validateRequestUserData.bind(this);
         (0, ow_1.default)(requests, ow_1.default.object
             .is((value) => (0, utils_1.isIterable)(value) || (0, utils_1.isAsyncIterable)(value))
             .message((value) => `Expected an iterable or async iterable, got ${(0, utils_1.getObjectType)(value)}`));
@@ -788,6 +827,7 @@ class BasicCrawler {
                     continue;
                 }
                 if (await isAllowedBasedOnRobotsTxtFile(url)) {
+                    await validateRequestUserData(request);
                     yield request;
                 }
                 else {
@@ -1177,6 +1217,9 @@ class BasicCrawler {
             }
             await this.handleSkippedRequest(skippedOptions);
         };
+        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
+        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
+        await this.validateRequestUserData({ label: options.label, userData: options.userData });
         return (0, core_1.enqueueLinks)({
             requestQueue,
             robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "3.17.1-beta.54",
+    "version": "3.17.1-beta.55",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -49,9 +49,9 @@
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/core": "3.17.1-beta.54",
-        "@crawlee/types": "3.17.1-beta.54",
-        "@crawlee/utils": "3.17.1-beta.54",
+        "@crawlee/core": "3.17.1-beta.55",
+        "@crawlee/types": "3.17.1-beta.55",
+        "@crawlee/utils": "3.17.1-beta.55",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
         "got-scraping": "^4.2.1",
@@ -67,5 +67,5 @@
             }
         }
     },
-    "gitHead": "45d0756fd5f016539d6e2f80f72340a652ddc48f"
+    "gitHead": "93d29e305f01ca6ca9bb79264a5ca553b19766ef"
 }