@crawlee/basic 3.17.1-beta.9 → 3.18.0

package/index.mjs

@@ -48,6 +48,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -76,6 +77,7 @@ export const createDeserialize = mod.createDeserialize;
 export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -96,5 +98,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/basic-crawler.d.ts

@@ -266,6 +266,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * 2. because they don't match enqueueLinks filters,
      * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
      * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+     *
+     * When `enqueueLinks` is called with its own `onSkippedRequest` callback, both are invoked — this one first,
+     * then the `enqueueLinks` one.
      */
     onSkippedRequest?: SkippedRequestCallback;
     /** @internal */
@@ -436,7 +439,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
         userAgent?: string;
     };
     protected onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
+    private _ownsEventManager;
     private loggedPerRun;
     private experiments;
     private readonly robotsTxtFileCache;
@@ -545,6 +548,24 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      */
     stop(reason?: string): void;
     getRequestQueue(): Promise<RequestProvider>;
+    /**
+     * The request handler exactly as the user supplied it — a {@link Router} when one is in use, whether it
+     * was passed as `requestHandler` or auto-wired from {@link BasicCrawler.router|`crawler.router`}.
+     *
+     * Router-aware features read per-label metadata off this handler (currently the `userData` schema map), so
+     * it must resolve to the *unwrapped* handler. Subclasses that hand a wrapper to `BasicCrawler` instead of
+     * the user's own function — {@link BrowserCrawler} and its descendants do — have to override this, or
+     * those features silently no-op against the wrapper.
+     */
+    protected get userRequestHandler(): RequestHandler<Context>;
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    protected validateRequestUserData(source: Source | string): Promise<void>;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
     protected get pendingRequestCountApproximation(): number;
     protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
@@ -618,8 +639,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     /**
      * Wrapper around the crawling context's `enqueueLinks` method:
      * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
+     * - Combines the `enqueueLinks` options with the crawler configuration - the user options take precedence,
+     *   but the crawler limits are always enforced (the `limit` is capped by the remaining `maxRequestsPerCrawl`
+     *   budget and skipped requests are always reported to the crawler too).
      * @internal
      */
     protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestQueue: RequestProvider): Promise<BatchAddRequestsResult>;
@@ -719,5 +741,6 @@ interface HandlePropertyNameChangeData<New, Old> {
  * await crawler.run();
  * ```
  */
-export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 export {};

package/internals/basic-crawler.js

@@ -336,11 +336,11 @@ class BasicCrawler {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "_closeEvents", {
+        Object.defineProperty(this, "_ownsEventManager", {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: void 0
+            value: false
         });
         Object.defineProperty(this, "loggedPerRun", {
             enumerable: true,
@@ -655,8 +655,8 @@ class BasicCrawler {
             await this.autoscaledPool.run();
         }
         finally {
-            await this.teardown();
             await this.stats.stopCapturing();
+            await this.teardown();
             process.off('SIGINT', sigintHandler);
             this.events.off("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
             this.events.off("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
@@ -688,8 +688,12 @@ class BasicCrawler {
                 finished = true;
             }
             periodicLogger.stop();
-            // Don't await, we don't want to block the execution
-            void this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+            // Give the event loop a single tick to flush the HTTP
+            // 1ms is enough because we already have a keep-alive connection to the API
+            await Promise.race([
+                this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' }),
+                (0, utils_1.sleep)(1),
+            ]);
             this.running = false;
             this.hasFinishedBefore = true;
         }
@@ -725,6 +729,44 @@ class BasicCrawler {
         }
         return this.requestQueue;
     }
+    /**
+     * The request handler exactly as the user supplied it — a {@link Router} when one is in use, whether it
+     * was passed as `requestHandler` or auto-wired from {@link BasicCrawler.router|`crawler.router`}.
+     *
+     * Router-aware features read per-label metadata off this handler (currently the `userData` schema map), so
+     * it must resolve to the *unwrapped* handler. Subclasses that hand a wrapper to `BasicCrawler` instead of
+     * the user's own function — {@link BrowserCrawler} and its descendants do — have to override this, or
+     * those features silently no-op against the wrapper.
+     */
+    get userRequestHandler() {
+        return this.requestHandler;
+    }
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    async validateRequestUserData(source) {
+        if (typeof source === 'string') {
+            return;
+        }
+        const getSchema = this.userRequestHandler.getSchema;
+        if (typeof getSchema !== 'function') {
+            return;
+        }
+        // Resolve the label via its public accessors only — the top-level `label` of a `RequestOptions` or the
+        // `Request.label` getter — rather than reaching into `userData`, where the request happens to store it.
+        const target = source;
+        const schema = getSchema(target.label);
+        if (!schema) {
+            return;
+        }
+        // Store the parsed value rather than the raw input, so the queue holds the same coerced `userData` the
+        // handler will see. Assigning through a `Request` instance's setter keeps its internal `__crawlee` meta.
+        target.userData = await (0, core_1.validateUserData)(target.label, schema, target.userData ?? {});
+    }
     async useState(defaultValue = {}) {
         const kvs = await core_1.KeyValueStore.open(null, { config: this.config });
         return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
@@ -773,6 +815,7 @@ class BasicCrawler {
         const skippedBecauseOfMaxCrawlDepth = new Set();
         const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
         const maxCrawlDepth = this.maxCrawlDepth;
+        const validateRequestUserData = this.validateRequestUserData.bind(this);
         (0, ow_1.default)(requests, ow_1.default.object
             .is((value) => (0, utils_1.isIterable)(value) || (0, utils_1.isAsyncIterable)(value))
             .message((value) => `Expected an iterable or async iterable, got ${(0, utils_1.getObjectType)(value)}`));
@@ -784,6 +827,7 @@ class BasicCrawler {
                     continue;
                 }
                 if (await isAllowedBasedOnRobotsTxtFile(url)) {
+                    await validateRequestUserData(request);
                     yield request;
                 }
                 else {
@@ -888,7 +932,7 @@ class BasicCrawler {
     async _init() {
         if (!this.events.isInitialized()) {
             await this.events.init();
-            this._closeEvents = true;
+            this._ownsEventManager = true;
         }
         this.autoscaledPool = new core_1.AutoscaledPool(this.autoscaledPoolOptions, this.config);
         if (this.useSessionPool) {
@@ -1147,8 +1191,9 @@ class BasicCrawler {
     /**
      * Wrapper around the crawling context's `enqueueLinks` method:
      * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
+     * - Combines the `enqueueLinks` options with the crawler configuration - the user options take precedence,
+     *   but the crawler limits are always enforced (the `limit` is capped by the remaining `maxRequestsPerCrawl`
+     *   budget and skipped requests are always reported to the crawler too).
      * @internal
      */
     async enqueueLinksWithCrawlDepth(options, request, requestQueue) {
@@ -1159,28 +1204,37 @@ class BasicCrawler {
                 return false;
             }
             // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(newRequest) ?? newRequest;
+            // Its return value is passed through as is, so a falsy one still skips the request.
+            return options.transformRequestFunction ? options.transformRequestFunction(newRequest) : newRequest;
         };
+        const limit = this.calculateEnqueuedRequestLimit(options.limit);
         // Create a request-scoped callback that logs enqueueLimit once per request handler call
         // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
         let loggedEnqueueLimitForThisRequest = false;
         const onSkippedRequest = async (skippedOptions) => {
             if (skippedOptions.reason === 'enqueueLimit') {
                 if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
-                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
+                    this.log.info(limit === options.limit
+                        ? `Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`
+                        : `Skipping URLs in the handler for ${request.url} due to the remaining maxRequestsPerCrawl budget of ${limit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
                     loggedEnqueueLimitForThisRequest = true;
                 }
             }
             await this.handleSkippedRequest(skippedOptions);
+            await options.onSkippedRequest?.(skippedOptions);
         };
+        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
+        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
+        await this.validateRequestUserData({ label: options.label, userData: options.userData });
         return (0, core_1.enqueueLinks)({
-            requestQueue,
-            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
-            onSkippedRequest,
-            limit: this.calculateEnqueuedRequestLimit(options.limit),
-            // Allow user options to override defaults set above ⤴
             ...options,
+            // The options below are merged with the user options, so an explicitly `undefined` value
+            // (e.g. `enqueueLinks({ urls, limit: config.limit })`) cannot discard the crawler defaults ⤵
+            requestQueue: options.requestQueue ?? requestQueue,
+            robotsTxtFile: options.robotsTxtFile ?? (await this.getRobotsTxtFileForUrl(request.url)),
+            respectRobotsTxtFile: options.respectRobotsTxtFile ?? this.respectRobotsTxtFile,
+            onSkippedRequest,
+            limit,
             transformRequestFunction: transformRequestFunctionWrapper,
         });
     }
@@ -1359,11 +1413,16 @@ class BasicCrawler {
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
-        this.events.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+        // When this crawler initialized the event manager, its close() call emits
+        // the final persistence event after the crawler-specific state has been
+        // saved. External event managers still need an explicit event here.
+        if (!this._ownsEventManager) {
+            this.events.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+        }
         if (this.useSessionPool) {
-            await this.sessionPool.teardown();
+            await this.sessionPool.teardown({ persistState: this._ownsEventManager });
         }
-        if (this._closeEvents) {
+        if (this._ownsEventManager) {
             await this.events.close();
         }
         await this.autoscaledPool?.abort();
@@ -1502,30 +1561,6 @@ Object.defineProperty(BasicCrawler, "optionsShape", {
         statisticsOptions: ow_1.default.optional.object,
     }
 });
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
- * Defaults to the {@link BasicCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
- *
- * ```ts
- * import { BasicCrawler, createBasicRouter } from 'crawlee';
- *
- * const router = createBasicRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new BasicCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
 function createBasicRouter(routes) {
     return core_1.Router.create(routes);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "3.17.1-beta.9",
+    "version": "3.18.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -45,12 +45,13 @@
         "access": "public"
     },
     "dependencies": {
+        "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
-        "@apify/timeout": "^0.3.0",
+        "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/core": "3.17.1-beta.9",
-        "@crawlee/types": "3.17.1-beta.9",
-        "@crawlee/utils": "3.17.1-beta.9",
+        "@crawlee/core": "3.18.0",
+        "@crawlee/types": "3.18.0",
+        "@crawlee/utils": "3.18.0",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
         "got-scraping": "^4.2.1",
@@ -66,5 +67,5 @@
             }
         }
     },
-    "gitHead": "c547ce4489d8617620a51516cc885187503f3136"
+    "gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
 }