@crawlee/core 3.17.1-beta.22 → 3.17.1-beta.24

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.22",
+    "version": "3.17.1-beta.24",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,9 +59,9 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.22",
-        "@crawlee/types": "3.17.1-beta.22",
-        "@crawlee/utils": "3.17.1-beta.22",
+        "@crawlee/memory-storage": "3.17.1-beta.24",
+        "@crawlee/types": "3.17.1-beta.24",
+        "@crawlee/utils": "3.17.1-beta.24",
         "@sapphire/async-queue": "^1.5.1",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
@@ -83,5 +83,5 @@
             }
         }
     },
-    "gitHead": "913fa8ac27d9a04e8a8b87df05d1c6c9ab7f3a87"
+    "gitHead": "187d1180d326bac968c64807edf90aa590f8aceb"
 }

package/router.d.ts

@@ -2,13 +2,26 @@ import type { Dictionary } from '@crawlee/types';
 import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons';
 import type { Request } from './request';
 import type { Awaitable } from './typedefs';
-export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext> extends Router<Context> {
+/**
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ */
+export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+    request: LoadedRequest<Request<UserData>>;
+};
+/**
+ * The set of labels accepted by {@link Router.addHandler}. When the router declares a concrete
+ * route map (e.g. `{ PRODUCT: ...; CATEGORY: ... }`), only those labels (plus symbols) are
+ * allowed — unknown labels become a compile-time error. When the map is left open (the default
+ * `Record<string, ...>`), any string or symbol label is accepted, preserving the original behaviour.
+ */
+export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;
+export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
     (ctx: Context): Awaitable<void>;
 }
 export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
-export type RouterRoutes<Context, UserData extends Dictionary> = {
-    [label in string | symbol]: (ctx: Omit<Context, 'request'> & {
-        request: Request<UserData>;
+export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
+    [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
+        request: Request<Routes[Label]>;
     }) => Awaitable<void>;
 };
 /**
@@ -75,8 +88,30 @@ export type RouterRoutes<Context, UserData extends Dictionary> = {
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
  */
-export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>> {
+export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     private readonly routes;
     private readonly middlewares;
     /**
@@ -85,17 +120,22 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      */
     protected constructor();
     /**
-     * Registers new route handler for given label.
+     * Registers new route handler for given label. When the router declares a route map, the
+     * `label` is restricted to the declared labels and `request.userData` is typed accordingly.
+     */
+    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>): void;
+    /**
+     * Registers new route handler for given label, explicitly typing `request.userData` via the
+     * `UserData` type argument. Useful when the router has no declared route map (the open default)
+     * and you want to type a single handler, or to register a handler under a `symbol` label.
      */
-    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: string | symbol, handler: (ctx: Omit<Context, 'request'> & {
-        request: LoadedRequest<Request<UserData>>;
-    }) => Awaitable<void>): void;
+    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
     /**
-     * Registers default route handler.
+     * Registers default route handler. As a fallback it can receive any request (including labels not
+     * declared in the route map), so `request.userData` defaults to the context's `userData` type
+     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
-    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: Omit<Context, 'request'> & {
-        request: LoadedRequest<Request<UserData>>;
-    }) => Awaitable<void>): void;
+    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
@@ -129,5 +169,6 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * await crawler.run();
      * ```
      */
-    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+    static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 }

package/router.js

@@ -67,6 +67,28 @@ const defaultRoute = Symbol('default-route');
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
  */
 class Router {
     /**
@@ -87,15 +109,14 @@ class Router {
             value: []
         });
     }
-    /**
-     * Registers new route handler for given label.
-     */
     addHandler(label, handler) {
         this.validate(label);
         this.routes.set(label, handler);
     }
     /**
-     * Registers default route handler.
+     * Registers default route handler. As a fallback it can receive any request (including labels not
+     * declared in the route map), so `request.userData` defaults to the context's `userData` type
+     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler(handler) {
         this.validate(defaultRoute);
@@ -133,26 +154,6 @@ class Router {
             throw new Error(message);
         }
     }
-    /**
-     * Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
-     *
-     * ```ts
-     * import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
-     *
-     * const router = Router.create<CheerioCrawlingContext>();
-     * router.addHandler('label-a', async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     * router.addDefaultHandler(async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     *
-     * const crawler = new CheerioCrawler({
-     *     requestHandler: router,
-     * });
-     * await crawler.run();
-     * ```
-     */
     static create(routes) {
         const router = new Router();
         const obj = Object.create(Function.prototype);