@crawlee/core 4.0.0-beta.85 → 4.0.0-beta.86

package/crawlers/crawler_commons.d.ts

@@ -3,13 +3,65 @@ import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
-import type { Request, Source } from '../request.js';
+import type { Request, RequestOptions, Source } from '../request.js';
 import type { Dataset } from '../storages/dataset.js';
 import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
 import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
 import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
+/**
+ * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ * @internal
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
+    requestsFromUrl?: string;
+    regex?: RegExp;
+} & ({
+    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+        label: Label;
+    };
+}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
+    label?: undefined;
+})));
+/**
+ * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ * @internal
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
+ * @internal
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
+    [Label in keyof Routes & string]: {
+        label: Label;
+        userData?: Routes[Label];
+    };
+}[keyof Routes & string] | {
+    label?: undefined;
+    userData?: Dictionary;
+});
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ * @internal
+ */
+export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
 /** @internal */
 export type WithRequired<T, K extends keyof T> = T & {
     [P in K]-?: T[P];
@@ -187,3 +239,4 @@ export declare class RequestHandlerResult {
     private getKeyValueStoreChangedValue;
     private setKeyValueStoreChangedValue;
 }
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.85",
+    "version": "4.0.0-beta.86",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.85",
-        "@crawlee/types": "4.0.0-beta.85",
-        "@crawlee/utils": "4.0.0-beta.85",
+        "@crawlee/fs-storage": "4.0.0-beta.86",
+        "@crawlee/types": "4.0.0-beta.86",
+        "@crawlee/utils": "4.0.0-beta.86",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "8d6d3cd9da3dc9cb0b7822b28eb317019b6806bc"
+    "gitHead": "c2f251426f3ccfcc6efc920e6d492be57beac43b"
 }

package/router.d.ts

@@ -1,6 +1,6 @@
 import type { Dictionary } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
-import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext } from './crawlers/crawler_commons.js';
+import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
 import type { Request } from './request.js';
 import type { Awaitable } from './typedefs.js';
 /**
@@ -10,11 +10,18 @@ import type { Awaitable } from './typedefs.js';
  */
 export declare const defaultRoute: unique symbol;
 /**
- * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`.
+ * The crawling context received by a route handler, with `request.userData` narrowed to `UserData`, and
+ * `addRequests`/`enqueueLinks` typed according to the router's route map (`Routes`) so that enqueuing a
+ * request under a declared label requires the matching `userData` shape.
  */
-export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Context, 'request'> & {
+export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
     request: LoadedRequest<Request<UserData>>;
-};
+    addRequests: TypedContextAddRequests<Routes>;
+} & (Context extends {
+    enqueueLinks: infer EnqueueLinks;
+} ? {
+    enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes>;
+} : {});
 /**
  * A map of request labels to a [Standard Schema](https://standardschema.dev) (Zod, Valibot, ArkType, …)
  * validating that label's `request.userData`. Pass it to {@link Router.create} or a `createXRouter`
@@ -24,14 +31,29 @@ export type RouterHandlerContext<Context, UserData extends Dictionary> = Omit<Co
 export type RouteSchemas = Record<string, StandardSchemaV1> & {
     [defaultRoute]?: StandardSchemaV1;
 };
+/** Infers a label's `userData` type from its schema, falling back to a plain {@link Dictionary}. */
+type SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;
 /**
- * Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each
- * schema's output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
- * {@link defaultRoute} schema drives runtime validation only, so it is excluded from the typed route map.
+ * Derives a route map (label → `userData` type) from a {@link RouteSchemas} map by inferring each schema's
+ * output type. Outputs that are not object-shaped fall back to a plain {@link Dictionary}. The
+ * {@link defaultRoute} schema is kept under its symbol key so {@link Router.addDefaultHandler} can pick it
+ * up; string labels (the ones {@link Router.addHandler} and the crawler-level typing accept) ignore it.
  */
 export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
-    [Label in Extract<keyof Schemas, string>]: StandardSchemaV1.InferOutput<Schemas[Label]> extends Dictionary ? StandardSchemaV1.InferOutput<Schemas[Label]> : Dictionary;
-};
+    [Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
+} & (Schemas extends {
+    [defaultRoute]: StandardSchemaV1;
+} ? {
+    [defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;
+} : {});
+/**
+ * The `userData` type of the default route: inferred from the {@link defaultRoute} schema when the route map
+ * carries one, otherwise the provided `Fallback`.
+ * @internal
+ */
+export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
+    [defaultRoute]: infer DefaultUserData extends Dictionary;
+} ? DefaultUserData : Fallback;
 /**
  * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
  * value. Throws a {@link RequestValidationError} when validation fails.
@@ -173,19 +195,20 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * Registers new route handler for given label. When the router declares a route map, the
      * `label` is restricted to the declared labels and `request.userData` is typed accordingly.
      */
-    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label]>) => Awaitable<void>): void;
+    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>): void;
     /**
      * Registers new route handler for given label, explicitly typing `request.userData` via the
      * `UserData` type argument. Useful when the router has no declared route map (the open default)
      * and you want to type a single handler, or to register a handler under a `symbol` label.
      */
-    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
+    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;
     /**
      * Registers default route handler. As a fallback it can receive any request (including labels not
-     * declared in the route map), so `request.userData` defaults to the context's `userData` type
-     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
+     * declared in the route map). When the router was created with a {@link defaultRoute} schema,
+     * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
+     * Pass an explicit `UserData` type argument to narrow it.
      */
-    addDefaultHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(handler: (ctx: RouterHandlerContext<Context, UserData>) => Awaitable<void>): void;
+    addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;
     /**
      * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
      * to validate `request.userData` when requests are added.
@@ -234,3 +257,4 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
     static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
     static create<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'> = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 }
+export {};

package/router.js

@@ -155,8 +155,9 @@ export class Router {
     }
     /**
      * Registers default route handler. As a fallback it can receive any request (including labels not
-     * declared in the route map), so `request.userData` defaults to the context's `userData` type
-     * (loosely typed by default). Pass an explicit `UserData` type argument to narrow it.
+     * declared in the route map). When the router was created with a {@link defaultRoute} schema,
+     * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
+     * Pass an explicit `UserData` type argument to narrow it.
      */
     addDefaultHandler(handler) {
         this.validate(defaultRoute);