@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/router.js

@@ -1,5 +1,39 @@
-import { MissingRouteError } from './errors.js';
-const defaultRoute = Symbol('default-route');
+import { MissingRouteError, RequestValidationError } from './errors.js';
+/**
+ * The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
+ * Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
+ * that falls through to the default handler (i.e. whose label has no route of its own).
+ */
+export const defaultRoute = Symbol('default-route');
+/** Whether a validation issue points at the top-level `label` key. */
+function isLabelIssue(issue) {
+    if (issue.path?.length !== 1) {
+        return false;
+    }
+    const [segment] = issue.path;
+    return (typeof segment === 'object' ? segment.key : segment) === 'label';
+}
+/**
+ * Validates `userData` against a {@link RouteSchemas|Standard Schema}, returning the parsed (and coerced)
+ * value. Throws a {@link RequestValidationError} when validation fails.
+ * @internal
+ */
+export async function validateUserData(label, schema, userData) {
+    const { label: _label, ...rest } = (userData ?? {});
+    // `label` is a Crawlee-managed key that lives inside `userData`, so validating it is opt-in: we validate
+    // without it first, letting schemas that don't describe it pass (including `.strict()` ones). A schema that
+    // *does* declare `label` reports an issue for the now-missing key — so we re-validate with it included,
+    // honouring the declaration. Unlike `userData.__crawlee`, `label` is enumerable, so schemas do see it.
+    let result = await schema['~standard'].validate(rest);
+    if (result.issues?.some(isLabelIssue)) {
+        result = await schema['~standard'].validate({ ...rest, label });
+    }
+    if (result.issues) {
+        throw new RequestValidationError(label, result.issues);
+    }
+    // Restore the label so it survives schemas that strip undeclared keys.
+    return { ...result.value, label };
+}
 /**
  * Simple router that works based on request labels. This instance can then serve as a `requestHandler` of your crawler.
  *
@@ -64,95 +98,207 @@ const defaultRoute = Symbol('default-route');
  *    ctx.log.info('...');
  * });
  * ```
+ *
+ * To get `request.userData` typed per label, declare a route map and pass it as the second
+ * type argument. The label passed to {@link Router.addHandler} then drives the type of
+ * `request.userData`, and unknown labels are rejected at compile time:
+ *
+ * ```ts
+ * import { createCheerioRouter, CheerioCrawlingContext } from 'crawlee';
+ *
+ * interface Routes {
+ *     PRODUCT: { sku: string; price: number };
+ *     CATEGORY: { categoryId: string };
+ * }
+ *
+ * const router = createCheerioRouter<CheerioCrawlingContext, Routes>();
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.sku;   // string
+ *     request.userData.price; // number
+ * });
+ *
+ * router.addHandler('TYPO', async () => {}); // compile error: not a known label
+ * ```
+ *
+ * Passing a [Standard Schema](https://standardschema.dev) per label instead of a plain type both infers the
+ * `request.userData` types *and* validates them at runtime — when the request is handled, and when it is
+ * added to the crawler (`crawler.addRequests`, `context.addRequests`, `enqueueLinks`). A failing request
+ * throws a {@link RequestValidationError}.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { createCheerioRouter } from 'crawlee';
+ *
+ * const router = createCheerioRouter({
+ *     PRODUCT: z.object({ sku: z.string(), price: z.number() }),
+ *     CATEGORY: z.object({ categoryId: z.string() }),
+ * });
+ *
+ * router.addHandler('PRODUCT', async ({ request }) => {
+ *     request.userData.price; // number, inferred from the schema and validated at runtime
+ * });
+ * ```
+ *
+ * A single route can take longer than the rest without raising the crawler-wide
+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
+ *
+ * ```ts
+ * // LIST pages scroll through a lot of content, DETAIL pages are quick
+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
+ * ```
+ *
+ * When the time a route needs is only apparent once it is already running, call
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
+ *
+ * ```ts
+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {
+ *     const pageCount = await countPages(page);
+ *     extendTimeout(pageCount * 10); // ask for 10 more seconds per page
+ *     await scrapeAllPages(page);
+ * });
+ * ```
  */
 export class Router {
-    routes = new Map();
-    middlewares = [];
+    #routes = new Map();
+    #schemas = new Map();
+    #timeouts = new Map();
+    #middlewares = [];
     /**
      * use Router.create() instead!
      * @ignore
      */
     constructor() { }
-    /**
-     * Registers new route handler for given label.
-     */
-    addHandler(label, handler) {
+    addHandler(label, handler, options = {}) {
         this.validate(label);
-        this.routes.set(label, handler);
+        this.#routes.set(label, handler);
+        if (options.requestHandlerTimeoutSecs !== undefined) {
+            this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
+        }
     }
     /**
-     * Registers default route handler.
+     * Registers default route handler. As a fallback it can receive any request (including labels not
+     * declared in the route map). When the router was created with a {@link defaultRoute} schema,
+     * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
+     * Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
+     * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
+     * through to it.
      */
-    addDefaultHandler(handler) {
+    addDefaultHandler(handler, options = {}) {
         this.validate(defaultRoute);
-        this.routes.set(defaultRoute, handler);
+        this.#routes.set(defaultRoute, handler);
+        if (options.requestHandlerTimeoutSecs !== undefined) {
+            this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
+        }
+    }
+    /**
+     * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
+     * to validate `request.userData` when requests are added.
+     * @internal
+     */
+    getSchema(label) {
+        if (label != null) {
+            const schema = this.#schemas.get(label);
+            if (schema) {
+                return schema;
+            }
+            // A label with its own route is fully specified; don't fall back to the default-route schema.
+            if (this.#routes.has(label)) {
+                return undefined;
+            }
+        }
+        // Requests with no route of their own fall through to the default handler, so validate their
+        // `userData` against the default-route schema, if one was registered.
+        return this.#schemas.get(defaultRoute);
     }
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
      */
     use(middleware) {
-        this.middlewares.push(middleware);
+        this.#middlewares.push(middleware);
+    }
+    /**
+     * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
+     * override it and the crawler's own timeout should apply. Falls back to the default route the same way
+     * {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
+     * the default route asked for. Used by the crawler; not meant to be called directly.
+     */
+    getTimeoutSecs(label) {
+        if (label && this.#routes.has(label)) {
+            return this.#timeouts.get(label);
+        }
+        return this.#timeouts.get(defaultRoute);
+    }
+    /**
+     * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
+     * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
+     */
+    getMaxTimeoutSecs() {
+        return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
     }
     /**
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
     getHandler(label) {
-        if (label && this.routes.has(label)) {
-            return this.routes.get(label);
+        if (label && this.#routes.has(label)) {
+            return this.#routes.get(label);
         }
-        if (this.routes.has(defaultRoute)) {
-            return this.routes.get(defaultRoute);
+        if (this.#routes.has(defaultRoute)) {
+            return this.#routes.get(defaultRoute);
         }
         throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
             ' You must set up a route for this label or a default route.' +
             ' Use `requestHandler`, `router.addHandler` or `router.addDefaultHandler`.');
     }
+    /**
+     * Validates `request.userData` against the schema registered for its label (if any), replacing it with
+     * the parsed value. Throws a {@link RequestValidationError} when validation fails.
+     */
+    async validateRequest(context) {
+        const label = context.request.label;
+        const schema = this.getSchema(label);
+        if (schema) {
+            context.request.userData = (await validateUserData(label, schema, context.request.userData));
+        }
+    }
     /**
      * Throws when the label already exists in our registry.
      */
     validate(label) {
-        if (this.routes.has(label)) {
+        if (this.#routes.has(label)) {
             const message = label === defaultRoute
                 ? `Default route is already defined!`
                 : `Route for label '${String(label)}' is already defined!`;
             throw new Error(message);
         }
     }
-    /**
-     * Creates new router instance. This instance can then serve as a `requestHandler` of your crawler.
-     *
-     * ```ts
-     * import { Router, CheerioCrawler, CheerioCrawlingContext } from 'crawlee';
-     *
-     * const router = Router.create<CheerioCrawlingContext>();
-     * router.addHandler('label-a', async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     * router.addDefaultHandler(async (ctx) => {
-     *    ctx.log.info('...');
-     * });
-     *
-     * const crawler = new CheerioCrawler({
-     *     requestHandler: router,
-     * });
-     * await crawler.run();
-     * ```
-     */
-    static create(routes) {
+    static create(routesOrSchemas) {
         const router = new Router();
         const obj = Object.create(Function.prototype);
         obj.addHandler = router.addHandler.bind(router);
         obj.addDefaultHandler = router.addDefaultHandler.bind(router);
+        obj.getSchema = router.getSchema.bind(router);
         obj.getHandler = router.getHandler.bind(router);
+        obj.getTimeoutSecs = router.getTimeoutSecs.bind(router);
+        obj.getMaxTimeoutSecs = router.getMaxTimeoutSecs.bind(router);
         obj.use = router.use.bind(router);
-        for (const [label, handler] of Object.entries(routes ?? {})) {
-            router.addHandler(label, handler);
+        // `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
+        for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {
+            const value = routesOrSchemas[label];
+            if (typeof value === 'function') {
+                router.addHandler(label, value);
+            }
+            else {
+                router.#schemas.set(label, value);
+            }
         }
         const func = async function (context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
-            for (const middleware of router.middlewares) {
+            await router.validateRequest(context);
+            for (const middleware of router.#middlewares) {
                 await middleware(context);
             }
             return router.getHandler(label)(context);
@@ -161,4 +307,3 @@ export class Router {
         return func;
     }
 }
-//# sourceMappingURL=router.js.map

package/serialization.d.ts

@@ -29,4 +29,3 @@ export declare function deserializeArray<T extends string | Buffer>(compressedDa
  * @internal
  */
 export declare function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
-//# sourceMappingURL=serialization.d.ts.map

package/serialization.js

@@ -11,32 +11,31 @@ const pipeline = util.promisify(streamPipeline);
  * @internal
  */
 class ArrayToJson extends Readable {
-    data;
-    offset = 0;
-    batchSize;
+    #offset = 0;
+    #batchSize;
+    #data;
     constructor(data, options = {}) {
         super({
             ...options,
             autoDestroy: true,
             emitClose: true,
         });
-        this.data = data;
         const { batchSize = 10000 } = options;
-        this.batchSize = batchSize;
-        this.data = data;
+        this.#batchSize = batchSize;
+        this.#data = data;
         this.push('[');
     }
     _read() {
         try {
-            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
             if (items.length) {
                 const json = JSON.stringify(items);
                 // Strip brackets to flatten the batch.
                 const itemString = json.substring(1, json.length - 1);
-                if (this.offset > 0)
+                if (this.#offset > 0)
                     this.push(',', 'utf8');
                 this.push(itemString, 'utf8');
-                this.offset += this.batchSize;
+                this.#offset += this.#batchSize;
             }
             else {
                 this.push(']');
@@ -116,7 +115,6 @@ function createChunkCollector(options = {}) {
 }
 function pluckValue(streamArray) {
     const realPush = streamArray.push.bind(streamArray);
-    streamArray.push = (obj) => realPush(obj && obj.value);
+    streamArray.push = (obj) => realPush(obj?.value ?? null);
     return streamArray;
 }
-//# sourceMappingURL=serialization.js.map

package/service_locator.d.ts

@@ -0,0 +1,165 @@
+import type { StorageBackend } from '@crawlee/types';
+import { Configuration } from './configuration.js';
+import type { EventManager } from './events/event_manager.js';
+import type { CrawleeLogger } from './log.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
+interface ServiceLocatorInterface {
+    /**
+     * Get the configuration.
+     * Creates a default Configuration instance if none has been set.
+     */
+    getConfiguration(): Configuration;
+    /**
+     * Set the configuration.
+     *
+     * @param configuration The configuration to set
+     * @throws {ServiceConflictError} If a different configuration has already been retrieved
+     */
+    setConfiguration(configuration: Configuration): void;
+    /**
+     * Get the event manager.
+     * Creates a default LocalEventManager instance if none has been set.
+     */
+    getEventManager(): EventManager;
+    /**
+     * Set the event manager.
+     *
+     * @param eventManager The event manager to set
+     * @throws {ServiceConflictError} If a different event manager has already been retrieved
+     */
+    setEventManager(eventManager: EventManager): void;
+    /**
+     * Get the storage backend.
+     * Creates a default storage backend if none has been set — `FileSystemStorageBackend` when
+     * `persistStorage` is enabled (the default), `MemoryStorageBackend` otherwise.
+     */
+    getStorageBackend(): StorageBackend;
+    /**
+     * Set the storage backend.
+     *
+     * @param storageBackend The storage backend to set
+     * @throws {ServiceConflictError} If a different storage backend has already been retrieved
+     */
+    setStorageBackend(storageBackend: StorageBackend): void;
+    /**
+     * Get the logger.
+     * Returns the default `@apify/log` logger if none has been set.
+     */
+    getLogger(): CrawleeLogger;
+    /**
+     * Set the logger.
+     *
+     * @param logger The logger to set
+     * @throws {ServiceConflictError} If a different logger has already been retrieved
+     */
+    setLogger(logger: CrawleeLogger): void;
+    /**
+     * Get a child logger with the given prefix.
+     * Equivalent to `getLogger().child({ prefix })`.
+     */
+    getChildLog(prefix: string): CrawleeLogger;
+    /**
+     * Get the storage instance manager (shared across all storage types).
+     */
+    getStorageInstanceManager(): StorageInstanceManager;
+    /**
+     * Returns the currently set services without triggering the implicit creation of defaults.
+     * Used to inherit already-materialized services into crawler-scoped service locators.
+     * @internal
+     */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    /**
+     * Resets the service locator to its initial state.
+     * Used mainly for testing purposes.
+     * @internal
+     */
+    reset(): void;
+}
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export declare class ServiceLocator implements ServiceLocatorInterface {
+    #private;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
+    /** @internal */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    getConfiguration(): Configuration;
+    setConfiguration(configuration: Configuration): void;
+    getEventManager(): EventManager;
+    setEventManager(eventManager: EventManager): void;
+    getStorageBackend(): StorageBackend;
+    setStorageBackend(storageBackend: StorageBackend): void;
+    getLogger(): CrawleeLogger;
+    setLogger(logger: CrawleeLogger): void;
+    getChildLog(prefix: string): CrawleeLogger;
+    getStorageInstanceManager(): StorageInstanceManager;
+    reset(): void;
+}
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
+    run: <T>(fn: () => T) => T;
+    enterScope: () => void;
+    exitScope: () => void;
+};
+export declare const serviceLocator: ServiceLocatorInterface;
+export {};