@crawlee/core 4.0.0-beta.95 → 4.0.0-beta.97

package/configuration.d.ts

@@ -24,6 +24,11 @@ export declare const crawleeConfigFields: {
     memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
     /** @default 60_000 */
     persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /**
+     * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
+     * the request handler timeout (twice it, and never below 5 minutes).
+     */
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
     /** @default 1_000 */
     systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
     /** @default 'INPUT' */
@@ -100,6 +105,7 @@ export interface Configuration extends ResolvedConfigValues {
  * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
  * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
  * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`

package/configuration.js

@@ -54,6 +54,11 @@ export const crawleeConfigFields = {
     memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
     /** @default 60_000 */
     persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
+    /**
+     * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
+     * the request handler timeout (twice it, and never below 5 minutes).
+     */
+    internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
     /** @default 1_000 */
     systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
     /** @default 'INPUT' */
@@ -118,6 +123,7 @@ export const crawleeConfigFields = {
  * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
  * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
  * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`

package/crawlers/crawler_commons.d.ts

@@ -184,6 +184,25 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
      */
     registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+    /**
+     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
+     * once it is already running - a listing page that turns out to have far more to scroll through than
+     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
+     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
+     *
+     * ```ts
+     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
+     *     const pageCount = await countPages(page);
+     *     extendTimeout(pageCount * 10);
+     *     await scrapeAllPages(page);
+     * });
+     * ```
+     *
+     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
+     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
+     * nothing.
+     */
+    extendTimeout(secs: number): void;
 }
 /**
  * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.

package/crawlers/index.d.ts

@@ -1,6 +1,5 @@
 export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/crawlers/index.js

@@ -1,6 +1,5 @@
 export * from './context_pipeline.js';
 export * from './crawler_commons.js';
-export * from './crawler_utils.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.95",
+    "version": "4.0.0-beta.97",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -51,11 +51,11 @@
         "@apify/datastructures": "^2.0.3",
         "@apify/log": "^2.5.18",
         "@apify/pseudo_url": "^2.0.59",
-        "@apify/timeout": "^0.3.2",
+        "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.95",
-        "@crawlee/types": "4.0.0-beta.95",
-        "@crawlee/utils": "4.0.0-beta.95",
+        "@crawlee/fs-storage": "4.0.0-beta.97",
+        "@crawlee/types": "4.0.0-beta.97",
+        "@crawlee/utils": "4.0.0-beta.97",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "ba8602d011706fb5c9930232402156eb5b657eb3"
+    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
 }

package/router.d.ts

@@ -70,6 +70,20 @@ export interface RouterHandler<Context extends Omit<RestrictedCrawlingContext, '
     (ctx: Context): Awaitable<void>;
 }
 export type GetUserDataFromRequest<T> = T extends Request<infer Y> ? Y : never;
+/**
+ * Per-route overrides, passed as the last argument of {@link Router.addHandler|`addHandler`} and
+ * {@link Router.addDefaultHandler|`addDefaultHandler`}.
+ */
+export interface RouteOptions {
+    /**
+     * Overrides the crawler's `requestHandlerTimeoutSecs` for this route only. Useful when one kind of page
+     * needs markedly more time than the rest - a listing page behind an infinite scroll, say - and you do not
+     * want to raise the timeout for every other page to accommodate it.
+     *
+     * Applies only to this route's handler. The navigation and the navigation hooks keep their own timeouts.
+     */
+    requestHandlerTimeoutSecs?: number;
+}
 export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
     [Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
         request: Request<Routes[Label]>;
@@ -180,10 +194,31 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
  *     request.userData.price; // number, inferred from the schema and validated at runtime
  * });
  * ```
+ *
+ * A single route can take longer than the rest without raising the crawler-wide
+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
+ *
+ * ```ts
+ * // LIST pages scroll through a lot of content, DETAIL pages are quick
+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
+ * ```
+ *
+ * When the time a route needs is only apparent once it is already running, call
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
+ *
+ * ```ts
+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {
+ *     const pageCount = await countPages(page);
+ *     extendTimeout(pageCount * 10); // ask for 10 more seconds per page
+ *     await scrapeAllPages(page);
+ * });
+ * ```
  */
 export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
     private readonly routes;
     private readonly schemas;
+    private readonly timeouts;
     private readonly middlewares;
     /**
      * use Router.create() instead!
@@ -192,22 +227,26 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
     private constructor();
     /**
      * Registers new route handler for given label. When the router declares a route map, the
-     * `label` is restricted to the declared labels and `request.userData` is typed accordingly.
+     * `label` is restricted to the declared labels and `request.userData` is typed accordingly. Pass
+     * {@link RouteOptions|`options`} to give this route its own `requestHandlerTimeoutSecs`,
+     * overriding the crawler's default for requests with this label.
      */
-    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>): void;
+    addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>, options?: RouteOptions): void;
     /**
      * Registers new route handler for given label, explicitly typing `request.userData` via the
      * `UserData` type argument. Useful when the router has no declared route map (the open default)
      * and you want to type a single handler, or to register a handler under a `symbol` label.
      */
-    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;
+    addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
     /**
      * Registers default route handler. As a fallback it can receive any request (including labels not
      * declared in the route map). When the router was created with a {@link defaultRoute} schema,
      * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
-     * Pass an explicit `UserData` type argument to narrow it.
+     * Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
+     * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
+     * through to it.
      */
-    addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>): void;
+    addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
     /**
      * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
      * to validate `request.userData` when requests are added.
@@ -219,6 +258,18 @@ export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enq
      * Multiple middlewares can be registered, they will be fired in the same order.
      */
     use(middleware: (ctx: Context) => Awaitable<void>): void;
+    /**
+     * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
+     * override it and the crawler's own timeout should apply. Falls back to the default route the same way
+     * {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
+     * the default route asked for. Used by the crawler; not meant to be called directly.
+     */
+    getTimeoutSecs(label?: string | symbol): number | undefined;
+    /**
+     * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
+     * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
+     */
+    getMaxTimeoutSecs(): number | undefined;
     /**
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */

package/router.js

@@ -139,29 +139,58 @@ export async function validateUserData(label, schema, userData) {
  *     request.userData.price; // number, inferred from the schema and validated at runtime
  * });
  * ```
+ *
+ * A single route can take longer than the rest without raising the crawler-wide
+ * `requestHandlerTimeoutSecs` for everything - pass a per-route timeout as the last argument:
+ *
+ * ```ts
+ * // LIST pages scroll through a lot of content, DETAIL pages are quick
+ * router.addHandler('LIST', async (ctx) => { ... }, { requestHandlerTimeoutSecs: 120 });
+ * router.addHandler('DETAIL', async (ctx) => { ... }); // keeps the crawler's default
+ * ```
+ *
+ * When the time a route needs is only apparent once it is already running, call
+ * {@link CrawlingContext.extendTimeout|`context.extendTimeout`} from inside the handler:
+ *
+ * ```ts
+ * router.addHandler('LIST', async ({ page, extendTimeout }) => {
+ *     const pageCount = await countPages(page);
+ *     extendTimeout(pageCount * 10); // ask for 10 more seconds per page
+ *     await scrapeAllPages(page);
+ * });
+ * ```
  */
 export class Router {
     routes = new Map();
     schemas = new Map();
+    timeouts = new Map();
     middlewares = [];
     /**
      * use Router.create() instead!
      * @ignore
      */
     constructor() { }
-    addHandler(label, handler) {
+    addHandler(label, handler, options = {}) {
         this.validate(label);
         this.routes.set(label, handler);
+        if (options.requestHandlerTimeoutSecs !== undefined) {
+            this.timeouts.set(label, options.requestHandlerTimeoutSecs);
+        }
     }
     /**
      * Registers default route handler. As a fallback it can receive any request (including labels not
      * declared in the route map). When the router was created with a {@link defaultRoute} schema,
      * `request.userData` is typed from it; otherwise it defaults to the context's (loosely typed) `userData`.
-     * Pass an explicit `UserData` type argument to narrow it.
+     * Pass an explicit `UserData` type argument to narrow it. Pass {@link RouteOptions|`options`} to give the
+     * default route its own `requestHandlerTimeoutSecs`, overriding the crawler's default for requests that fall
+     * through to it.
      */
-    addDefaultHandler(handler) {
+    addDefaultHandler(handler, options = {}) {
         this.validate(defaultRoute);
         this.routes.set(defaultRoute, handler);
+        if (options.requestHandlerTimeoutSecs !== undefined) {
+            this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
+        }
     }
     /**
      * Returns the {@link RouteSchemas|Standard Schema} registered for a label, if any. Used by the crawler
@@ -190,6 +219,25 @@ export class Router {
     use(middleware) {
         this.middlewares.push(middleware);
     }
+    /**
+     * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
+     * override it and the crawler's own timeout should apply. Falls back to the default route the same way
+     * {@link Router.getHandler|`getHandler`} does, so a label with no route of its own inherits whatever
+     * the default route asked for. Used by the crawler; not meant to be called directly.
+     */
+    getTimeoutSecs(label) {
+        if (label && this.routes.has(label)) {
+            return this.timeouts.get(label);
+        }
+        return this.timeouts.get(defaultRoute);
+    }
+    /**
+     * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
+     * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
+     */
+    getMaxTimeoutSecs() {
+        return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;
+    }
     /**
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
@@ -233,6 +281,8 @@ export class Router {
         obj.addDefaultHandler = router.addDefaultHandler.bind(router);
         obj.getSchema = router.getSchema.bind(router);
         obj.getHandler = router.getHandler.bind(router);
+        obj.getTimeoutSecs = router.getTimeoutSecs.bind(router);
+        obj.getMaxTimeoutSecs = router.getMaxTimeoutSecs.bind(router);
         obj.use = router.use.bind(router);
         // `Reflect.ownKeys` (unlike `Object.entries`) also yields the `defaultRoute` symbol key.
         for (const label of Reflect.ownKeys(routesOrSchemas ?? {})) {

package/crawlers/crawler_utils.d.ts

@@ -1,9 +0,0 @@
-import type { ISession } from '@crawlee/types';
-/**
- * Handles timeout request
- * @internal
- */
-export declare function handleRequestTimeout({ session, errorMessage }: {
-    session?: ISession;
-    errorMessage: string;
-}): void;

package/crawlers/crawler_utils.js

@@ -1,11 +0,0 @@
-import { TimeoutError } from '@apify/timeout';
-/**
- * Handles timeout request
- * @internal
- */
-export function handleRequestTimeout({ session, errorMessage }) {
-    session?.markBad();
-    const timeoutMillis = /(\d+)\s?ms/.exec(errorMessage)?.[1]; // first capturing group
-    const timeoutSecs = Number(timeoutMillis) / 1000;
-    throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
-}