@crawlee/basic 4.0.0-beta.220 → 4.0.0-beta.221

package/internals/basic-crawler.d.ts

@@ -297,7 +297,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * 1. based on robots.txt file,
      * 2. because they don't match enqueueLinks filters,
      * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
-     * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+     * 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
+     * 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
      */
     onSkippedRequest?: SkippedRequestCallback;
     /**

package/internals/basic-crawler.js

@@ -583,13 +583,7 @@ export class BasicCrawler {
     async #checkRobotsTxt({ request }) {
         if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
-            request.state = RequestState.SKIPPED;
-            request.noRetry = true;
-            await this.#handleSkippedRequest({
-                request,
-                reason: 'robotsTxt',
-            });
-            throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
+            throw new ContextPipelineInterruptedError('robotsTxt');
         }
         return {};
     }
@@ -635,6 +629,10 @@ export class BasicCrawler {
                     this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
                 });
             },
+            skipRequest: (message) => {
+                this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
+                throw new ContextPipelineInterruptedError('manual', message);
+            },
         };
     }
     /**
@@ -700,10 +698,7 @@ export class BasicCrawler {
                 // eslint-disable-next-line dot-notation
                 const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
                 this.log.debug(message);
-                request.noRetry = true;
-                request.state = RequestState.SKIPPED;
-                await this.#handleSkippedRequest({ request, reason: 'redirect' });
-                throw new ContextPipelineInterruptedError(message);
+                throw new ContextPipelineInterruptedError('redirect');
             }
             return context;
         });
@@ -1525,23 +1520,30 @@ export class BasicCrawler {
         // Roll back *before* any error handler runs - error handlers write to real storage precisely
         // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
         currentStorageTransaction()?.rollback();
-        const skipped = rawError instanceof ContextPipelineInitializationError &&
-            rawError.cause instanceof ContextPipelineInterruptedError;
-        const err = this.#unwrapError(rawError);
-        const context = crawlingContext;
-        const retryCountBefore = request.retryCount;
         const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
         try {
-            // ContextPipelineInterruptedError means the request was intentionally skipped
-            // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-            if (skipped) {
-                this.statistics.discardRequestRecord(request.id || request.uniqueKey);
-                await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                return;
+            const unwrappedError = this.#unwrapError(rawError);
+            // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
+            const err = unwrappedError instanceof ContextPipelineInterruptedError
+                ? await this.#trySkipRequest(unwrappedError, request, requestSource)
+                : unwrappedError;
+            if (err) {
+                await this.#runErrorHandler(err, crawlingContext, requestSource, request);
             }
+        }
+        finally {
+            resumeRequestTimeout?.();
+        }
+    }
+    async #runErrorHandler(err, context, requestSource, request) {
+        const retryCountBefore = request.retryCount;
+        try {
             request.state = RequestState.ERROR_HANDLER;
             await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-            request.state = RequestState.DONE;
+            // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
+            if (request.state !== RequestState.SKIPPED) {
+                request.state = RequestState.DONE;
+            }
         }
         catch (secondaryError) {
             const unwrappedSecondaryError = this.#unwrapError(secondaryError);
@@ -1551,7 +1553,7 @@ export class BasicCrawler {
                 // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
                 // retry is counted, so that an error handler that always times out does not retry it forever.
                 this.log.warning(unwrappedSecondaryError.message);
-                if (!skipped && request.retryCount === retryCountBefore) {
+                if (request.retryCount === retryCountBefore) {
                     request.retryCount++;
                 }
             }
@@ -1575,12 +1577,9 @@ export class BasicCrawler {
                 throw unwrappedSecondaryError;
             }
         }
-        finally {
-            resumeRequestTimeout?.();
-        }
         // decrease the session score if the request fails (but the error handler did not throw);
         // skip when the error is a SessionError, which already retired the session
-        if (!skipped && !this.#errorAbsolvesSession(err)) {
+        if (!this.#errorAbsolvesSession(err)) {
             context.session?.markBad();
         }
     }
@@ -1671,8 +1670,26 @@ export class BasicCrawler {
         const shouldRetryRequest = this.#canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
-            await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
-            error);
+            try {
+                await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+                error);
+            }
+            catch (handlerError) {
+                if (handlerError instanceof ContextPipelineInterruptedError) {
+                    const skipError = await this.#trySkipRequest(handlerError, request, source);
+                    if (!skipError) {
+                        return;
+                    }
+                    // Retry for the original error. Routing `skipError` through `errorHandler` again
+                    // would loop forever with a handler that always skips.
+                    this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
+                        error: skipError.message,
+                    });
+                }
+                else {
+                    throw handlerError;
+                }
+            }
             if (error instanceof SessionError) {
                 crawlingContext.session?.retire();
             }
@@ -1711,16 +1728,35 @@ export class BasicCrawler {
         this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
+    /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
+    async #trySkipRequest(error, request, source) {
+        try {
+            await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            request.state = RequestState.SKIPPED;
+            request.noRetry = true;
+            await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+            return undefined;
+        }
+        catch (settleError) {
+            return this.#unwrapError(settleError);
+        }
+    }
     // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
         const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
-        if (this.#failedRequestHandler) {
+        try {
             await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
         }
+        catch (handlerError) {
+            if (!(handlerError instanceof ContextPipelineInterruptedError))
+                throw handlerError;
+            this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
+        }
     }
     /**
      * Resolves the most verbose error message from a thrown error
@@ -1747,7 +1783,9 @@ export class BasicCrawler {
      * failure says nothing about the session (a rate limit is a property of the domain).
      */
     #errorAbsolvesSession(error) {
-        return error instanceof SessionError || error instanceof RequestThrottledError;
+        return (error instanceof SessionError ||
+            error instanceof RequestThrottledError ||
+            error instanceof ContextPipelineInterruptedError);
     }
     #canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.

package/internals/crawlers/crawler_commons.d.ts

@@ -102,6 +102,28 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * A preconfigured logger for the request handler.
      */
     log: CrawleeLogger;
+    /**
+     * Stops processing the current request and marks it as handled without counting it as a failure: no retry, no
+     * `failedRequestHandler`, and no error log. The request's `state` becomes {@link RequestState.SKIPPED|`SKIPPED`}
+     * and {@link BasicCrawlerOptions.onSkippedRequest|`onSkippedRequest`} fires with the `'manual'` reason.
+     *
+     * Storage writes made for the request so far are rolled back, as with a failure, unless
+     * {@link BasicCrawlerOptions.transactionalStorage|`transactionalStorage`} is disabled.
+     *
+     * Works in `errorHandler` too, skipping a request that would otherwise be retried. In `failedRequestHandler` it
+     * has no effect, since the request has already failed.
+     *
+     * ```ts
+     * preNavigationHooks: [
+     *     async ({ request, skipRequest }) => {
+     *         if (await alreadyScraped(request.url)) skipRequest('already scraped');
+     *     },
+     * ],
+     * ```
+     *
+     * @param message Passed to `onSkippedRequest`.
+     */
+    skipRequest(message?: string): never;
 }
 export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
     /**

package/internals/crawling_request.d.ts

@@ -1,7 +1,7 @@
 import type { EnqueueStrategyOption } from '@crawlee/core';
 import { Request } from '@crawlee/core';
 import type { Dictionary, RequestSchema } from '@crawlee/types';
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth' | 'manual';
 export declare enum RequestState {
     UNPROCESSED = 0,
     BEFORE_NAV = 1,

package/internals/enqueue_links/shared.d.ts

@@ -21,6 +21,8 @@ export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
 export type SkippedRequestCallback = (args: {
     request: Request;
     reason: SkippedRequestReason;
+    /** The `message` passed to {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}, if any. */
+    message?: string;
 }) => Awaitable<void>;
 /**
  * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the

package/internals/errors.d.ts

@@ -1,4 +1,5 @@
 import { CriticalError, NonRetryableError } from '@crawlee/core';
+import type { SkippedRequestReason } from './crawling_request.js';
 /**
  * @ignore
  */
@@ -38,8 +39,15 @@ export declare class PersistentRateLimitError extends CriticalError {
 export declare class MissingSessionError extends Error {
     constructor(sessionId?: string);
 }
+/**
+ * Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
+ * Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
+ * @internal
+ */
 export declare class ContextPipelineInterruptedError extends Error {
-    constructor(message?: string);
+    readonly reason: SkippedRequestReason;
+    readonly skipMessage?: string | undefined;
+    constructor(reason: SkippedRequestReason, skipMessage?: string | undefined);
 }
 export declare class ContextPipelineCleanupError extends CriticalError {
     constructor(error: unknown, options?: ErrorOptions);

package/internals/errors.js

@@ -44,9 +44,18 @@ export class MissingSessionError extends Error {
         super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
     }
 }
+/**
+ * Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
+ * Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
+ * @internal
+ */
 export class ContextPipelineInterruptedError extends Error {
-    constructor(message) {
-        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+    reason;
+    skipMessage;
+    constructor(reason, skipMessage) {
+        super(skipMessage ?? 'Request was skipped');
+        this.reason = reason;
+        this.skipMessage = skipMessage;
     }
 }
 export class ContextPipelineCleanupError extends CriticalError {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.220",
+    "version": "4.0.0-beta.221",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.220",
-        "@crawlee/http-client": "4.0.0-beta.220",
-        "@crawlee/types": "4.0.0-beta.220",
-        "@crawlee/utils": "4.0.0-beta.220",
+        "@crawlee/core": "4.0.0-beta.221",
+        "@crawlee/http-client": "4.0.0-beta.221",
+        "@crawlee/types": "4.0.0-beta.221",
+        "@crawlee/utils": "4.0.0-beta.221",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.220"
+        "@crawlee/impit-client": "4.0.0-beta.221"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "c9ef485f92968109ea496d121c9189b873c07ce8"
+    "gitHead": "08c8472ab3ae7ec4d81fedd099f616c7ed10a263"
 }