@crawlee/core 4.0.0-beta.97 → 4.0.0-beta.99

package/crawlers/crawler_commons.d.ts

@@ -17,7 +17,6 @@ export type IsAny<T> = 0 extends 1 & T ? true : false;
  * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
  * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
  * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
- * @internal
  */
 export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
     requestsFromUrl?: string;
@@ -38,7 +37,6 @@ export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> =
 /**
  * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
  * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
- * @internal
  */
 export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
 /**
@@ -59,10 +57,8 @@ type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dicti
  * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
  * the router's route map, while preserving everything else about the signature (argument optionality and
  * return type, which differ between crawler types).
- * @internal
  */
 export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
-/** @internal */
 export type WithRequired<T, K extends keyof T> = T & {
     [P in K]-?: T[P];
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.97",
+    "version": "4.0.0-beta.99",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.97",
-        "@crawlee/types": "4.0.0-beta.97",
-        "@crawlee/utils": "4.0.0-beta.97",
+        "@crawlee/fs-storage": "4.0.0-beta.99",
+        "@crawlee/types": "4.0.0-beta.99",
+        "@crawlee/utils": "4.0.0-beta.99",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
+    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
 }

package/router.d.ts

@@ -48,7 +48,6 @@ export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
 /**
  * The `userData` type of the default route: inferred from the {@link defaultRoute} schema when the route map
  * carries one, otherwise the provided `Fallback`.
- * @internal
  */
 export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
     [defaultRoute]: infer DefaultUserData extends Dictionary;

package/storages/request_queue.d.ts

@@ -310,9 +310,6 @@ export interface RequestQueueOperationOptions {
      */
     cache?: boolean;
 }
-/**
- * @internal
- */
 export interface RequestQueueOperationInfo extends QueueOperationInfo {
     uniqueKey: string;
     forefront: boolean;

package/storages/request_queue.js

@@ -17,11 +17,6 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @internal
  */
 const MAX_CACHED_REQUESTS = 2_000_000;
-/**
- * The maximum number of consecutive no-progress retries for unprocessed requests in `addRequestsBatched()`.
- * @internal
- */
-const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -327,32 +322,27 @@ export class RequestQueue {
         const requestIterator = generateRequests();
         const chunks = peekableAsyncIterable(chunkedAsyncIterable(requestIterator, effectiveChunkSize));
         const chunksIterator = chunks[Symbol.asyncIterator]();
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true, unsuccessfulAttempts = 0) => {
-            const resultsToReturn = [];
-            const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
-            resultsToReturn.push(...apiResult.processedRequests);
-            if (apiResult.unprocessedRequests.length) {
-                // Count attempts that make no progress, so permanently rejected requests (e.g. a malformed
-                // `userData` shape causing a 400) don't loop forever. Any progress resets the counter.
-                const attempts = apiResult.processedRequests.length ? 0 : unsuccessfulAttempts + 1;
-                if (attempts >= MAX_UNPROCESSED_REQUESTS_RETRIES) {
-                    this.log.warning(`Some requests were consistently rejected by the request queue and will be skipped after ${MAX_UNPROCESSED_REQUESTS_RETRIES} attempts. This usually means the request data is malformed (e.g. an invalid 'userData' shape).`, { unprocessedRequests: apiResult.unprocessedRequests });
-                    return resultsToReturn;
-                }
-                await sleep(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false, attempts)));
-            }
-            return resultsToReturn;
-        };
         /**
          * Process a chunk: send it to the queue, then update the remaining budget if maxNewRequests is active.
+         *
+         * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
+         * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
+         * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
+         * Retrying transient failures is the storage backend's job, not the frontend's.
          */
         const processChunk = async (chunk, cache = true) => {
-            const results = await attemptToAddToQueueAndAddAnyUnprocessed(chunk, cache);
+            const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
+                forefront: options.forefront,
+                cache,
+            });
+            if (unprocessedRequests.length > 0) {
+                this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
+                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
+            }
             if (maxNewRequests !== undefined) {
-                remainingBudget -= results.filter((r) => !r.wasAlreadyPresent).length;
+                remainingBudget -= processedRequests.filter((r) => !r.wasAlreadyPresent).length;
             }
-            return results;
+            return processedRequests;
         };
         /**
          * Build the final result. When maxNewRequests is set, drains any remaining items

package/storages/storage_instance_manager.d.ts

@@ -17,8 +17,6 @@ type Hashable = string;
  * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
  * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
  * assigns a reserved default alias.
- *
- * @ignore
  */
 export declare class StorageInstanceManager {
     private readonly cache;

package/storages/storage_instance_manager.js

@@ -118,8 +118,6 @@ class StorageCache {
  * The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
  * and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
  * assigns a reserved default alias.
- *
- * @ignore
  */
 export class StorageInstanceManager {
     cache = new StorageCache();