@crawlee/cheerio 3.17.1-beta.8 → 3.17.1-beta.80

package/index.mjs

@@ -53,6 +53,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -85,6 +86,7 @@ export const createFileRouter = mod.createFileRouter;
 export const createHttpRouter = mod.createHttpRouter;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -105,5 +107,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/cheerio-crawler.d.ts

@@ -1,5 +1,5 @@
 import type { IncomingMessage } from 'node:http';
-import type { BasicCrawlingContext, Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterRoutes, SkippedRequestCallback } from '@crawlee/http';
+import type { BasicCrawlingContext, Configuration, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RequestProvider, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, SkippedRequestCallback } from '@crawlee/http';
 import { HttpCrawler } from '@crawlee/http';
 import type { Dictionary } from '@crawlee/types';
 import { type CheerioRoot, type RobotsTxtFile } from '@crawlee/utils';
@@ -184,6 +184,7 @@ export declare function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternal
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/http").RouterHandler<Context>;
+export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 export {};

package/internals/cheerio-crawler.js

@@ -112,7 +112,9 @@ class CheerioCrawler extends http_1.HttpCrawler {
             body,
             enqueueLinks: async (enqueueOptions) => {
                 return cheerioCrawlerEnqueueLinks({
-                    options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
+                    // `originalEnqueueLinks` clamps `limit` by the remaining `maxRequestsPerCrawl` budget itself;
+                    // pre-clamping it here would make the crawler log the internal limit as a user-provided one
+                    options: enqueueOptions,
                     $,
                     requestQueue: await this.getRequestQueue(),
                     robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
@@ -174,8 +176,8 @@ async function cheerioCrawlerEnqueueLinks(options) {
     if (containsEnqueueLinks(options)) {
         return options.enqueueLinks({
             urls,
-            baseUrl,
             ...enqueueLinksOptions,
+            baseUrl,
         });
     }
     return (0, http_1.enqueueLinks)({
@@ -183,34 +185,10 @@ async function cheerioCrawlerEnqueueLinks(options) {
         robotsTxtFile: options.robotsTxtFile,
         onSkippedRequest: options.onSkippedRequest,
         urls,
-        baseUrl,
         ...enqueueLinksOptions,
+        baseUrl,
     });
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link CheerioCrawler}.
- * Defaults to the {@link CheerioCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<CheerioCrawlingContext>()`.
- *
- * ```ts
- * import { CheerioCrawler, createCheerioRouter } from 'crawlee';
- *
- * const router = createCheerioRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new CheerioCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-function createCheerioRouter(routes) {
-    return http_1.Router.create(routes);
+function createCheerioRouter(routesOrSchemas) {
+    return http_1.Router.create(routesOrSchemas);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/cheerio",
-    "version": "3.17.1-beta.8",
+    "version": "3.17.1-beta.80",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -53,9 +53,9 @@
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/http": "3.17.1-beta.8",
-        "@crawlee/types": "3.17.1-beta.8",
-        "@crawlee/utils": "3.17.1-beta.8",
+        "@crawlee/http": "3.17.1-beta.80",
+        "@crawlee/types": "3.17.1-beta.80",
+        "@crawlee/utils": "3.17.1-beta.80",
         "cheerio": "1.0.0-rc.12",
         "htmlparser2": "^9.0.0",
         "tslib": "^2.4.0"
@@ -67,5 +67,5 @@
             }
         }
     },
-    "gitHead": "4f4b8c1d53b9185095ac68017aff0ef2b8182b6b"
+    "gitHead": "5b3c2d52f128643de6f515e87eeb531d55a187f0"
 }