@crawlee/http 3.17.1-beta.7 → 3.17.1-beta.71

package/index.mjs

@@ -52,6 +52,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -82,6 +83,7 @@ export const createFileRouter = mod.createFileRouter;
 export const createHttpRouter = mod.createHttpRouter;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -102,5 +104,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/file-download.d.ts

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Request } from 'got-scraping';
-import type { ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterRoutes } from '../index';
+import type { ErrorHandler, GetUserDataFromRequest, HttpCrawlerOptions, InternalHttpCrawlingContext, InternalHttpHook, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index';
 import { HttpCrawler } from '../index';
 export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData, JSONData>>;
@@ -124,6 +124,7 @@ export declare class FileDownload extends HttpCrawler<FileDownloadCrawlingContex
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("../index").RouterHandler<Context>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 export {};

package/internals/file-download.js

@@ -209,30 +209,6 @@ class FileDownload extends index_1.HttpCrawler {
     }
 }
 exports.FileDownload = FileDownload;
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link FileDownload}.
- * Defaults to the {@link FileDownloadCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<FileDownloadCrawlingContext>()`.
- *
- * ```ts
- * import { FileDownload, createFileRouter } from 'crawlee';
- *
- * const router = createFileRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new FileDownload({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-function createFileRouter(routes) {
-    return index_1.Router.create(routes);
+function createFileRouter(routesOrSchemas) {
+    return index_1.Router.create(routesOrSchemas);
 }

package/internals/http-crawler.d.ts

@@ -1,6 +1,6 @@
 import type { IncomingMessage } from 'node:http';
 import type { Readable } from 'node:stream';
-import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterRoutes, Session } from '@crawlee/basic';
+import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas, Session } from '@crawlee/basic';
 import { BasicCrawler, Configuration, CrawlerExtension } from '@crawlee/basic';
 import type { HttpResponse } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
@@ -426,7 +426,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
 // @ts-ignore optional peer dependency or compatibility with es2022
         searchParams?: string | import("got-scraping", { with: { "resolution-mode": "import" } }).SearchParameters | URLSearchParams | undefined;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        dnsLookup?: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).default["lookup"] | undefined;
+        dnsLookup?: import("net").LookupFunction | undefined;
 // @ts-ignore optional peer dependency or compatibility with es2022
         dnsCache?: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).default | boolean | undefined;
         context?: Record<string, unknown> | undefined;
@@ -525,6 +525,7 @@ interface RequestFunctionOptions {
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
 export {};

package/internals/http-crawler.js

@@ -709,30 +709,6 @@ function parseContentTypeFromResponse(response) {
         charset: parsedContentType.parameters.charset,
     };
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
- * Defaults to the {@link HttpCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<HttpCrawlingContext>()`.
- *
- * ```ts
- * import { HttpCrawler, createHttpRouter } from 'crawlee';
- *
- * const router = createHttpRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new HttpCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
-function createHttpRouter(routes) {
-    return basic_1.Router.create(routes);
+function createHttpRouter(routesOrSchemas) {
+    return basic_1.Router.create(routesOrSchemas);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "3.17.1-beta.7",
+    "version": "3.17.1-beta.71",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -53,11 +53,12 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.0",
+        "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/basic": "3.17.1-beta.7",
-        "@crawlee/types": "3.17.1-beta.7",
-        "@crawlee/utils": "3.17.1-beta.7",
+        "@crawlee/basic": "3.17.1-beta.71",
+        "@crawlee/core": "3.17.1-beta.71",
+        "@crawlee/types": "3.17.1-beta.71",
+        "@crawlee/utils": "3.17.1-beta.71",
         "@types/content-type": "^1.1.5",
         "cheerio": "1.0.0-rc.12",
         "content-type": "^1.0.4",
@@ -75,5 +76,5 @@
             }
         }
     },
-    "gitHead": "541c9687ee7f8421189d489ea817aaa08cb9b4ec"
+    "gitHead": "94c4c029140e4068184ad3f03d0a71fd8d6984dc"
 }