@crawlee/http 4.0.0-beta.134 → 4.0.0-beta.136

package/internals/file-download.d.ts

@@ -1,11 +1,9 @@
 import { Transform } from 'node:stream';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
-import { ResponseWithUrl } from '@crawlee/http-client';
+import type { ContextPipeline, CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
-declare const kBodyDrained: unique symbol;
 export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
 export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
@@ -88,16 +86,7 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  */
 export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
     constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/basic").ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
-        request: LoadedRequest<Request>;
-        response: ResponseWithUrl;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        [kBodyDrained]: Promise<void>;
-    }>;
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, FileDownloadCrawlingContext>;
     private initiateDownload;
 }
 /**
@@ -126,4 +115,3 @@ export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingConte
  */
 // @ts-ignore optional peer dependency or compatibility with es2022
 export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context, Record<string, GetUserDataFromRequest<Context["request"]>>>;
-export {};

package/internals/http-crawler.d.ts

@@ -2,7 +2,7 @@ import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions,
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils/internal';
+import type { CheerioAPI } from 'cheerio';
 import type { JsonValue } from 'type-fest';
 import { z } from 'zod';
 /**
@@ -172,7 +172,7 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
 }
 export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
 }
@@ -249,6 +249,9 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
@@ -304,6 +307,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
     };
+    /** @internal */
     protected static optionsSchema: z.ZodObject<{
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;

package/internals/http-crawler.js

@@ -1,9 +1,9 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
 import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
-import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
+import { RequestThrottledError, getCookiesFromResponse } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
-import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
+import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
 import { z } from 'zod';
@@ -117,6 +117,9 @@ export class HttpCrawler extends BasicCrawler {
     #suggestResponseEncoding;
     #forceResponseEncoding;
     #supportedMimeTypes;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: schemas.anyNumber.default(30),
@@ -128,6 +131,7 @@ export class HttpCrawler extends BasicCrawler {
         preNavigationHooks: schemas.anyArray.default(() => []),
         postNavigationHooks: schemas.anyArray.default(() => []),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
     /**
      * All `HttpCrawlerOptions` parameters are passed via an options object.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.134",
+    "version": "4.0.0-beta.136",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.134",
-        "@crawlee/core": "4.0.0-beta.134",
-        "@crawlee/http-client": "4.0.0-beta.134",
-        "@crawlee/types": "4.0.0-beta.134",
-        "@crawlee/utils": "4.0.0-beta.134",
+        "@crawlee/basic": "4.0.0-beta.136",
+        "@crawlee/core": "4.0.0-beta.136",
+        "@crawlee/http-client": "4.0.0-beta.136",
+        "@crawlee/types": "4.0.0-beta.136",
+        "@crawlee/utils": "4.0.0-beta.136",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "0f269d5b24ec2829f3d44c19a35973fa20d95999"
+    "gitHead": "d425bb9841885b4b71b84ae6fb0bafbdc4a7e978"
 }