@crawlee/http 3.7.1-beta.4 → 3.7.1-beta.6

package/internals/http-crawler.d.ts

@@ -1,7 +1,12 @@
+// @ts-ignore optional peer dependency or compatibility with es2022
 /// <reference types="node" />
+// @ts-ignore optional peer dependency or compatibility with es2022
 /// <reference types="node" />
+// @ts-ignore optional peer dependency or compatibility with es2022
 /// <reference types="node" />
+// @ts-ignore optional peer dependency or compatibility with es2022
 /// <reference types="node" />
+// @ts-ignore optional peer dependency or compatibility with es2022
 /// <reference types="node" />
 import type { IncomingMessage } from 'node:http';
 import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, ProxyConfiguration, Request, RequestHandler, RouterRoutes, Session } from '@crawlee/basic';
@@ -9,7 +14,7 @@ import { BasicCrawler, CrawlerExtension, Configuration } from '@crawlee/basic';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import * as cheerio from 'cheerio';
 import type { RequestLike, ResponseLike } from 'content-type';
-// @ts-ignore optional peer dependency
+// @ts-ignore optional peer dependency or compatibility with es2022
 import type { OptionsInit, Method, PlainResponse } from 'got-scraping';
 import type { JsonValue } from 'type-fest';
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -233,43 +238,81 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
     protected ignoreHttpErrorStatusCodes: Set<number>;
     protected readonly supportedMimeTypes: Set<string>;
     protected static optionsShape: {
+// @ts-ignore optional peer dependency or compatibility with es2022
         handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         ignoreSslErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         additionalMimeTypes: import("ow").ArrayPredicate<string>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
     /**
@@ -325,59 +368,85 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
     protected _getRequestOptions(request: Request, session?: Session, proxyUrl?: string, gotOptions?: OptionsInit): {
+// @ts-ignore optional peer dependency or compatibility with es2022
         request?: import("got-scraping", { with: { "resolution-mode": "import" } }).RequestFunction | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         agent?: import("got-scraping", { with: { "resolution-mode": "import" } }).Agents | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         h2session?: import("http2").ClientHttp2Session | undefined;
         decompress?: boolean | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         timeout?: import("got-scraping", { with: { "resolution-mode": "import" } }).Delays | undefined;
         prefixUrl?: string | URL | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         body?: string | Buffer | import("stream").Readable | Generator<unknown, any, unknown> | AsyncGenerator<unknown, any, unknown> | import("form-data-encoder", { with: { "resolution-mode": "import" } }).FormDataLike | undefined;
         form?: Record<string, any> | undefined;
         json?: unknown;
         url?: string | URL | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         cookieJar?: import("got-scraping", { with: { "resolution-mode": "import" } }).PromiseCookieJar | import("got-scraping", { with: { "resolution-mode": "import" } }).ToughCookieJar | undefined;
         signal?: AbortSignal | undefined;
         ignoreInvalidCookies?: boolean | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         searchParams?: string | import("got-scraping", { with: { "resolution-mode": "import" } }).SearchParameters | URLSearchParams | undefined;
         dnsLookup?: {
+// @ts-ignore optional peer dependency or compatibility with es2022
             (hostname: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
+// @ts-ignore optional peer dependency or compatibility with es2022
             (hostname: string, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
+// @ts-ignore optional peer dependency or compatibility with es2022
             (hostname: string, options: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).LookupOptions & {
                 all: true;
+// @ts-ignore optional peer dependency or compatibility with es2022
             }, callback: (error: NodeJS.ErrnoException | null, result: readonly import("cacheable-lookup", { with: { "resolution-mode": "import" } }).EntryObject[]) => void): void;
+// @ts-ignore optional peer dependency or compatibility with es2022
             (hostname: string, options: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).LookupOptions, callback: (error: NodeJS.ErrnoException | null, address: string, family: import("cacheable-lookup", { with: { "resolution-mode": "import" } }).IPFamily) => void): void;
         } | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         dnsCache?: boolean | import("cacheable-lookup", { with: { "resolution-mode": "import" } }).default | undefined;
         context?: Record<string, unknown> | undefined;
         followRedirect?: boolean | undefined;
         maxRedirects?: number | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         cache?: string | boolean | import("cacheable-request", { with: { "resolution-mode": "import" } }).StorageAdapter | undefined;
         throwHttpErrors?: boolean | undefined;
         username?: string | undefined;
         password?: string | undefined;
         http2?: boolean | undefined;
         allowGetBody?: boolean | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         headers?: import("got-scraping", { with: { "resolution-mode": "import" } }).Headers | undefined;
         methodRewriting?: boolean | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         dnsLookupIpVersion?: import("got-scraping", { with: { "resolution-mode": "import" } }).DnsLookupIpVersion;
+// @ts-ignore optional peer dependency or compatibility with es2022
         parseJson?: import("got-scraping", { with: { "resolution-mode": "import" } }).ParseJsonFunction | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         stringifyJson?: import("got-scraping", { with: { "resolution-mode": "import" } }).StringifyJsonFunction | undefined;
         localAddress?: string | undefined;
         method?: Method | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         createConnection?: import("got-scraping", { with: { "resolution-mode": "import" } }).CreateConnectionFunction | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         cacheOptions?: import("got-scraping", { with: { "resolution-mode": "import" } }).CacheOptions | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         https?: import("got-scraping", { with: { "resolution-mode": "import" } }).HttpsOptions | undefined;
         encoding?: BufferEncoding | undefined;
         resolveBodyOnly?: boolean | undefined;
         isStream?: boolean | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         responseType?: import("got-scraping", { with: { "resolution-mode": "import" } }).ResponseType | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         pagination?: import("got-scraping", { with: { "resolution-mode": "import" } }).PaginationOptions<unknown, unknown> | undefined;
         setHost?: boolean | undefined;
         maxHeaderSize?: number | undefined;
         enableUnixSockets?: boolean | undefined;
     } & {
+// @ts-ignore optional peer dependency or compatibility with es2022
         hooks?: Partial<import("got-scraping", { with: { "resolution-mode": "import" } }).Hooks> | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
         retry?: Partial<import("got-scraping", { with: { "resolution-mode": "import" } }).RetryOptions> | undefined;
+// @ts-ignore optional peer dependency or compatibility with es2022
     } & import("got-scraping", { with: { "resolution-mode": "import" } }).Context & {
         isStream: true;
     };
@@ -429,6 +498,7 @@ interface RequestFunctionOptions {
  * await crawler.run();
  * ```
  */
+// @ts-ignore optional peer dependency or compatibility with es2022
 export declare function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/basic").RouterHandler<Context>;
 export {};
 //# sourceMappingURL=http-crawler.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "3.7.1-beta.4",
+    "version": "3.7.1-beta.6",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -55,9 +55,9 @@
     "dependencies": {
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/basic": "3.7.1-beta.4",
-        "@crawlee/types": "3.7.1-beta.4",
-        "@crawlee/utils": "3.7.1-beta.4",
+        "@crawlee/basic": "3.7.1-beta.6",
+        "@crawlee/types": "3.7.1-beta.6",
+        "@crawlee/utils": "3.7.1-beta.6",
         "@types/content-type": "^1.1.5",
         "cheerio": "^1.0.0-rc.12",
         "content-type": "^1.0.4",
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "9957e0ac9336718d611936819914cc71e49c2d6c"
+    "gitHead": "01664e58799374a6acc2aa9db397566c1acc69d4"
 }