@crawlee/http 4.0.0-beta.105 → 4.0.0-beta.107

package/internals/http-crawler.d.ts

@@ -242,14 +242,8 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * @category Crawlers
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
-    private preNavigationHooks;
-    private postNavigationHooks;
-    private saveResponseCookies;
-    private navigationTimeoutMillis;
+    #private;
     private ignoreSslErrors;
-    private suggestResponseEncoding?;
-    private forceResponseEncoding?;
-    private readonly supportedMimeTypes;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -381,11 +375,11 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * Handles timeout request
      */
     private handleRequestTimeout;
-    private _abortDownloadOfBody;
+    private abortDownloadOfBody;
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _requestAsBrowser;
+    private requestAsBrowser;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/http-crawler.js

@@ -109,14 +109,15 @@ export class HttpCrawler extends BasicCrawler {
     // extension-aware for consumer DX, but internally the pipeline composes hooks against the
     // concrete crawling context, which does not statically carry `ContextExtension`. The members
     // added by `extendContext` are present at runtime regardless.
-    preNavigationHooks;
-    postNavigationHooks;
-    saveResponseCookies;
-    navigationTimeoutMillis;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    #navigationTimeoutMillis;
+    // kept as TS-private: tests read it at runtime
     ignoreSslErrors;
-    suggestResponseEncoding;
-    forceResponseEncoding;
-    supportedMimeTypes;
+    #suggestResponseEncoding;
+    #forceResponseEncoding;
+    #supportedMimeTypes;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number,
@@ -141,28 +142,28 @@ export class HttpCrawler extends BasicCrawler {
             contextPipelineBuilder: contextPipelineBuilder ??
                 (() => this.buildContextPipeline()),
         });
-        this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
+        this.#supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
             this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         this.ignoreSslErrors = ignoreSslErrors;
-        this.suggestResponseEncoding = suggestResponseEncoding;
-        this.forceResponseEncoding = forceResponseEncoding;
+        this.#suggestResponseEncoding = suggestResponseEncoding;
+        this.#forceResponseEncoding = forceResponseEncoding;
         // Cast away the extension-aware option types to the base internal storage types (see the field
         // declarations above). This is sound - the hooks only ever receive the base context plus the
         // members `extendContext` added at runtime.
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = [
-            ({ request, response }) => this._abortDownloadOfBody(request, response),
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = [
+            ({ request, response }) => this.abortDownloadOfBody(request, response),
             ...postNavigationHooks,
         ];
-        this.saveResponseCookies = saveResponseCookies;
+        this.#saveResponseCookies = saveResponseCookies;
     }
     getNavigationTimeoutMillis() {
-        return this.navigationTimeoutMillis;
+        return this.#navigationTimeoutMillis;
     }
     /**
      * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
@@ -185,9 +186,9 @@ export class HttpCrawler extends BasicCrawler {
         // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
         // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
         // window the navigation uses instead of each step being timed on its own.
-        const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;
+        const navigationTimedOut = `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`;
         const windowGuard = (step) => skipGuard(async (ctx) => {
-            const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+            const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
             if (remaining <= 0) {
                 throw new TimeoutError(navigationTimedOut);
             }
@@ -196,11 +197,11 @@ export class HttpCrawler extends BasicCrawler {
         let pipeline = ContextPipeline.create().compose({
             action: this.prepareHttpRequest.bind(this),
         });
-        for (const hook of this.preNavigationHooks) {
+        for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
         let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
-        for (const hook of this.postNavigationHooks) {
+        for (const hook of this.#postNavigationHooks) {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
@@ -234,7 +235,7 @@ export class HttpCrawler extends BasicCrawler {
         // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
         // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
         // client abort.
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)), `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
@@ -264,11 +265,11 @@ export class HttpCrawler extends BasicCrawler {
         // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
         // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
         // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
-        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
         if (remaining <= 0) {
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -287,14 +288,16 @@ export class HttpCrawler extends BasicCrawler {
             }
             return $;
         };
-        this._throwOnBlockedRequest(response.status);
-        if (this.saveResponseCookies) {
+        this.throwOnBlockedRequest(response.status);
+        if (this.#saveResponseCookies) {
             try {
                 for (const cookie of getCookiesFromResponse(response)) {
                     if (!cookie)
                         continue;
                     try {
-                        crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });
+                        await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+                            ignoreError: false,
+                        });
                     }
                     catch (e) {
                         this.log.debug(`Could not set cookie: ${e.message}`);
@@ -347,7 +350,7 @@ export class HttpCrawler extends BasicCrawler {
     async requestFunction({ request, session, proxyUrl }) {
         const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
-            return await this._requestAsBrowser(opts, session);
+            return await this.requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
@@ -355,7 +358,7 @@ export class HttpCrawler extends BasicCrawler {
                 return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
-                throw new SessionError(this._getMessageFromError(e));
+                throw new SessionError(this.getMessageFromError(e));
             }
             else {
                 throw e;
@@ -391,10 +394,10 @@ export class HttpCrawler extends BasicCrawler {
             throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
         }
         else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
-            if (!charset && !this.forceResponseEncoding) {
+            if (!charset && !this.#forceResponseEncoding) {
                 const rawBytes = Buffer.from(await response.arrayBuffer());
                 const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
-                const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+                const charsetToUse = metaCharset ?? this.#suggestResponseEncoding ?? 'utf-8';
                 const body = iconv.encodingExists(charsetToUse)
                     ? iconv.decode(rawBytes, charsetToUse)
                     : rawBytes.toString('utf8');
@@ -419,7 +422,7 @@ export class HttpCrawler extends BasicCrawler {
             url: request.url,
             method: request.method,
             proxyUrl,
-            timeout: this.navigationTimeoutMillis,
+            timeout: this.#navigationTimeoutMillis,
             sessionToken: session,
             headers: request.headers,
             https: {
@@ -428,7 +431,7 @@ export class HttpCrawler extends BasicCrawler {
             body: undefined,
         };
         if (requestOptions.headers?.cookie || requestOptions.headers?.Cookie) {
-            requestOptions.headers.Cookie = this._getCookieHeaderFromRequest(request);
+            requestOptions.headers.Cookie = this.getCookieHeaderFromRequest(request);
             delete requestOptions.headers.cookie;
         }
         // Disable SSL verification for MITM proxies
@@ -443,11 +446,11 @@ export class HttpCrawler extends BasicCrawler {
         return requestOptions;
     }
     encodeResponse(request, response, encoding) {
-        if (this.forceResponseEncoding) {
-            encoding = this.forceResponseEncoding;
+        if (this.#forceResponseEncoding) {
+            encoding = this.#forceResponseEncoding;
         }
-        else if (!encoding && this.suggestResponseEncoding) {
-            encoding = this.suggestResponseEncoding;
+        else if (!encoding && this.#suggestResponseEncoding) {
+            encoding = this.#suggestResponseEncoding;
         }
         // Fall back to utf-8 if we still don't have encoding.
         const utf8 = 'utf8';
@@ -481,12 +484,12 @@ export class HttpCrawler extends BasicCrawler {
     extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
-                this.supportedMimeTypes.add(mimeType);
+                this.#supportedMimeTypes.add(mimeType);
                 continue;
             }
             try {
                 const parsedType = contentTypeParser.parse(mimeType);
-                this.supportedMimeTypes.add(parsedType.type);
+                this.#supportedMimeTypes.add(parsedType.type);
             }
             catch (err) {
                 throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
@@ -498,22 +501,22 @@ export class HttpCrawler extends BasicCrawler {
      */
     handleRequestTimeout(session) {
         session.markBad();
-        throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        throw new Error(`Request timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
     }
-    _abortDownloadOfBody(request, response) {
+    abortDownloadOfBody(request, response) {
         const { status } = response;
         const { type } = parseContentTypeFromResponse(response);
         const isTransientContentType = status >= 500 || this.blockedStatusCodes.has(status);
-        if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
+        if (!this.#supportedMimeTypes.has(type) && !this.#supportedMimeTypes.has('*/*') && !isTransientContentType) {
             request.noRetry = true;
             throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
-                `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
+                `but only ${Array.from(this.#supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
         }
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    _requestAsBrowser = async (options, session) => {
+    requestAsBrowser = async (options, session) => {
         const opts = processHttpRequestOptions({
             ...options,
             responseType: 'text',
@@ -521,7 +524,7 @@ export class HttpCrawler extends BasicCrawler {
         // When saveResponseCookies is false, the response cookies must not mutate the
         // session jar. Reads still go through the session (so session.setCookie() in pre-nav
         // hooks keeps working) but a per-request clone is passed in so writes are discarded.
-        const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
         // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
         // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
         // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.105",
+    "version": "4.0.0-beta.107",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.105",
-        "@crawlee/core": "4.0.0-beta.105",
-        "@crawlee/http-client": "4.0.0-beta.105",
-        "@crawlee/types": "4.0.0-beta.105",
-        "@crawlee/utils": "4.0.0-beta.105",
+        "@crawlee/basic": "4.0.0-beta.107",
+        "@crawlee/core": "4.0.0-beta.107",
+        "@crawlee/http-client": "4.0.0-beta.107",
+        "@crawlee/types": "4.0.0-beta.107",
+        "@crawlee/utils": "4.0.0-beta.107",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
+    "gitHead": "5a3cb6242d0ae261df93c5e03cfebbb9b736e6b7"
 }