@crawlee/http-client 4.0.0-beta.23

package/README.md

@@ -0,0 +1,153 @@
+<h1 align="center">
+    <a href="https://crawlee.dev">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+        </picture>
+    </a>
+    <br>
+    <small>A web scraping and browser automation library</small>
+</h1>
+
+<p align=center>
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
+
+<p align=center>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+</p>
+
+Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
+
+Your crawlers will appear human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data, and store it to disk or cloud while staying configurable to suit your project's needs.
+
+Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) NPM package.
+
+> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
+
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
+
+## Installation
+
+We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
+
+> Crawlee requires **Node.js 16 or higher**.
+
+### With Crawlee CLI
+
+The fastest way to try Crawlee out is to use the **Crawlee CLI** and choose the **Getting started example**. The CLI will install all the necessary dependencies and add boilerplate code for you to play with.
+
+```bash
+npx crawlee create my-crawler
+```
+
+```bash
+cd my-crawler
+npm start
+```
+
+### Manual installation
+If you prefer adding Crawlee **into your own project**, try the example below. Because it uses `PlaywrightCrawler` we also need to install [Playwright](https://playwright.dev). It's not bundled with Crawlee to reduce install size.
+
+```bash
+npm install crawlee playwright
+```
+
+```js
+import { PlaywrightCrawler, Dataset } from 'crawlee';
+
+// PlaywrightCrawler crawls the web using a headless
+// browser controlled by the Playwright library.
+const crawler = new PlaywrightCrawler({
+    // Use the requestHandler to process each of the crawled pages.
+    async requestHandler({ request, page, enqueueLinks, log }) {
+        const title = await page.title();
+        log.info(`Title of ${request.loadedUrl} is '${title}'`);
+
+        // Save results as JSON to ./storage/datasets/default
+        await Dataset.pushData({ title, url: request.loadedUrl });
+
+        // Extract links from the current page
+        // and add them to the crawling queue.
+        await enqueueLinks();
+    },
+    // Uncomment this option to see the browser window.
+    // headless: false,
+});
+
+// Add first URL to the queue and start the crawl.
+await crawler.run(['https://crawlee.dev']);
+```
+
+By default, Crawlee stores data to `./storage` in the current working directory. You can override this directory via Crawlee configuration. For details, see [Configuration guide](https://crawlee.dev/js/docs/guides/configuration), [Request storage](https://crawlee.dev/js/docs/guides/request-storage) and [Result storage](https://crawlee.dev/js/docs/guides/result-storage).
+
+### Installing pre-release versions
+
+We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
+
+```bash
+npm install crawlee@3.12.3-beta.13
+```
+
+If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
+
+```json
+{
+    "overrides": {
+       "apify": {
+           "@crawlee/core": "3.12.3-beta.13",
+           "@crawlee/types": "3.12.3-beta.13",
+           "@crawlee/utils": "3.12.3-beta.13"
+       }
+    }
+}
+```
+
+## 🛠 Features
+
+- Single interface for **HTTP and headless browser** crawling
+- Persistent **queue** for URLs to crawl (breadth & depth first)
+- Pluggable **storage** of both tabular data and files
+- Automatic **scaling** with available system resources
+- Integrated **proxy rotation** and session management
+- Lifecycles customizable with **hooks**
+- **CLI** to bootstrap your projects
+- Configurable **routing**, **error handling** and **retries**
+- **Dockerfiles** ready to deploy
+- Written in **TypeScript** with generics
+
+### 👾 HTTP crawling
+
+- Zero config **HTTP2 support**, even for proxies
+- Automatic generation of **browser-like headers**
+- Replication of browser **TLS fingerprints**
+- Integrated fast **HTML parsers**. Cheerio and JSDOM
+- Yes, you can scrape **JSON APIs** as well
+
+### 💻 Real browser crawling
+
+- JavaScript **rendering** and **screenshots**
+- **Headless** and **headful** support
+- Zero-config generation of **human-like fingerprints**
+- Automatic **browser management**
+- Use **Playwright** and **Puppeteer** with the same interface
+- **Chrome**, **Firefox**, **Webkit** and many others
+
+## Usage on the Apify platform
+
+Crawlee is open-source and runs anywhere, but since it's developed by [Apify](https://apify.com), it's easy to set up on the Apify platform and run in the cloud. Visit the [Apify SDK website](https://sdk.apify.com) to learn more about deploying Crawlee to the Apify platform.
+
+## Support
+
+If you find any bug or issue with Crawlee, please [submit an issue on GitHub](https://github.com/apify/crawlee/issues). For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/apify), in GitHub Discussions or you can join our [Discord server](https://discord.com/invite/jyEM2PRvMU).
+
+## Contributing
+
+Your code contributions are welcome, and you'll be praised to eternity! If you have any ideas for improvements, either submit an issue or create a pull request. For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/crawlee/blob/master/CONTRIBUTING.md).
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE.md](https://github.com/apify/crawlee/blob/master/LICENSE.md) file for details.

package/base-http-client.d.ts

@@ -0,0 +1,27 @@
+import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types';
+export interface CustomFetchOptions {
+    proxyUrl?: string;
+}
+/**
+ * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
+ * behaviors (redirect handling, proxy and cookie handling). Concrete clients
+ * implement only the low-level network call in `fetch`.
+ */
+export declare abstract class BaseHttpClient implements BaseHttpClientInterface {
+    /**
+     * Perform the raw network request and return a single Response without any
+     * automatic redirect following or special error handling.
+     */
+    protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
+    private applyCookies;
+    private setCookies;
+    private resolveRequestContext;
+    private createAbortSignal;
+    private isRedirect;
+    private buildRedirectRequest;
+    /**
+     * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
+     */
+    sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise<Response>;
+}
+//# sourceMappingURL=base-http-client.d.ts.map

package/base-http-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-http-client.d.ts","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,IAAI,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAGpG,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,8BAAsB,cAAe,YAAW,uBAAuB;IACnE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEtF,YAAY;YASZ,UAAU;IAMxB,OAAO,CAAC,qBAAqB;IAW7B,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,oBAAoB;IA+B5B;;OAEG;IACG,WAAW,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;CA8B9F"}

package/base-http-client.js

@@ -0,0 +1,85 @@
+import { CookieJar } from 'tough-cookie';
+/**
+ * Base HTTP client that provides fetch-like `sendRequest` with Crawlee-managed
+ * behaviors (redirect handling, proxy and cookie handling). Concrete clients
+ * implement only the low-level network call in `fetch`.
+ */
+export class BaseHttpClient {
+    async applyCookies(request, cookieJar) {
+        const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean);
+        if (cookies?.length > 0) {
+            request.headers.set('cookie', cookies.join('; '));
+        }
+        return request;
+    }
+    async setCookies(response, cookieJar) {
+        const setCookieHeaders = response.headers.getSetCookie();
+        await Promise.all(setCookieHeaders.map((header) => cookieJar.setCookie(header, response.url)));
+    }
+    resolveRequestContext(options) {
+        const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
+        const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar();
+        const timeout = options?.timeout;
+        return { proxyUrl, cookieJar: cookieJar, timeout };
+    }
+    createAbortSignal(timeout) {
+        return timeout ? AbortSignal.timeout(timeout) : undefined;
+    }
+    isRedirect(response) {
+        const status = response.status;
+        return status >= 300 && status < 400 && !!response.headers.get('location');
+    }
+    buildRedirectRequest(currentRequest, response, initialRequest) {
+        const location = response.headers.get('location');
+        const nextUrl = new URL(location, response.url ?? currentRequest.url);
+        const prevMethod = (currentRequest.method ?? 'GET').toUpperCase();
+        let nextMethod = prevMethod;
+        let nextBody = null;
+        if (response.status === 303 ||
+            ((response.status === 301 || response.status === 302) && prevMethod === 'POST')) {
+            nextMethod = 'GET';
+            nextBody = null;
+        }
+        else {
+            const clonedRequest = initialRequest.clone();
+            nextBody = clonedRequest.body;
+        }
+        const nextHeaders = new Headers();
+        currentRequest.headers.forEach((value, key) => nextHeaders.set(key, value));
+        return new Request(nextUrl.toString(), {
+            method: nextMethod,
+            headers: nextHeaders,
+            body: nextBody,
+            credentials: currentRequest.credentials,
+            redirect: 'manual',
+        });
+    }
+    /**
+     * Public fetch-like method that handles redirects and uses provided proxy and cookie jar.
+     */
+    async sendRequest(initialRequest, options) {
+        const maxRedirects = 10;
+        let currentRequest = initialRequest;
+        let redirectCount = 0;
+        const { proxyUrl, cookieJar, timeout } = this.resolveRequestContext(options);
+        currentRequest = initialRequest.clone();
+        while (true) {
+            await this.applyCookies(currentRequest, cookieJar);
+            const response = await this.fetch(currentRequest, {
+                signal: this.createAbortSignal(timeout),
+                proxyUrl,
+                redirect: 'manual',
+            });
+            await this.setCookies(response, cookieJar);
+            if (this.isRedirect(response)) {
+                if (redirectCount++ >= maxRedirects) {
+                    throw new Error(`Too many redirects (${maxRedirects}) while requesting ${currentRequest.url}`);
+                }
+                currentRequest = this.buildRedirectRequest(currentRequest, response, initialRequest);
+                continue;
+            }
+            return response;
+        }
+    }
+}
+//# sourceMappingURL=base-http-client.js.map

package/base-http-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base-http-client.js","sourceRoot":"","sources":["../src/base-http-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAMzC;;;;GAIG;AACH,MAAM,OAAgB,cAAc;IAOxB,KAAK,CAAC,YAAY,CAAC,OAAgB,EAAE,SAAoB;QAC7D,MAAM,OAAO,GAAG,CAAC,MAAM,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE9G,IAAI,OAAO,EAAE,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,QAAkB,EAAE,SAAoB;QAC7D,MAAM,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAEzD,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACnG,CAAC;IAEO,qBAAqB,CAAC,OAA4B;QAKtD,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC;QACvE,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,SAAS,EAAE,CAAC;QACvF,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QACjC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAsB,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC;IAEO,iBAAiB,CAAC,OAAgB;QACtC,OAAO,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,CAAC;IAEO,UAAU,CAAC,QAAkB;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IAEO,oBAAoB,CAAC,cAAuB,EAAE,QAAkB,EAAE,cAAuB;QAC7F,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;QAEtE,MAAM,UAAU,GAAG,CAAC,cAAc,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,IAAI,UAAU,GAAG,UAAU,CAAC;QAC5B,IAAI,QAAQ,GAAoB,IAAI,CAAC;QAErC,IACI,QAAQ,CAAC,MAAM,KAAK,GAAG;YACvB,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,EACjF,CAAC;YACC,UAAU,GAAG,KAAK,CAAC;YACnB,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,CAAC;YACJ,MAAM,aAAa,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;YAC7C,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC;QAClC,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;QAClC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;QAE5E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YACnC,MAAM,EAAE,UAAU;YAClB,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAG,cAAsB,CAAC,WAAW;YAChD,QAAQ,EAAE,QAAQ;SACrB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,cAAuB,EAAE,OAA4B;QACnE,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,cAAc,GAAG,cAAc,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC7E,cAAc,GAAG,cAAc,CAAC,KAAK,EAAE,CAAC;QAExC,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE;gBAC9C,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC;gBACvC,QAAQ;gBACR,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE3C,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,IAAI,aAAa,EAAE,IAAI,YAAY,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,sBAAsB,cAAc,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnG,CAAC;gBACD,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;gBACrF,SAAS;YACb,CAAC;YAED,OAAO,QAAQ,CAAC;QACpB,CAAC;IACL,CAAC;CACJ"}

package/fetch-http-client.d.ts

@@ -0,0 +1,10 @@
+import { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
+/**
+ * A HTTP client implementation using the native `fetch` API.
+ *
+ * This implementation does not support proxying.
+ */
+export declare class FetchHttpClient extends BaseHttpClient {
+    fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+}
+//# sourceMappingURL=fetch-http-client.d.ts.map

package/fetch-http-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-http-client.d.ts","sourceRoot":"","sources":["../src/fetch-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAEhF;;;;GAIG;AACH,qBAAa,eAAgB,SAAQ,cAAc;IAChC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;CAGxG"}

package/fetch-http-client.js

@@ -0,0 +1,12 @@
+import { BaseHttpClient } from './base-http-client.js';
+/**
+ * A HTTP client implementation using the native `fetch` API.
+ *
+ * This implementation does not support proxying.
+ */
+export class FetchHttpClient extends BaseHttpClient {
+    async fetch(request, options) {
+        return fetch(request, options);
+    }
+}
+//# sourceMappingURL=fetch-http-client.js.map

package/fetch-http-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-http-client.js","sourceRoot":"","sources":["../src/fetch-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAA2B,MAAM,uBAAuB,CAAC;AAEhF;;;;GAIG;AACH,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACtC,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,OAA0C;QAC7E,OAAO,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;CACJ"}

package/index.d.ts

@@ -0,0 +1,4 @@
+export { BaseHttpClient, type CustomFetchOptions } from './base-http-client.js';
+export { ResponseWithUrl, type IResponseWithUrl } from './response.js';
+export { FetchHttpClient } from './fetch-http-client.js';
+//# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/index.js

@@ -0,0 +1,4 @@
+export { BaseHttpClient } from './base-http-client.js';
+export { ResponseWithUrl } from './response.js';
+export { FetchHttpClient } from './fetch-http-client.js';
+//# sourceMappingURL=index.js.map

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAA2B,MAAM,uBAAuB,CAAC;AAChF,OAAO,EAAE,eAAe,EAAyB,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/package.json

@@ -0,0 +1,61 @@
+{
+    "name": "@crawlee/http-client",
+    "version": "4.0.0-beta.23",
+    "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
+    "engines": {
+        "node": ">=22.0.0"
+    },
+    "type": "module",
+    "exports": {
+        ".": "./index.js",
+        "./package.json": "./package.json"
+    },
+    "keywords": [
+        "apify",
+        "headless",
+        "chrome",
+        "puppeteer",
+        "crawler",
+        "scraper"
+    ],
+    "author": {
+        "name": "Apify",
+        "email": "support@apify.com",
+        "url": "https://apify.com"
+    },
+    "contributors": [
+        "Jan Curn <jan@apify.com>",
+        "Marek Trunkat <marek@apify.com>",
+        "Ondra Urban <ondra@apify.com>"
+    ],
+    "license": "Apache-2.0",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/apify/crawlee"
+    },
+    "bugs": {
+        "url": "https://github.com/apify/crawlee/issues"
+    },
+    "homepage": "https://crawlee.dev",
+    "scripts": {
+        "build": "yarn clean && yarn compile && yarn copy",
+        "clean": "rimraf ./dist",
+        "compile": "tsc -p tsconfig.build.json",
+        "copy": "tsx ../../scripts/copy.ts"
+    },
+    "publishConfig": {
+        "access": "public"
+    },
+    "dependencies": {
+        "@crawlee/types": "4.0.0-beta.23",
+        "tough-cookie": "^6.0.0"
+    },
+    "lerna": {
+        "command": {
+            "publish": {
+                "assets": []
+            }
+        }
+    },
+    "gitHead": "1d348c0c13605e8c4749727419f2c01d1162d642"
+}

package/response.d.ts

@@ -0,0 +1,15 @@
+export interface IResponseWithUrl extends Response {
+    url: string;
+}
+/**
+ * A Response class that includes the original request URL.
+ *
+ * This class extends `Response` from `fetch` API and is fully compatible with this.
+ */
+export declare class ResponseWithUrl extends Response implements IResponseWithUrl {
+    url: string;
+    constructor(body: BodyInit | null, init: ResponseInit & {
+        url?: string;
+    });
+}
+//# sourceMappingURL=response.d.ts.map

package/response.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../src/response.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAiB,SAAQ,QAAQ;IAC9C,GAAG,EAAE,MAAM,CAAC;CACf;AAKD;;;;GAIG;AACH,qBAAa,eAAgB,SAAQ,QAAS,YAAW,gBAAgB;IAC5D,GAAG,EAAE,MAAM,CAAC;gBACT,IAAI,EAAE,QAAQ,GAAG,IAAI,EAAE,IAAI,EAAE,YAAY,GAAG;QAAE,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE;CAM3E"}

package/response.js

@@ -0,0 +1,16 @@
+// See https://github.com/nodejs/undici/blob/d7707ee8fd5da2d0cc64b5fae421b965faf803c8/lib/web/fetch/constants.js#L6
+const nullBodyStatus = [101, 204, 205, 304];
+/**
+ * A Response class that includes the original request URL.
+ *
+ * This class extends `Response` from `fetch` API and is fully compatible with this.
+ */
+export class ResponseWithUrl extends Response {
+    url;
+    constructor(body, init) {
+        const bodyParsed = nullBodyStatus.includes(init.status ?? 200) ? null : body;
+        super(bodyParsed, init);
+        this.url = init.url ?? '';
+    }
+}
+//# sourceMappingURL=response.js.map

package/response.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"response.js","sourceRoot":"","sources":["../src/response.ts"],"names":[],"mappings":"AAIA,mHAAmH;AACnH,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAE5C;;;;GAIG;AACH,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAChC,GAAG,CAAS;IACrB,YAAY,IAAqB,EAAE,IAAqC;QACpE,MAAM,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAE7E,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;IAC9B,CAAC;CACJ"}