@crawlee/types 4.0.0-beta.8 → 4.0.0-beta.81

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/browser.d.ts

@@ -1,3 +1,4 @@
+import type { ISession } from './session.js';
 import type { Dictionary } from './utility-types.js';
 export interface Cookie {
     /**
@@ -60,4 +61,106 @@ export interface BrowserLikeResponse {
     url(): string;
     headers(): Dictionary<string | string[]>;
 }
-//# sourceMappingURL=browser.d.ts.map
+/**
+ * A snapshot of the relevant state of a page, as extracted by
+ * {@link IBrowserPool.extractPageState}.
+ */
+export interface PageState {
+    /**
+     * Cookies currently set in the page's browsing context.
+     */
+    cookies: Cookie[];
+}
+/**
+ * Options accepted by {@link IBrowserPool.newPage}.
+ */
+export interface NewPageOptions {
+    /**
+     * Assign a custom ID to the page. If you don't provide one, a random string
+     * ID is generated.
+     */
+    id?: string;
+    /**
+     * The crawling session that will use the returned page.
+     *
+     * The pool derives proxy configuration from the session's {@link
+     * ProxyInfo|proxy} (including TLS-error handling) — there are intentionally
+     * no standalone `proxyUrl` / `ignoreTlsErrors` options; configure them
+     * through the session instead.
+     *
+     * Session injection is **best-effort**: the pool may use the session's
+     * {@link ProxyInfo|proxy}, cookies, or fingerprint data to configure the
+     * page or the underlying browser, but none of this is guaranteed. Different
+     * pool implementations (or pool configurations such as `useIncognitoPages`)
+     * may support different subsets of session properties — or ignore them
+     * entirely.
+     *
+     * The crawler is still responsible for deterministic session setup (e.g.
+     * injecting cookies into the page before navigation) that must happen
+     * regardless of pool implementation.
+     */
+    session?: ISession;
+}
+/**
+ * Minimal contract that any object passed to a browser crawler as its `browserPool`
+ * option must satisfy.
+ *
+ * Lifecycle (`destroy`) is the responsibility of whoever owns the pool — since a
+ * user-supplied pool is never owned by the crawler, the crawler never tears it
+ * down.
+ *
+ * Implement this interface to plug a custom page-provisioning strategy into any
+ * Crawlee browser crawler — for example a remote browser farm, a session-aware
+ * pool that pins pages to fingerprints differently, or a thin wrapper around the
+ * built-in `BrowserPool`.
+ *
+ * @category Browser management
+ */
+export interface IBrowserPool<Page = unknown> {
+    /**
+     * Opens a new page. The pool decides which browser to use, launching a new
+     * one if needed.
+     */
+    newPage(options?: NewPageOptions): Promise<Page>;
+    /**
+     * Signals the pool that the caller is done with the page. The pool is
+     * responsible for closing the page and performing any necessary cleanup
+     * (e.g. retiring the underlying browser when a session has gone bad).
+     *
+     * @param page The page to release back to the pool.
+     * @param options.error If the page is being released because of an error,
+     *   pass the error here. In particular, if the error is a
+     *   {@link SessionError}, implementations should treat it as a signal
+     *   to purge all state associated with the session (e.g. discard any
+     *   browser that served the page).
+     */
+    closePage(page: Page, options?: {
+        error?: Error;
+    }): Promise<void>;
+    /**
+     * Extracts the relevant state from a page so the caller can persist it —
+     * for example, back-propagating cookies into the crawling {@link
+     * ISession|session}.
+     *
+     * @param page The page to read state from.
+     */
+    extractPageState(page: Page): Promise<PageState>;
+    /**
+     * Injects state (currently just cookies) into a page. This is the
+     * counterpart to {@link IBrowserPool.extractPageState} and lets the
+     * caller set up a page — for example, seeding it with the crawling
+     * {@link ISession|session}'s cookies before navigation.
+     *
+     * As with {@link IBrowserPool.newPage}, the caller decides *what* state
+     * to inject, while the pool decides *how*.
+     *
+     * Isolation between pages is **best-effort**: depending on the pool
+     * implementation and its configuration, multiple pages may share a browsing
+     * context, so injected state (such as cookies) can bleed across pages
+     * served by the same underlying browser.
+     *
+     * @param page The page to inject state into.
+     * @param state The state to inject.
+     */
+    injectPageState(page: Page, state: PageState): Promise<void>;
+}

package/browser.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=browser.js.map

package/http-client.d.ts

@@ -0,0 +1,77 @@
+import type { Readable } from 'node:stream';
+import type { CookieJar } from 'tough-cookie';
+import type { ISession } from './session.js';
+import type { AllowedHttpMethods } from './utility-types.js';
+export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+/**
+ * HTTP Request as accepted by {@link BaseHttpClient} methods.
+ */
+export interface HttpRequest {
+    url: string | URL;
+    method?: AllowedHttpMethods;
+    headers?: Headers;
+    body?: Readable;
+    signal?: AbortSignal;
+    timeout?: number;
+    cookieJar?: CookieJar;
+    followRedirect?: boolean | ((response: any) => boolean);
+    maxRedirects?: number;
+    encoding?: BufferEncoding;
+    throwHttpErrors?: boolean;
+    proxyUrl?: string;
+    headerGeneratorOptions?: Record<string, unknown>;
+    useHeaderGenerator?: boolean;
+    headerGenerator?: {
+        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
+    };
+    insecureHTTPParser?: boolean;
+    sessionToken?: object;
+}
+/**
+ * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
+ */
+export interface HttpRequestOptions extends HttpRequest {
+    /** Search (query string) parameters to be appended to the request URL */
+    searchParams?: SearchParams;
+    /** A form to be sent in the HTTP request body (URL encoding will be used) */
+    form?: Record<string, string>;
+    /** Arbitrary object to be JSON-serialized and sent as the HTTP request body */
+    json?: unknown;
+    /** Basic HTTP Auth username */
+    username?: string;
+    /** Basic HTTP Auth password */
+    password?: string;
+}
+/**
+ * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
+ */
+export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
+    url?: string | URL;
+    headers: Headers;
+}) => void;
+export interface SendRequestOptions {
+    session?: ISession;
+    cookieJar?: CookieJar;
+    /** Timeout for the HTTP request in milliseconds. */
+    timeoutMillis?: number;
+    /** An AbortSignal to cancel the HTTP request. */
+    signal?: AbortSignal;
+    /**
+     * Overrides the proxy URL set in the `session` for this request.
+     *
+     * Note that setting this manually can interfere with session proxy rotation.
+     */
+    proxyUrl?: string;
+}
+export interface StreamOptions extends SendRequestOptions {
+    onRedirect?: RedirectHandler;
+}
+/**
+ * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
+ */
+export interface BaseHttpClient {
+    /**
+     * Perform an HTTP Request and return the complete response.
+     */
+    sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
+}

package/http-client.js

@@ -0,0 +1 @@
+export {};

package/index.d.ts

@@ -1,4 +1,7 @@
-export * from './storages.js';
-export * from './utility-types.js';
-export * from './browser.js';
-//# sourceMappingURL=index.d.ts.map
+export type * from './status-message.js';
+export type * from './storages.js';
+export type * from './utility-types.js';
+export type * from './browser.js';
+export type * from './http-client.js';
+export type * from './session.js';
+export type * from './logger.js';

package/index.js

@@ -1,4 +1 @@
-export * from './storages.js';
-export * from './utility-types.js';
-export * from './browser.js';
-//# sourceMappingURL=index.js.map
+export {};

package/logger.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Configuration options for Crawlee logger implementations.
+ */
+export interface CrawleeLoggerOptions {
+    /** Prefix to be prepended to each logged line. */
+    prefix?: string | null;
+}
+/**
+ * Interface for Crawlee logger implementations.
+ * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
+ * compatibility with the default `@apify/log` implementation.
+ */
+export interface CrawleeLogger {
+    /**
+     * Returns the logger configuration.
+     */
+    getOptions(): CrawleeLoggerOptions;
+    /**
+     * Configures logger options.
+     */
+    setOptions(options: Partial<CrawleeLoggerOptions>): void;
+    /**
+     * Creates a new instance of logger that inherits settings from a parent logger.
+     */
+    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    /**
+     * Logs an `ERROR` message.
+     */
+    error(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs an `ERROR` level message with a nicely formatted exception.
+     */
+    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs a `SOFT_FAIL` level message.
+     */
+    softFail(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs a `WARNING` level message.
+     */
+    warning(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs a `WARNING` level message only once.
+     */
+    warningOnce(message: string): void;
+    /**
+     * Logs an `INFO` message.
+     */
+    info(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs a `DEBUG` message.
+     */
+    debug(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs a `PERF` level message for performance tracking.
+     */
+    perf(message: string, data?: Record<string, unknown>): void;
+    /**
+     * Logs given message only once as WARNING for deprecated features.
+     */
+    deprecated(message: string): void;
+    /**
+     * Logs a message at the given level. Useful when the log level is determined dynamically.
+     */
+    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+}

package/logger.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/types",
-    "version": "4.0.0-beta.8",
+    "version": "4.0.0-beta.81",
     "description": "Shared types for the crawlee projects",
     "engines": {
         "node": ">=22.0.0"
@@ -34,7 +34,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -43,6 +43,7 @@
         "access": "public"
     },
     "dependencies": {
+        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -52,5 +53,5 @@
             }
         }
     },
-    "gitHead": "a0f673e03208ad1dd8956ce3b49a2d1c9e3b2ed5"
+    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
 }

package/session.d.ts

@@ -0,0 +1,150 @@
+import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
+/**
+ * The main purpose of the ProxyInfo object is to provide information
+ * about the current proxy connection used by the crawler for the request.
+ * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
+ * });
+ *
+ * // Getting proxyInfo object by calling class method directly
+ * const proxyInfo = await proxyConfiguration.newProxyInfo();
+ *
+ * // In crawler
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   requestHandler({ proxyInfo }) {
+ *      // Getting used proxy URL
+ *       const proxyUrl = proxyInfo.url;
+ *   }
+ * })
+ *
+ * ```
+ */
+export interface ProxyInfo {
+    /**
+     * The URL of the proxy.
+     */
+    url: string;
+    /**
+     * Username for the proxy.
+     */
+    username?: string;
+    /**
+     * User's password for the proxy.
+     */
+    password: string;
+    /**
+     * Hostname of your proxy.
+     */
+    hostname: string;
+    /**
+     * Proxy port.
+     */
+    port: number | string;
+    /**
+     * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
+     *
+     * @default false
+     */
+    ignoreTlsErrors?: boolean;
+}
+/**
+ * Identifies the browser-like profile a {@link Session} is impersonating, so
+ * repeated requests with the same session look consistent to the target server.
+ *
+ * These fields are *hints* — `browser`, `platform`, `device`. Consumers
+ * (`@crawlee/browser-pool`, `@crawlee/impit-client`, …) derive their own rich
+ * state from them (e.g. a full browser fingerprint, a TLS impersonation profile)
+ * and cache it on their own; the session itself is read-only intent.
+ */
+export interface SessionFingerprint {
+    /** Browser family — consumed by HTTP clients that impersonate (e.g. `impit`). */
+    browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
+    /** Platform hint — used by header generators and as a virtual session key. */
+    platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
+    /** Device class — drives header generation and viewport defaults. */
+    device?: 'desktop' | 'mobile';
+}
+/**
+ * Persistable {@link Session} state.
+ */
+export interface SessionState {
+    id: string;
+    cookieJar: SerializedCookieJar;
+    proxyInfo?: ProxyInfo;
+    userData: object;
+    fingerprint?: SessionFingerprint;
+    errorScore: number;
+    maxErrorScore: number;
+    errorScoreDecrement: number;
+    usageCount: number;
+    maxUsageCount: number;
+    expiresAt: string;
+    createdAt: string;
+    retired: boolean;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export interface ISession {
+    readonly id: string;
+    cookieJar: CookieJar;
+    proxyInfo?: ProxyInfo;
+    fingerprint?: SessionFingerprint;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     */
+    markGood(): void;
+    /**
+     * Marks session as blocked.
+     * This method should be used if the session usage was unsuccessful
+     * and you are sure that it is because of the session configuration and not any external matters.
+     * For example when server returns 403 status code.
+     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+}
+/**
+ * Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
+ *
+ * Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
+ * `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
+ * is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
+ * the crawler, the crawler never tears it down.
+ *
+ * Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
+ * for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
+ * built-in `SessionPool` with different rotation rules.
+ *
+ * @category Scaling
+ */
+export interface ISessionPool {
+    /**
+     * Returns a usable {@link ISession}. Without an id, the pool decides which session to return
+     * (creating a new one when appropriate). With an id, the pool returns the matching session if
+     * it is still usable.
+     *
+     * In case the `SessionPool` cannot provide a usable session given the configuration,
+     * this method may return `undefined`.
+     */
+    getSession(sessionId?: string): Promise<ISession | undefined>;
+}

package/session.js

@@ -0,0 +1 @@
+export {};

package/status-message.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
+ *
+ * Setting a status message is not a storage concern — the crawler broadcasts it through the event
+ * system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
+ * status-reporting backend.
+ */
+export interface SetStatusMessageOptions {
+    /** Whether this is the final status message of the run. */
+    isStatusMessageTerminal?: boolean;
+    /** The log level to log the message with. Defaults to `'DEBUG'`. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}

package/status-message.js

@@ -0,0 +1 @@
+export {};