@crawlee/types 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/browser.d.ts

@@ -1,5 +1,4 @@
 import type { ISession } from './session.js';
-import type { Dictionary } from './utility-types.js';
 export interface Cookie {
     /**
      * Cookie name.
@@ -57,10 +56,6 @@ export interface Cookie {
      */
     sourcePort?: number;
 }
-export interface BrowserLikeResponse {
-    url(): string;
-    headers(): Dictionary<string | string[]>;
-}
 /**
  * A snapshot of the relevant state of a page, as extracted by
  * {@link IBrowserPool.extractPageState}.
@@ -125,14 +120,14 @@ export interface IBrowserPool<Page = unknown> {
     /**
      * Signals the pool that the caller is done with the page. The pool is
      * responsible for closing the page and performing any necessary cleanup
-     * (e.g. retiring the underlying browser when a session has gone bad).
+     * (e.g. retiring the underlying browser once the page's session is finished).
      *
      * @param page The page to release back to the pool.
-     * @param options.error If the page is being released because of an error,
-     *   pass the error here. In particular, if the error is a
-     *   {@link SessionError}, implementations should treat it as a signal
-     *   to purge all state associated with the session (e.g. discard any
-     *   browser that served the page).
+     * @param options.error If the error is a {@link SessionError}, the
+     *   session that served the page is finished. A plain `SessionError`
+     *   means it was blocked: discard any browser state tied to it. A
+     *   {@link SessionRetiredError} means it merely reached its usage or
+     *   age limit, so a pool may keep a warm page instead.
      */
     closePage(page: Page, options?: {
         error?: Error;

package/cookies.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Cookie shape used by {@link CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
+ * on `tough-cookie` itself.
+ */
+export interface SessionCookie {
+    key: string;
+    value: string;
+    expires: Date | 'Infinity' | null;
+    maxAge: number | 'Infinity' | '-Infinity' | null;
+    domain: string | null;
+    path: string | null;
+    secure: boolean;
+    httpOnly: boolean;
+    extensions: string[] | null;
+    creation: Date | 'Infinity' | null;
+    creationIndex: number;
+    hostOnly: boolean | null;
+    pathIsDefault: boolean | null;
+    lastAccessed: Date | 'Infinity' | null;
+    sameSite: string | undefined;
+    toJSON(): Record<string, unknown>;
+    clone(): SessionCookie | undefined;
+    validate(): boolean;
+    setExpires(exp: string | Date): void;
+    setMaxAge(age: number): void;
+    cookieString(): string;
+    toString(): string;
+    TTL(now?: number): number;
+    expiryTime(now?: Date): number | undefined;
+    expiryDate(now?: Date): Date | undefined;
+    isPersistent(): boolean;
+    canonicalizedDomain(): string | undefined;
+    cdomain(): string | undefined;
+}
+/**
+ * Options for {@link CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
+ * `SetCookieOptions`.
+ */
+export interface CookieJarSetCookieOptions {
+    loose?: boolean;
+    sameSiteContext?: 'strict' | 'lax' | 'none';
+    ignoreError?: boolean;
+    http?: boolean;
+    now?: Date;
+}
+/**
+ * Options for {@link CookieJar}'s cookie-reading methods. Structurally compatible with
+ * `tough-cookie`'s `GetCookiesOptions`.
+ */
+export interface CookieJarGetCookiesOptions {
+    http?: boolean;
+    expire?: boolean;
+    allPaths?: boolean;
+    sameSiteContext?: 'none' | 'lax' | 'strict';
+    sort?: boolean;
+}
+/**
+ * Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
+ * what backs {@link Session} by default — so `@crawlee/types` does not have to depend on
+ * `tough-cookie` for it.
+ */
+export interface CookieJar {
+    setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
+    getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
+    getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
+    getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
+    serialize(): Promise<SerializedCookieJar>;
+    toJSON(): SerializedCookieJar | undefined;
+    clone(): Promise<CookieJar>;
+}
+/**
+ * JSON representation of a {@link CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `SerializedCookieJar`.
+ */
+export interface SerializedCookieJar {
+    version: string;
+    storeType: string | null;
+    rejectPublicSuffixes: boolean;
+    cookies: Record<string, unknown>[];
+    [key: string]: unknown;
+}

package/cookies.js

@@ -0,0 +1 @@
+export {};

package/http-client.d.ts

@@ -1,5 +1,5 @@
 import type { Readable } from 'node:stream';
-import type { CookieJar } from 'tough-cookie';
+import type { CookieJar } from './cookies.js';
 import type { ISession } from './session.js';
 import type { AllowedHttpMethods } from './utility-types.js';
 export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
@@ -15,17 +15,8 @@ export interface HttpRequest {
     timeout?: number;
     cookieJar?: CookieJar;
     followRedirect?: boolean | ((response: any) => boolean);
-    maxRedirects?: number;
     encoding?: BufferEncoding;
-    throwHttpErrors?: boolean;
     proxyUrl?: string;
-    headerGeneratorOptions?: Record<string, unknown>;
-    useHeaderGenerator?: boolean;
-    headerGenerator?: {
-        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
-    };
-    insecureHTTPParser?: boolean;
-    sessionToken?: object;
 }
 /**
  * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
@@ -42,13 +33,6 @@ export interface HttpRequestOptions extends HttpRequest {
     /** Basic HTTP Auth password */
     password?: string;
 }
-/**
- * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
- */
-export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
-    url?: string | URL;
-    headers: Headers;
-}) => void;
 export interface SendRequestOptions {
     session?: ISession;
     cookieJar?: CookieJar;
@@ -62,9 +46,12 @@ export interface SendRequestOptions {
      * Note that setting this manually can interfere with session proxy rotation.
      */
     proxyUrl?: string;
-}
-export interface StreamOptions extends SendRequestOptions {
-    onRedirect?: RedirectHandler;
+    /**
+     * When `true`, TLS certificate errors are ignored for this request. Also enabled by
+     * `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
+     * disable TLS verification (e.g. the native fetch fallback) ignore it.
+     */
+    ignoreTlsErrors?: boolean;
 }
 /**
  * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.

package/index.d.ts

@@ -2,6 +2,7 @@ export type * from './status-message.js';
 export type * from './storages.js';
 export type * from './utility-types.js';
 export type * from './browser.js';
+export type * from './cookies.js';
 export type * from './http-client.js';
 export type * from './session.js';
 export type * from './logger.js';

package/logger.d.ts

@@ -5,6 +5,13 @@ export interface CrawleeLoggerOptions {
     /** Prefix to be prepended to each logged line. */
     prefix?: string | null;
 }
+/**
+ * Options for a single log call.
+ */
+export interface LogOptions {
+    /** Log the message only the first time this logger sees that text, ignoring later calls with it. */
+    once?: boolean;
+}
 /**
  * Interface for Crawlee logger implementations.
  * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
@@ -26,7 +33,7 @@ export interface CrawleeLogger {
     /**
      * Logs an `ERROR` message.
      */
-    error(message: string, data?: Record<string, unknown>): void;
+    error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs an `ERROR` level message with a nicely formatted exception.
      */
@@ -34,27 +41,27 @@ export interface CrawleeLogger {
     /**
      * Logs a `SOFT_FAIL` level message.
      */
-    softFail(message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `WARNING` level message.
      */
-    warning(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
-     * Logs a `WARNING` level message only once.
+     * Logs a `WARNING` level message only once. Shorthand for `warning(message, undefined, { once: true })`.
      */
     warningOnce(message: string): void;
     /**
      * Logs an `INFO` message.
      */
-    info(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `DEBUG` message.
      */
-    debug(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `PERF` level message for performance tracking.
      */
-    perf(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs given message only once as WARNING for deprecated features.
      */

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/types",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "Shared types for the crawlee projects",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -43,7 +43,6 @@
         "access": "public"
     },
     "dependencies": {
-        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -53,5 +52,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/session.d.ts

@@ -1,4 +1,4 @@
-import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
+import type { CookieJar, SerializedCookieJar } from './cookies.js';
 /**
  * The main purpose of the ProxyInfo object is to provide information
  * about the current proxy connection used by the crawler for the request.
@@ -49,6 +49,8 @@ export interface ProxyInfo {
     port: number | string;
     /**
      * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
+     * The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
+     * requests when this is set.
      *
      * @default false
      */
@@ -73,6 +75,7 @@ export interface SessionFingerprint {
 }
 /**
  * Persistable {@link Session} state.
+ * @internal
  */
 export interface SessionState {
     id: string;
@@ -105,6 +108,11 @@ export interface ISession {
      * Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
      */
     isUsable(): boolean;
+    /**
+     * Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
+     * Unlike other reasons for not being usable, this means the identity itself is burned.
+     */
+    isBlocked(): boolean;
     /**
      * This method should be called after a successful session usage.
      */

package/storages.d.ts

@@ -174,7 +174,14 @@ export interface KeyValueStoreBackend {
      * instead of inferring the end from `items.length < limit`.
      */
     listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
-    /** Get the public URL for a record, or `undefined` if unavailable. */
+    /**
+     * Get the public URL a record with this key has, or `undefined` if the backend exposes no public
+     * URLs at all.
+     *
+     * The URL is derived from the key; implementations MUST NOT check that the record exists. Callers
+     * legitimately ask for the URL of a record that is not on the backend yet — most notably one
+     * buffered in an uncommitted storage transaction — and existence is `recordExists`'s job.
+     */
     getPublicUrl(key: string): Promise<string | undefined>;
     /** Check whether a record with the given key exists. */
     recordExists(key: string): Promise<boolean>;
@@ -329,15 +336,26 @@ export interface RequestQueueBackend {
      * still being processed. Clients that do not lock may ignore it.
      */
     setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+    /**
+     * Extends the lock on a request previously handed out by {@link fetchNextRequest}, for a consumer
+     * that needs more time than the sizing hint reserved.
+     *
+     * @returns `true` when prolonged, `false` when this client does not hold the request locked (or
+     * does not lock at all). `false` is information, not an error: the request may be processed twice.
+     * Non-locking backends leave this unimplemented.
+     */
+    extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;
 }
 /**
  * Identifies a storage by its ID, name, or alias. At most one may be provided.
  *
  * - `{ id }` — open a pre-existing storage by its unique ID.
- * - `{ name }` — open or create a globally named storage (persists across runs).
+ * - `{ name }` — open or create a globally named storage (persists across runs). The name `default`
+ *   is reserved: it resolves to the default storage, and is emptied on start along with it.
  * - `{ alias }` — open or create a run-scoped unnamed storage identified by this alias.
  *   The alias is used locally (e.g. as a directory name or cache key) but the storage
- *   itself has no persistent name. Use this for non-default unnamed storages.
+ *   itself has no persistent name. Use this for non-default unnamed storages. Like the
+ *   default storage, an aliased one is emptied on start unless `purgeOnStart` is disabled.
  * - `{}` / omitted — open the default storage.
  */
 export type StorageIdentifier = {
@@ -409,6 +427,10 @@ export interface StorageBackend {
      * `StorageBackend` implementations automatically get separate cache partitions.
      */
     getStorageBackendCacheKey?(): string;
+    /**
+     * Empty the run-scoped storages — the default one and every alias-keyed one, including any left
+     * behind by a previous run. Named storages persist across runs.
+     */
     purge?(): Promise<void>;
     teardown?(): Promise<void>;
     stats?: {

package/utility-types.d.ts

@@ -1,6 +1,4 @@
 export type Dictionary<T = any> = Record<PropertyKey, T>;
-/** @ignore */
 export type Constructor<T = unknown> = new (...args: any[]) => T;
-/** @ignore */
 export type Awaitable<T> = T | PromiseLike<T>;
 export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';