@crawlee/types 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/browser.d.ts

@@ -1,5 +1,4 @@
 import type { ISession } from './session.js';
-import type { Dictionary } from './utility-types.js';
 export interface Cookie {
     /**
      * Cookie name.
@@ -57,10 +56,6 @@ export interface Cookie {
      */
     sourcePort?: number;
 }
-export interface BrowserLikeResponse {
-    url(): string;
-    headers(): Dictionary<string | string[]>;
-}
 /**
  * A snapshot of the relevant state of a page, as extracted by
  * {@link IBrowserPool.extractPageState}.
@@ -125,14 +120,14 @@ export interface IBrowserPool<Page = unknown> {
     /**
      * Signals the pool that the caller is done with the page. The pool is
      * responsible for closing the page and performing any necessary cleanup
-     * (e.g. retiring the underlying browser when a session has gone bad).
+     * (e.g. retiring the underlying browser once the page's session is finished).
      *
      * @param page The page to release back to the pool.
-     * @param options.error If the page is being released because of an error,
-     *   pass the error here. In particular, if the error is a
-     *   {@link SessionError}, implementations should treat it as a signal
-     *   to purge all state associated with the session (e.g. discard any
-     *   browser that served the page).
+     * @param options.error If the error is a {@link SessionError}, the
+     *   session that served the page is finished. A plain `SessionError`
+     *   means it was blocked: discard any browser state tied to it. A
+     *   {@link SessionRetiredError} means it merely reached its usage or
+     *   age limit, so a pool may keep a warm page instead.
      */
     closePage(page: Page, options?: {
         error?: Error;

package/http-client.d.ts

@@ -15,17 +15,8 @@ export interface HttpRequest {
     timeout?: number;
     cookieJar?: CookieJar;
     followRedirect?: boolean | ((response: any) => boolean);
-    maxRedirects?: number;
     encoding?: BufferEncoding;
-    throwHttpErrors?: boolean;
     proxyUrl?: string;
-    headerGeneratorOptions?: Record<string, unknown>;
-    useHeaderGenerator?: boolean;
-    headerGenerator?: {
-        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
-    };
-    insecureHTTPParser?: boolean;
-    sessionToken?: object;
 }
 /**
  * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
@@ -42,13 +33,6 @@ export interface HttpRequestOptions extends HttpRequest {
     /** Basic HTTP Auth password */
     password?: string;
 }
-/**
- * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
- */
-export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
-    url?: string | URL;
-    headers: Headers;
-}) => void;
 export interface SendRequestOptions {
     session?: ISession;
     cookieJar?: CookieJar;
@@ -69,9 +53,6 @@ export interface SendRequestOptions {
      */
     ignoreTlsErrors?: boolean;
 }
-export interface StreamOptions extends SendRequestOptions {
-    onRedirect?: RedirectHandler;
-}
 /**
  * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
  */

package/logger.d.ts

@@ -5,6 +5,13 @@ export interface CrawleeLoggerOptions {
     /** Prefix to be prepended to each logged line. */
     prefix?: string | null;
 }
+/**
+ * Options for a single log call.
+ */
+export interface LogOptions {
+    /** Log the message only the first time this logger sees that text, ignoring later calls with it. */
+    once?: boolean;
+}
 /**
  * Interface for Crawlee logger implementations.
  * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
@@ -26,7 +33,7 @@ export interface CrawleeLogger {
     /**
      * Logs an `ERROR` message.
      */
-    error(message: string, data?: Record<string, unknown>): void;
+    error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs an `ERROR` level message with a nicely formatted exception.
      */
@@ -34,27 +41,27 @@ export interface CrawleeLogger {
     /**
      * Logs a `SOFT_FAIL` level message.
      */
-    softFail(message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `WARNING` level message.
      */
-    warning(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
-     * Logs a `WARNING` level message only once.
+     * Logs a `WARNING` level message only once. Shorthand for `warning(message, undefined, { once: true })`.
      */
     warningOnce(message: string): void;
     /**
      * Logs an `INFO` message.
      */
-    info(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `DEBUG` message.
      */
-    debug(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs a `PERF` level message for performance tracking.
      */
-    perf(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     /**
      * Logs given message only once as WARNING for deprecated features.
      */

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/types",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "Shared types for the crawlee projects",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -52,5 +52,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/session.d.ts

@@ -75,6 +75,7 @@ export interface SessionFingerprint {
 }
 /**
  * Persistable {@link Session} state.
+ * @internal
  */
 export interface SessionState {
     id: string;
@@ -107,6 +108,11 @@ export interface ISession {
      * Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
      */
     isUsable(): boolean;
+    /**
+     * Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
+     * Unlike other reasons for not being usable, this means the identity itself is burned.
+     */
+    isBlocked(): boolean;
     /**
      * This method should be called after a successful session usage.
      */

package/storages.d.ts

@@ -174,7 +174,14 @@ export interface KeyValueStoreBackend {
      * instead of inferring the end from `items.length < limit`.
      */
     listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
-    /** Get the public URL for a record, or `undefined` if unavailable. */
+    /**
+     * Get the public URL a record with this key has, or `undefined` if the backend exposes no public
+     * URLs at all.
+     *
+     * The URL is derived from the key; implementations MUST NOT check that the record exists. Callers
+     * legitimately ask for the URL of a record that is not on the backend yet — most notably one
+     * buffered in an uncommitted storage transaction — and existence is `recordExists`'s job.
+     */
     getPublicUrl(key: string): Promise<string | undefined>;
     /** Check whether a record with the given key exists. */
     recordExists(key: string): Promise<boolean>;
@@ -329,6 +336,15 @@ export interface RequestQueueBackend {
      * still being processed. Clients that do not lock may ignore it.
      */
     setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+    /**
+     * Extends the lock on a request previously handed out by {@link fetchNextRequest}, for a consumer
+     * that needs more time than the sizing hint reserved.
+     *
+     * @returns `true` when prolonged, `false` when this client does not hold the request locked (or
+     * does not lock at all). `false` is information, not an error: the request may be processed twice.
+     * Non-locking backends leave this unimplemented.
+     */
+    extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;
 }
 /**
  * Identifies a storage by its ID, name, or alias. At most one may be provided.
@@ -413,7 +429,7 @@ export interface StorageBackend {
     getStorageBackendCacheKey?(): string;
     /**
      * Empty the run-scoped storages — the default one and every alias-keyed one, including any left
-     * behind by a previous run. Named storages persist across runs, as does the default store's `INPUT`.
+     * behind by a previous run. Named storages persist across runs.
      */
     purge?(): Promise<void>;
     teardown?(): Promise<void>;

package/utility-types.d.ts

@@ -1,6 +1,4 @@
 export type Dictionary<T = any> = Record<PropertyKey, T>;
-/** @ignore */
 export type Constructor<T = unknown> = new (...args: any[]) => T;
-/** @ignore */
 export type Awaitable<T> = T | PromiseLike<T>;
 export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';