@crawlee/types 4.0.0-beta.99 → 4.0.0-rc.0

package/cookies.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Cookie shape used by {@link CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
+ * on `tough-cookie` itself.
+ */
+export interface SessionCookie {
+    key: string;
+    value: string;
+    expires: Date | 'Infinity' | null;
+    maxAge: number | 'Infinity' | '-Infinity' | null;
+    domain: string | null;
+    path: string | null;
+    secure: boolean;
+    httpOnly: boolean;
+    extensions: string[] | null;
+    creation: Date | 'Infinity' | null;
+    creationIndex: number;
+    hostOnly: boolean | null;
+    pathIsDefault: boolean | null;
+    lastAccessed: Date | 'Infinity' | null;
+    sameSite: string | undefined;
+    toJSON(): Record<string, unknown>;
+    clone(): SessionCookie | undefined;
+    validate(): boolean;
+    setExpires(exp: string | Date): void;
+    setMaxAge(age: number): void;
+    cookieString(): string;
+    toString(): string;
+    TTL(now?: number): number;
+    expiryTime(now?: Date): number | undefined;
+    expiryDate(now?: Date): Date | undefined;
+    isPersistent(): boolean;
+    canonicalizedDomain(): string | undefined;
+    cdomain(): string | undefined;
+}
+/**
+ * Options for {@link CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
+ * `SetCookieOptions`.
+ */
+export interface CookieJarSetCookieOptions {
+    loose?: boolean;
+    sameSiteContext?: 'strict' | 'lax' | 'none';
+    ignoreError?: boolean;
+    http?: boolean;
+    now?: Date;
+}
+/**
+ * Options for {@link CookieJar}'s cookie-reading methods. Structurally compatible with
+ * `tough-cookie`'s `GetCookiesOptions`.
+ */
+export interface CookieJarGetCookiesOptions {
+    http?: boolean;
+    expire?: boolean;
+    allPaths?: boolean;
+    sameSiteContext?: 'none' | 'lax' | 'strict';
+    sort?: boolean;
+}
+/**
+ * Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
+ * what backs {@link Session} by default — so `@crawlee/types` does not have to depend on
+ * `tough-cookie` for it.
+ */
+export interface CookieJar {
+    setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
+    getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
+    getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
+    getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
+    serialize(): Promise<SerializedCookieJar>;
+    toJSON(): SerializedCookieJar | undefined;
+    clone(): Promise<CookieJar>;
+}
+/**
+ * JSON representation of a {@link CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `SerializedCookieJar`.
+ */
+export interface SerializedCookieJar {
+    version: string;
+    storeType: string | null;
+    rejectPublicSuffixes: boolean;
+    cookies: Record<string, unknown>[];
+    [key: string]: unknown;
+}

package/cookies.js

@@ -0,0 +1 @@
+export {};

package/http-client.d.ts

@@ -1,5 +1,5 @@
 import type { Readable } from 'node:stream';
-import type { CookieJar } from 'tough-cookie';
+import type { CookieJar } from './cookies.js';
 import type { ISession } from './session.js';
 import type { AllowedHttpMethods } from './utility-types.js';
 export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
@@ -62,6 +62,12 @@ export interface SendRequestOptions {
      * Note that setting this manually can interfere with session proxy rotation.
      */
     proxyUrl?: string;
+    /**
+     * When `true`, TLS certificate errors are ignored for this request. Also enabled by
+     * `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
+     * disable TLS verification (e.g. the native fetch fallback) ignore it.
+     */
+    ignoreTlsErrors?: boolean;
 }
 export interface StreamOptions extends SendRequestOptions {
     onRedirect?: RedirectHandler;

package/index.d.ts

@@ -2,6 +2,7 @@ export type * from './status-message.js';
 export type * from './storages.js';
 export type * from './utility-types.js';
 export type * from './browser.js';
+export type * from './cookies.js';
 export type * from './http-client.js';
 export type * from './session.js';
 export type * from './logger.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/types",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.0",
     "description": "Shared types for the crawlee projects",
     "engines": {
         "node": ">=22.0.0"
@@ -43,7 +43,6 @@
         "access": "public"
     },
     "dependencies": {
-        "tough-cookie": "^6.0.0",
         "tslib": "^2.8.1"
     },
     "lerna": {
@@ -53,5 +52,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }

package/session.d.ts

@@ -1,4 +1,4 @@
-import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
+import type { CookieJar, SerializedCookieJar } from './cookies.js';
 /**
  * The main purpose of the ProxyInfo object is to provide information
  * about the current proxy connection used by the crawler for the request.
@@ -49,6 +49,8 @@ export interface ProxyInfo {
     port: number | string;
     /**
      * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
+     * The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
+     * requests when this is set.
      *
      * @default false
      */

package/storages.d.ts

@@ -334,10 +334,12 @@ export interface RequestQueueBackend {
  * Identifies a storage by its ID, name, or alias. At most one may be provided.
  *
  * - `{ id }` — open a pre-existing storage by its unique ID.
- * - `{ name }` — open or create a globally named storage (persists across runs).
+ * - `{ name }` — open or create a globally named storage (persists across runs). The name `default`
+ *   is reserved: it resolves to the default storage, and is emptied on start along with it.
  * - `{ alias }` — open or create a run-scoped unnamed storage identified by this alias.
  *   The alias is used locally (e.g. as a directory name or cache key) but the storage
- *   itself has no persistent name. Use this for non-default unnamed storages.
+ *   itself has no persistent name. Use this for non-default unnamed storages. Like the
+ *   default storage, an aliased one is emptied on start unless `purgeOnStart` is disabled.
  * - `{}` / omitted — open the default storage.
  */
 export type StorageIdentifier = {
@@ -409,6 +411,10 @@ export interface StorageBackend {
      * `StorageBackend` implementations automatically get separate cache partitions.
      */
     getStorageBackendCacheKey?(): string;
+    /**
+     * Empty the run-scoped storages — the default one and every alias-keyed one, including any left
+     * behind by a previous run. Named storages persist across runs, as does the default store's `INPUT`.
+     */
     purge?(): Promise<void>;
     teardown?(): Promise<void>;
     stats?: {