@crawlee/types 4.0.0-beta.21 → 4.0.0-beta.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
package/browser.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { Dictionary } from './utility-types.js';
1
+ import type { ISession } from './session.js';
2
2
  export interface Cookie {
3
3
  /**
4
4
  * Cookie name.
@@ -56,8 +56,106 @@ export interface Cookie {
56
56
  */
57
57
  sourcePort?: number;
58
58
  }
59
- export interface BrowserLikeResponse {
60
- url(): string;
61
- headers(): Dictionary<string | string[]>;
59
+ /**
60
+ * A snapshot of the relevant state of a page, as extracted by
61
+ * {@link IBrowserPool.extractPageState}.
62
+ */
63
+ export interface PageState {
64
+ /**
65
+ * Cookies currently set in the page's browsing context.
66
+ */
67
+ cookies: Cookie[];
68
+ }
69
+ /**
70
+ * Options accepted by {@link IBrowserPool.newPage}.
71
+ */
72
+ export interface NewPageOptions {
73
+ /**
74
+ * Assign a custom ID to the page. If you don't provide one, a random string
75
+ * ID is generated.
76
+ */
77
+ id?: string;
78
+ /**
79
+ * The crawling session that will use the returned page.
80
+ *
81
+ * The pool derives proxy configuration from the session's {@link
82
+ * ProxyInfo|proxy} (including TLS-error handling) — there are intentionally
83
+ * no standalone `proxyUrl` / `ignoreTlsErrors` options; configure them
84
+ * through the session instead.
85
+ *
86
+ * Session injection is **best-effort**: the pool may use the session's
87
+ * {@link ProxyInfo|proxy}, cookies, or fingerprint data to configure the
88
+ * page or the underlying browser, but none of this is guaranteed. Different
89
+ * pool implementations (or pool configurations such as `useIncognitoPages`)
90
+ * may support different subsets of session properties — or ignore them
91
+ * entirely.
92
+ *
93
+ * The crawler is still responsible for deterministic session setup (e.g.
94
+ * injecting cookies into the page before navigation) that must happen
95
+ * regardless of pool implementation.
96
+ */
97
+ session?: ISession;
98
+ }
99
+ /**
100
+ * Minimal contract that any object passed to a browser crawler as its `browserPool`
101
+ * option must satisfy.
102
+ *
103
+ * Lifecycle (`destroy`) is the responsibility of whoever owns the pool — since a
104
+ * user-supplied pool is never owned by the crawler, the crawler never tears it
105
+ * down.
106
+ *
107
+ * Implement this interface to plug a custom page-provisioning strategy into any
108
+ * Crawlee browser crawler — for example a remote browser farm, a session-aware
109
+ * pool that pins pages to fingerprints differently, or a thin wrapper around the
110
+ * built-in `BrowserPool`.
111
+ *
112
+ * @category Browser management
113
+ */
114
+ export interface IBrowserPool<Page = unknown> {
115
+ /**
116
+ * Opens a new page. The pool decides which browser to use, launching a new
117
+ * one if needed.
118
+ */
119
+ newPage(options?: NewPageOptions): Promise<Page>;
120
+ /**
121
+ * Signals the pool that the caller is done with the page. The pool is
122
+ * responsible for closing the page and performing any necessary cleanup
123
+ * (e.g. retiring the underlying browser once the page's session is finished).
124
+ *
125
+ * @param page The page to release back to the pool.
126
+ * @param options.error If the error is a {@link SessionError}, the
127
+ * session that served the page is finished. A plain `SessionError`
128
+ * means it was blocked: discard any browser state tied to it. A
129
+ * {@link SessionRetiredError} means it merely reached its usage or
130
+ * age limit, so a pool may keep a warm page instead.
131
+ */
132
+ closePage(page: Page, options?: {
133
+ error?: Error;
134
+ }): Promise<void>;
135
+ /**
136
+ * Extracts the relevant state from a page so the caller can persist it —
137
+ * for example, back-propagating cookies into the crawling {@link
138
+ * ISession|session}.
139
+ *
140
+ * @param page The page to read state from.
141
+ */
142
+ extractPageState(page: Page): Promise<PageState>;
143
+ /**
144
+ * Injects state (currently just cookies) into a page. This is the
145
+ * counterpart to {@link IBrowserPool.extractPageState} and lets the
146
+ * caller set up a page — for example, seeding it with the crawling
147
+ * {@link ISession|session}'s cookies before navigation.
148
+ *
149
+ * As with {@link IBrowserPool.newPage}, the caller decides *what* state
150
+ * to inject, while the pool decides *how*.
151
+ *
152
+ * Isolation between pages is **best-effort**: depending on the pool
153
+ * implementation and its configuration, multiple pages may share a browsing
154
+ * context, so injected state (such as cookies) can bleed across pages
155
+ * served by the same underlying browser.
156
+ *
157
+ * @param page The page to inject state into.
158
+ * @param state The state to inject.
159
+ */
160
+ injectPageState(page: Page, state: PageState): Promise<void>;
62
161
  }
63
- //# sourceMappingURL=browser.d.ts.map
package/browser.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=browser.js.map
package/cookies.d.ts ADDED
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Cookie shape used by {@link CookieJar}. Structurally compatible with `tough-cookie`'s
3
+ * `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
4
+ * on `tough-cookie` itself.
5
+ */
6
+ export interface SessionCookie {
7
+ key: string;
8
+ value: string;
9
+ expires: Date | 'Infinity' | null;
10
+ maxAge: number | 'Infinity' | '-Infinity' | null;
11
+ domain: string | null;
12
+ path: string | null;
13
+ secure: boolean;
14
+ httpOnly: boolean;
15
+ extensions: string[] | null;
16
+ creation: Date | 'Infinity' | null;
17
+ creationIndex: number;
18
+ hostOnly: boolean | null;
19
+ pathIsDefault: boolean | null;
20
+ lastAccessed: Date | 'Infinity' | null;
21
+ sameSite: string | undefined;
22
+ toJSON(): Record<string, unknown>;
23
+ clone(): SessionCookie | undefined;
24
+ validate(): boolean;
25
+ setExpires(exp: string | Date): void;
26
+ setMaxAge(age: number): void;
27
+ cookieString(): string;
28
+ toString(): string;
29
+ TTL(now?: number): number;
30
+ expiryTime(now?: Date): number | undefined;
31
+ expiryDate(now?: Date): Date | undefined;
32
+ isPersistent(): boolean;
33
+ canonicalizedDomain(): string | undefined;
34
+ cdomain(): string | undefined;
35
+ }
36
+ /**
37
+ * Options for {@link CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
38
+ * `SetCookieOptions`.
39
+ */
40
+ export interface CookieJarSetCookieOptions {
41
+ loose?: boolean;
42
+ sameSiteContext?: 'strict' | 'lax' | 'none';
43
+ ignoreError?: boolean;
44
+ http?: boolean;
45
+ now?: Date;
46
+ }
47
+ /**
48
+ * Options for {@link CookieJar}'s cookie-reading methods. Structurally compatible with
49
+ * `tough-cookie`'s `GetCookiesOptions`.
50
+ */
51
+ export interface CookieJarGetCookiesOptions {
52
+ http?: boolean;
53
+ expire?: boolean;
54
+ allPaths?: boolean;
55
+ sameSiteContext?: 'none' | 'lax' | 'strict';
56
+ sort?: boolean;
57
+ }
58
+ /**
59
+ * Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
60
+ * what backs {@link Session} by default — so `@crawlee/types` does not have to depend on
61
+ * `tough-cookie` for it.
62
+ */
63
+ export interface CookieJar {
64
+ setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
65
+ getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
66
+ getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
67
+ getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
68
+ serialize(): Promise<SerializedCookieJar>;
69
+ toJSON(): SerializedCookieJar | undefined;
70
+ clone(): Promise<CookieJar>;
71
+ }
72
+ /**
73
+ * JSON representation of a {@link CookieJar}. Structurally compatible with `tough-cookie`'s
74
+ * `SerializedCookieJar`.
75
+ */
76
+ export interface SerializedCookieJar {
77
+ version: string;
78
+ storeType: string | null;
79
+ rejectPublicSuffixes: boolean;
80
+ cookies: Record<string, unknown>[];
81
+ [key: string]: unknown;
82
+ }
package/cookies.js ADDED
@@ -0,0 +1 @@
1
+ export {};
package/http-client.d.ts CHANGED
@@ -1,15 +1,8 @@
1
1
  import type { Readable } from 'node:stream';
2
+ import type { CookieJar } from './cookies.js';
2
3
  import type { ISession } from './session.js';
3
4
  import type { AllowedHttpMethods } from './utility-types.js';
4
5
  export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
5
- interface ToughCookieJar {
6
- getCookieString: ((currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookies: string) => void) => string) & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => string);
7
- setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record<string, unknown>, callback: (error: Error | null, cookie: unknown) => void) => void) & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void);
8
- }
9
- interface PromiseCookieJar {
10
- getCookieString: (url: string) => Promise<string>;
11
- setCookie: (rawCookie: string, url: string) => Promise<unknown>;
12
- }
13
6
  /**
14
7
  * HTTP Request as accepted by {@link BaseHttpClient} methods.
15
8
  */
@@ -20,19 +13,10 @@ export interface HttpRequest {
20
13
  body?: Readable;
21
14
  signal?: AbortSignal;
22
15
  timeout?: number;
23
- cookieJar?: ToughCookieJar | PromiseCookieJar;
16
+ cookieJar?: CookieJar;
24
17
  followRedirect?: boolean | ((response: any) => boolean);
25
- maxRedirects?: number;
26
18
  encoding?: BufferEncoding;
27
- throwHttpErrors?: boolean;
28
19
  proxyUrl?: string;
29
- headerGeneratorOptions?: Record<string, unknown>;
30
- useHeaderGenerator?: boolean;
31
- headerGenerator?: {
32
- getHeaders: (options: Record<string, unknown>) => Record<string, string>;
33
- };
34
- insecureHTTPParser?: boolean;
35
- sessionToken?: object;
36
20
  }
37
21
  /**
38
22
  * Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
@@ -49,29 +33,25 @@ export interface HttpRequestOptions extends HttpRequest {
49
33
  /** Basic HTTP Auth password */
50
34
  password?: string;
51
35
  }
52
- export interface IResponseWithUrl extends Response {
53
- url: string;
54
- }
55
- /**
56
- * Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
57
- */
58
- export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
59
- url?: string | URL;
60
- headers: Headers;
61
- }) => void;
62
36
  export interface SendRequestOptions {
63
37
  session?: ISession;
64
- cookieJar?: ToughCookieJar;
65
- timeout?: number;
38
+ cookieJar?: CookieJar;
39
+ /** Timeout for the HTTP request in milliseconds. */
40
+ timeoutMillis?: number;
41
+ /** An AbortSignal to cancel the HTTP request. */
42
+ signal?: AbortSignal;
66
43
  /**
67
44
  * Overrides the proxy URL set in the `session` for this request.
68
45
  *
69
46
  * Note that setting this manually can interfere with session proxy rotation.
70
47
  */
71
48
  proxyUrl?: string;
72
- }
73
- export interface StreamOptions extends SendRequestOptions {
74
- onRedirect?: RedirectHandler;
49
+ /**
50
+ * When `true`, TLS certificate errors are ignored for this request. Also enabled by
51
+ * `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
52
+ * disable TLS verification (e.g. the native fetch fallback) ignore it.
53
+ */
54
+ ignoreTlsErrors?: boolean;
75
55
  }
76
56
  /**
77
57
  * Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
@@ -81,10 +61,4 @@ export interface BaseHttpClient {
81
61
  * Perform an HTTP Request and return the complete response.
82
62
  */
83
63
  sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
84
- /**
85
- * Perform an HTTP Request and return after the response headers are received. The body may be read from a stream contained in the response.
86
- */
87
- stream(request: Request, options?: StreamOptions): Promise<Response>;
88
64
  }
89
- export {};
90
- //# sourceMappingURL=http-client.d.ts.map
package/http-client.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=http-client.js.map
package/index.d.ts CHANGED
@@ -1,6 +1,8 @@
1
- export * from './storages.js';
2
- export * from './utility-types.js';
3
- export * from './browser.js';
4
- export * from './http-client.js';
5
- export * from './session.js';
6
- //# sourceMappingURL=index.d.ts.map
1
+ export type * from './status-message.js';
2
+ export type * from './storages.js';
3
+ export type * from './utility-types.js';
4
+ export type * from './browser.js';
5
+ export type * from './cookies.js';
6
+ export type * from './http-client.js';
7
+ export type * from './session.js';
8
+ export type * from './logger.js';
package/index.js CHANGED
@@ -1,6 +1 @@
1
- export * from './storages.js';
2
- export * from './utility-types.js';
3
- export * from './browser.js';
4
- export * from './http-client.js';
5
- export * from './session.js';
6
- //# sourceMappingURL=index.js.map
1
+ export {};
package/logger.d.ts ADDED
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Configuration options for Crawlee logger implementations.
3
+ */
4
+ export interface CrawleeLoggerOptions {
5
+ /** Prefix to be prepended to each logged line. */
6
+ prefix?: string | null;
7
+ }
8
+ /**
9
+ * Options for a single log call.
10
+ */
11
+ export interface LogOptions {
12
+ /** Log the message only the first time this logger sees that text, ignoring later calls with it. */
13
+ once?: boolean;
14
+ }
15
+ /**
16
+ * Interface for Crawlee logger implementations.
17
+ * This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
18
+ * compatibility with the default `@apify/log` implementation.
19
+ */
20
+ export interface CrawleeLogger {
21
+ /**
22
+ * Returns the logger configuration.
23
+ */
24
+ getOptions(): CrawleeLoggerOptions;
25
+ /**
26
+ * Configures logger options.
27
+ */
28
+ setOptions(options: Partial<CrawleeLoggerOptions>): void;
29
+ /**
30
+ * Creates a new instance of logger that inherits settings from a parent logger.
31
+ */
32
+ child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
33
+ /**
34
+ * Logs an `ERROR` message.
35
+ */
36
+ error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
37
+ /**
38
+ * Logs an `ERROR` level message with a nicely formatted exception.
39
+ */
40
+ exception(exception: Error, message: string, data?: Record<string, unknown>): void;
41
+ /**
42
+ * Logs a `SOFT_FAIL` level message.
43
+ */
44
+ softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
45
+ /**
46
+ * Logs a `WARNING` level message.
47
+ */
48
+ warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
49
+ /**
50
+ * Logs a `WARNING` level message only once. Shorthand for `warning(message, undefined, { once: true })`.
51
+ */
52
+ warningOnce(message: string): void;
53
+ /**
54
+ * Logs an `INFO` message.
55
+ */
56
+ info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
57
+ /**
58
+ * Logs a `DEBUG` message.
59
+ */
60
+ debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
61
+ /**
62
+ * Logs a `PERF` level message for performance tracking.
63
+ */
64
+ perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
65
+ /**
66
+ * Logs given message only once as WARNING for deprecated features.
67
+ */
68
+ deprecated(message: string): void;
69
+ /**
70
+ * Logs a message at the given level. Useful when the log level is determined dynamically.
71
+ */
72
+ logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
73
+ }
package/logger.js ADDED
@@ -0,0 +1 @@
1
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/types",
3
- "version": "4.0.0-beta.21",
3
+ "version": "4.0.0-beta.211",
4
4
  "description": "Shared types for the crawlee projects",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -34,7 +34,7 @@
34
34
  },
35
35
  "homepage": "https://crawlee.dev",
36
36
  "scripts": {
37
- "build": "yarn clean && yarn compile && yarn copy",
37
+ "build": "pnpm clean && pnpm compile && pnpm copy",
38
38
  "clean": "rimraf ./dist",
39
39
  "compile": "tsc -p tsconfig.build.json",
40
40
  "copy": "tsx ../../scripts/copy.ts"
@@ -43,7 +43,6 @@
43
43
  "access": "public"
44
44
  },
45
45
  "dependencies": {
46
- "tough-cookie": "^6.0.0",
47
46
  "tslib": "^2.8.1"
48
47
  },
49
48
  "lerna": {
@@ -53,5 +52,5 @@
53
52
  }
54
53
  }
55
54
  },
56
- "gitHead": "e370ab2f4ffcf4f63b52a61c9b6e97081d525e64"
55
+ "gitHead": "17bab8ab5b98554ca2db1d09bc5f61c59194a760"
57
56
  }