@crawlee/types 4.0.0-beta.26 → 4.0.0-beta.261
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/browser.d.ts +103 -5
- package/browser.js +0 -1
- package/cookies.d.ts +82 -0
- package/cookies.js +1 -0
- package/http-client.d.ts +11 -22
- package/http-client.js +0 -1
- package/index.d.ts +8 -6
- package/index.js +1 -6
- package/logger.d.ts +73 -0
- package/logger.js +1 -0
- package/package.json +5 -6
- package/session.d.ts +53 -73
- package/session.js +0 -1
- package/status-message.d.ts +13 -0
- package/status-message.js +1 -0
- package/storages.d.ts +319 -176
- package/storages.js +0 -1
- package/utility-types.d.ts +0 -4
- package/utility-types.js +0 -1
- package/browser.d.ts.map +0 -1
- package/browser.js.map +0 -1
- package/http-client.d.ts.map +0 -1
- package/http-client.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/session.d.ts.map +0 -1
- package/session.js.map +0 -1
- package/storages.d.ts.map +0 -1
- package/storages.js.map +0 -1
- package/utility-types.d.ts.map +0 -1
- package/utility-types.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/browser.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ISession } from './session.js';
|
|
2
2
|
export interface Cookie {
|
|
3
3
|
/**
|
|
4
4
|
* Cookie name.
|
|
@@ -56,8 +56,106 @@ export interface Cookie {
|
|
|
56
56
|
*/
|
|
57
57
|
sourcePort?: number;
|
|
58
58
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
/**
|
|
60
|
+
* A snapshot of the relevant state of a page, as extracted by
|
|
61
|
+
* {@link IBrowserPool.extractPageState}.
|
|
62
|
+
*/
|
|
63
|
+
export interface PageState {
|
|
64
|
+
/**
|
|
65
|
+
* Cookies currently set in the page's browsing context.
|
|
66
|
+
*/
|
|
67
|
+
cookies: Cookie[];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Options accepted by {@link IBrowserPool.newPage}.
|
|
71
|
+
*/
|
|
72
|
+
export interface NewPageOptions {
|
|
73
|
+
/**
|
|
74
|
+
* Assign a custom ID to the page. If you don't provide one, a random string
|
|
75
|
+
* ID is generated.
|
|
76
|
+
*/
|
|
77
|
+
id?: string;
|
|
78
|
+
/**
|
|
79
|
+
* The crawling session that will use the returned page.
|
|
80
|
+
*
|
|
81
|
+
* The pool derives proxy configuration from the session's {@link
|
|
82
|
+
* ProxyInfo|proxy} (including TLS-error handling) — there are intentionally
|
|
83
|
+
* no standalone `proxyUrl` / `ignoreTlsErrors` options; configure them
|
|
84
|
+
* through the session instead.
|
|
85
|
+
*
|
|
86
|
+
* Session injection is **best-effort**: the pool may use the session's
|
|
87
|
+
* {@link ProxyInfo|proxy}, cookies, or fingerprint data to configure the
|
|
88
|
+
* page or the underlying browser, but none of this is guaranteed. Different
|
|
89
|
+
* pool implementations (or pool configurations such as `useIncognitoPages`)
|
|
90
|
+
* may support different subsets of session properties — or ignore them
|
|
91
|
+
* entirely.
|
|
92
|
+
*
|
|
93
|
+
* The crawler is still responsible for deterministic session setup (e.g.
|
|
94
|
+
* injecting cookies into the page before navigation) that must happen
|
|
95
|
+
* regardless of pool implementation.
|
|
96
|
+
*/
|
|
97
|
+
session?: ISession;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Minimal contract that any object passed to a browser crawler as its `browserPool`
|
|
101
|
+
* option must satisfy.
|
|
102
|
+
*
|
|
103
|
+
* Lifecycle (`destroy`) is the responsibility of whoever owns the pool — since a
|
|
104
|
+
* user-supplied pool is never owned by the crawler, the crawler never tears it
|
|
105
|
+
* down.
|
|
106
|
+
*
|
|
107
|
+
* Implement this interface to plug a custom page-provisioning strategy into any
|
|
108
|
+
* Crawlee browser crawler — for example a remote browser farm, a session-aware
|
|
109
|
+
* pool that pins pages to fingerprints differently, or a thin wrapper around the
|
|
110
|
+
* built-in `BrowserPool`.
|
|
111
|
+
*
|
|
112
|
+
* @category Browser management
|
|
113
|
+
*/
|
|
114
|
+
export interface IBrowserPool<Page = unknown> {
|
|
115
|
+
/**
|
|
116
|
+
* Opens a new page. The pool decides which browser to use, launching a new
|
|
117
|
+
* one if needed.
|
|
118
|
+
*/
|
|
119
|
+
newPage(options?: NewPageOptions): Promise<Page>;
|
|
120
|
+
/**
|
|
121
|
+
* Signals the pool that the caller is done with the page. The pool is
|
|
122
|
+
* responsible for closing the page and performing any necessary cleanup
|
|
123
|
+
* (e.g. retiring the underlying browser once the page's session is finished).
|
|
124
|
+
*
|
|
125
|
+
* @param page The page to release back to the pool.
|
|
126
|
+
* @param options.error If the error is a {@link SessionError}, the
|
|
127
|
+
* session that served the page is finished. A plain `SessionError`
|
|
128
|
+
* means it was blocked: discard any browser state tied to it. A
|
|
129
|
+
* {@link SessionRetiredError} means it merely reached its usage or
|
|
130
|
+
* age limit, so a pool may keep a warm page instead.
|
|
131
|
+
*/
|
|
132
|
+
closePage(page: Page, options?: {
|
|
133
|
+
error?: Error;
|
|
134
|
+
}): Promise<void>;
|
|
135
|
+
/**
|
|
136
|
+
* Extracts the relevant state from a page so the caller can persist it —
|
|
137
|
+
* for example, back-propagating cookies into the crawling {@link
|
|
138
|
+
* ISession|session}.
|
|
139
|
+
*
|
|
140
|
+
* @param page The page to read state from.
|
|
141
|
+
*/
|
|
142
|
+
extractPageState(page: Page): Promise<PageState>;
|
|
143
|
+
/**
|
|
144
|
+
* Injects state (currently just cookies) into a page. This is the
|
|
145
|
+
* counterpart to {@link IBrowserPool.extractPageState} and lets the
|
|
146
|
+
* caller set up a page — for example, seeding it with the crawling
|
|
147
|
+
* {@link ISession|session}'s cookies before navigation.
|
|
148
|
+
*
|
|
149
|
+
* As with {@link IBrowserPool.newPage}, the caller decides *what* state
|
|
150
|
+
* to inject, while the pool decides *how*.
|
|
151
|
+
*
|
|
152
|
+
* Isolation between pages is **best-effort**: depending on the pool
|
|
153
|
+
* implementation and its configuration, multiple pages may share a browsing
|
|
154
|
+
* context, so injected state (such as cookies) can bleed across pages
|
|
155
|
+
* served by the same underlying browser.
|
|
156
|
+
*
|
|
157
|
+
* @param page The page to inject state into.
|
|
158
|
+
* @param state The state to inject.
|
|
159
|
+
*/
|
|
160
|
+
injectPageState(page: Page, state: PageState): Promise<void>;
|
|
62
161
|
}
|
|
63
|
-
//# sourceMappingURL=browser.d.ts.map
|
package/browser.js
CHANGED
package/cookies.d.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cookie shape used by {@link CookieJar}. Structurally compatible with `tough-cookie`'s
|
|
3
|
+
* `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
|
|
4
|
+
* on `tough-cookie` itself.
|
|
5
|
+
*/
|
|
6
|
+
export interface SessionCookie {
|
|
7
|
+
key: string;
|
|
8
|
+
value: string;
|
|
9
|
+
expires: Date | 'Infinity' | null;
|
|
10
|
+
maxAge: number | 'Infinity' | '-Infinity' | null;
|
|
11
|
+
domain: string | null;
|
|
12
|
+
path: string | null;
|
|
13
|
+
secure: boolean;
|
|
14
|
+
httpOnly: boolean;
|
|
15
|
+
extensions: string[] | null;
|
|
16
|
+
creation: Date | 'Infinity' | null;
|
|
17
|
+
creationIndex: number;
|
|
18
|
+
hostOnly: boolean | null;
|
|
19
|
+
pathIsDefault: boolean | null;
|
|
20
|
+
lastAccessed: Date | 'Infinity' | null;
|
|
21
|
+
sameSite: string | undefined;
|
|
22
|
+
toJSON(): Record<string, unknown>;
|
|
23
|
+
clone(): SessionCookie | undefined;
|
|
24
|
+
validate(): boolean;
|
|
25
|
+
setExpires(exp: string | Date): void;
|
|
26
|
+
setMaxAge(age: number): void;
|
|
27
|
+
cookieString(): string;
|
|
28
|
+
toString(): string;
|
|
29
|
+
TTL(now?: number): number;
|
|
30
|
+
expiryTime(now?: Date): number | undefined;
|
|
31
|
+
expiryDate(now?: Date): Date | undefined;
|
|
32
|
+
isPersistent(): boolean;
|
|
33
|
+
canonicalizedDomain(): string | undefined;
|
|
34
|
+
cdomain(): string | undefined;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Options for {@link CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
|
|
38
|
+
* `SetCookieOptions`.
|
|
39
|
+
*/
|
|
40
|
+
export interface CookieJarSetCookieOptions {
|
|
41
|
+
loose?: boolean;
|
|
42
|
+
sameSiteContext?: 'strict' | 'lax' | 'none';
|
|
43
|
+
ignoreError?: boolean;
|
|
44
|
+
http?: boolean;
|
|
45
|
+
now?: Date;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Options for {@link CookieJar}'s cookie-reading methods. Structurally compatible with
|
|
49
|
+
* `tough-cookie`'s `GetCookiesOptions`.
|
|
50
|
+
*/
|
|
51
|
+
export interface CookieJarGetCookiesOptions {
|
|
52
|
+
http?: boolean;
|
|
53
|
+
expire?: boolean;
|
|
54
|
+
allPaths?: boolean;
|
|
55
|
+
sameSiteContext?: 'none' | 'lax' | 'strict';
|
|
56
|
+
sort?: boolean;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
|
|
60
|
+
* what backs {@link Session} by default — so `@crawlee/types` does not have to depend on
|
|
61
|
+
* `tough-cookie` for it.
|
|
62
|
+
*/
|
|
63
|
+
export interface CookieJar {
|
|
64
|
+
setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
|
|
65
|
+
getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
|
|
66
|
+
getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
|
|
67
|
+
getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
|
|
68
|
+
serialize(): Promise<SerializedCookieJar>;
|
|
69
|
+
toJSON(): SerializedCookieJar | undefined;
|
|
70
|
+
clone(): Promise<CookieJar>;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* JSON representation of a {@link CookieJar}. Structurally compatible with `tough-cookie`'s
|
|
74
|
+
* `SerializedCookieJar`.
|
|
75
|
+
*/
|
|
76
|
+
export interface SerializedCookieJar {
|
|
77
|
+
version: string;
|
|
78
|
+
storeType: string | null;
|
|
79
|
+
rejectPublicSuffixes: boolean;
|
|
80
|
+
cookies: Record<string, unknown>[];
|
|
81
|
+
[key: string]: unknown;
|
|
82
|
+
}
|
package/cookies.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/http-client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Readable } from 'node:stream';
|
|
2
|
-
import type { CookieJar } from '
|
|
2
|
+
import type { CookieJar } from './cookies.js';
|
|
3
3
|
import type { ISession } from './session.js';
|
|
4
4
|
import type { AllowedHttpMethods } from './utility-types.js';
|
|
5
5
|
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
|
|
@@ -15,17 +15,8 @@ export interface HttpRequest {
|
|
|
15
15
|
timeout?: number;
|
|
16
16
|
cookieJar?: CookieJar;
|
|
17
17
|
followRedirect?: boolean | ((response: any) => boolean);
|
|
18
|
-
maxRedirects?: number;
|
|
19
18
|
encoding?: BufferEncoding;
|
|
20
|
-
throwHttpErrors?: boolean;
|
|
21
19
|
proxyUrl?: string;
|
|
22
|
-
headerGeneratorOptions?: Record<string, unknown>;
|
|
23
|
-
useHeaderGenerator?: boolean;
|
|
24
|
-
headerGenerator?: {
|
|
25
|
-
getHeaders: (options: Record<string, unknown>) => Record<string, string>;
|
|
26
|
-
};
|
|
27
|
-
insecureHTTPParser?: boolean;
|
|
28
|
-
sessionToken?: object;
|
|
29
20
|
}
|
|
30
21
|
/**
|
|
31
22
|
* Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
|
|
@@ -42,26 +33,25 @@ export interface HttpRequestOptions extends HttpRequest {
|
|
|
42
33
|
/** Basic HTTP Auth password */
|
|
43
34
|
password?: string;
|
|
44
35
|
}
|
|
45
|
-
/**
|
|
46
|
-
* Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
|
|
47
|
-
*/
|
|
48
|
-
export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
|
|
49
|
-
url?: string | URL;
|
|
50
|
-
headers: Headers;
|
|
51
|
-
}) => void;
|
|
52
36
|
export interface SendRequestOptions {
|
|
53
37
|
session?: ISession;
|
|
54
38
|
cookieJar?: CookieJar;
|
|
55
|
-
|
|
39
|
+
/** Timeout for the HTTP request in milliseconds. */
|
|
40
|
+
timeoutMillis?: number;
|
|
41
|
+
/** An AbortSignal to cancel the HTTP request. */
|
|
42
|
+
signal?: AbortSignal;
|
|
56
43
|
/**
|
|
57
44
|
* Overrides the proxy URL set in the `session` for this request.
|
|
58
45
|
*
|
|
59
46
|
* Note that setting this manually can interfere with session proxy rotation.
|
|
60
47
|
*/
|
|
61
48
|
proxyUrl?: string;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
49
|
+
/**
|
|
50
|
+
* When `true`, TLS certificate errors are ignored for this request. Also enabled by
|
|
51
|
+
* `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
|
|
52
|
+
* disable TLS verification (e.g. the native fetch fallback) ignore it.
|
|
53
|
+
*/
|
|
54
|
+
ignoreTlsErrors?: boolean;
|
|
65
55
|
}
|
|
66
56
|
/**
|
|
67
57
|
* Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
|
|
@@ -72,4 +62,3 @@ export interface BaseHttpClient {
|
|
|
72
62
|
*/
|
|
73
63
|
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
74
64
|
}
|
|
75
|
-
//# sourceMappingURL=http-client.d.ts.map
|
package/http-client.js
CHANGED
package/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
|
|
1
|
+
export type * from './status-message.js';
|
|
2
|
+
export type * from './storages.js';
|
|
3
|
+
export type * from './utility-types.js';
|
|
4
|
+
export type * from './browser.js';
|
|
5
|
+
export type * from './cookies.js';
|
|
6
|
+
export type * from './http-client.js';
|
|
7
|
+
export type * from './session.js';
|
|
8
|
+
export type * from './logger.js';
|
package/index.js
CHANGED
package/logger.d.ts
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration options for Crawlee logger implementations.
|
|
3
|
+
*/
|
|
4
|
+
export interface CrawleeLoggerOptions {
|
|
5
|
+
/** Prefix to be prepended to each logged line. */
|
|
6
|
+
prefix?: string | null;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Options for a single log call.
|
|
10
|
+
*/
|
|
11
|
+
export interface LogOptions {
|
|
12
|
+
/** Log the message only the first time this logger sees that text, ignoring later calls with it. */
|
|
13
|
+
once?: boolean;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Interface for Crawlee logger implementations.
|
|
17
|
+
* This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
|
|
18
|
+
* compatibility with the default `@apify/log` implementation.
|
|
19
|
+
*/
|
|
20
|
+
export interface CrawleeLogger {
|
|
21
|
+
/**
|
|
22
|
+
* Returns the logger configuration.
|
|
23
|
+
*/
|
|
24
|
+
getOptions(): CrawleeLoggerOptions;
|
|
25
|
+
/**
|
|
26
|
+
* Configures logger options.
|
|
27
|
+
*/
|
|
28
|
+
setOptions(options: Partial<CrawleeLoggerOptions>): void;
|
|
29
|
+
/**
|
|
30
|
+
* Creates a new instance of logger that inherits settings from a parent logger.
|
|
31
|
+
*/
|
|
32
|
+
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
|
|
33
|
+
/**
|
|
34
|
+
* Logs an `ERROR` message.
|
|
35
|
+
*/
|
|
36
|
+
error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
37
|
+
/**
|
|
38
|
+
* Logs an `ERROR` level message with a nicely formatted exception.
|
|
39
|
+
*/
|
|
40
|
+
exception(exception: Error, message: string, data?: Record<string, unknown>): void;
|
|
41
|
+
/**
|
|
42
|
+
* Logs a `SOFT_FAIL` level message.
|
|
43
|
+
*/
|
|
44
|
+
softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
45
|
+
/**
|
|
46
|
+
* Logs a `WARNING` level message.
|
|
47
|
+
*/
|
|
48
|
+
warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
49
|
+
/**
|
|
50
|
+
* Logs a `WARNING` level message only once. Shorthand for `warning(message, undefined, { once: true })`.
|
|
51
|
+
*/
|
|
52
|
+
warningOnce(message: string): void;
|
|
53
|
+
/**
|
|
54
|
+
* Logs an `INFO` message.
|
|
55
|
+
*/
|
|
56
|
+
info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
57
|
+
/**
|
|
58
|
+
* Logs a `DEBUG` message.
|
|
59
|
+
*/
|
|
60
|
+
debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
61
|
+
/**
|
|
62
|
+
* Logs a `PERF` level message for performance tracking.
|
|
63
|
+
*/
|
|
64
|
+
perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
|
|
65
|
+
/**
|
|
66
|
+
* Logs given message only once as WARNING for deprecated features.
|
|
67
|
+
*/
|
|
68
|
+
deprecated(message: string): void;
|
|
69
|
+
/**
|
|
70
|
+
* Logs a message at the given level. Useful when the log level is determined dynamically.
|
|
71
|
+
*/
|
|
72
|
+
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
|
|
73
|
+
}
|
package/logger.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/types",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.261",
|
|
4
4
|
"description": "Shared types for the crawlee projects",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
},
|
|
35
35
|
"homepage": "https://crawlee.dev",
|
|
36
36
|
"scripts": {
|
|
37
|
-
"build": "
|
|
37
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
38
38
|
"clean": "rimraf ./dist",
|
|
39
39
|
"compile": "tsc -p tsconfig.build.json",
|
|
40
40
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -43,8 +43,7 @@
|
|
|
43
43
|
"access": "public"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"
|
|
47
|
-
"tslib": "^2.8.1"
|
|
46
|
+
"tslib": "^2.9.1"
|
|
48
47
|
},
|
|
49
48
|
"lerna": {
|
|
50
49
|
"command": {
|
|
@@ -53,5 +52,5 @@
|
|
|
53
52
|
}
|
|
54
53
|
}
|
|
55
54
|
},
|
|
56
|
-
"gitHead": "
|
|
55
|
+
"gitHead": "24e897c21c5f7ba19846bbca3eba4def5331a081"
|
|
57
56
|
}
|
package/session.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import type { CookieJar, SerializedCookieJar } from '
|
|
2
|
-
import type { Cookie } from './browser.js';
|
|
3
|
-
import type { Dictionary } from './utility-types.js';
|
|
1
|
+
import type { CookieJar, SerializedCookieJar } from './cookies.js';
|
|
4
2
|
/**
|
|
5
3
|
* The main purpose of the ProxyInfo object is to provide information
|
|
6
4
|
* about the current proxy connection used by the crawler for the request.
|
|
@@ -49,25 +47,42 @@ export interface ProxyInfo {
|
|
|
49
47
|
* Proxy port.
|
|
50
48
|
*/
|
|
51
49
|
port: number | string;
|
|
52
|
-
/**
|
|
53
|
-
* Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
|
|
54
|
-
*/
|
|
55
|
-
proxyTier?: number;
|
|
56
50
|
/**
|
|
57
51
|
* When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
|
|
52
|
+
* The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
|
|
53
|
+
* requests when this is set.
|
|
58
54
|
*
|
|
59
55
|
* @default false
|
|
60
56
|
*/
|
|
61
57
|
ignoreTlsErrors?: boolean;
|
|
62
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Identifies the browser-like profile a {@link Session} is impersonating, so
|
|
61
|
+
* repeated requests with the same session look consistent to the target server.
|
|
62
|
+
*
|
|
63
|
+
* These fields are *hints* — `browser`, `platform`, `device`. Consumers
|
|
64
|
+
* (`@crawlee/browser-pool`, `@crawlee/impit-client`, …) derive their own rich
|
|
65
|
+
* state from them (e.g. a full browser fingerprint, a TLS impersonation profile)
|
|
66
|
+
* and cache it on their own; the session itself is read-only intent.
|
|
67
|
+
*/
|
|
68
|
+
export interface SessionFingerprint {
|
|
69
|
+
/** Browser family — consumed by HTTP clients that impersonate (e.g. `impit`). */
|
|
70
|
+
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
|
|
71
|
+
/** Platform hint — used by header generators and as a virtual session key. */
|
|
72
|
+
platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
|
|
73
|
+
/** Device class — drives header generation and viewport defaults. */
|
|
74
|
+
device?: 'desktop' | 'mobile';
|
|
75
|
+
}
|
|
63
76
|
/**
|
|
64
77
|
* Persistable {@link Session} state.
|
|
78
|
+
* @internal
|
|
65
79
|
*/
|
|
66
80
|
export interface SessionState {
|
|
67
81
|
id: string;
|
|
68
82
|
cookieJar: SerializedCookieJar;
|
|
69
83
|
proxyInfo?: ProxyInfo;
|
|
70
84
|
userData: object;
|
|
85
|
+
fingerprint?: SessionFingerprint;
|
|
71
86
|
errorScore: number;
|
|
72
87
|
maxErrorScore: number;
|
|
73
88
|
errorScoreDecrement: number;
|
|
@@ -75,6 +90,7 @@ export interface SessionState {
|
|
|
75
90
|
maxUsageCount: number;
|
|
76
91
|
expiresAt: string;
|
|
77
92
|
createdAt: string;
|
|
93
|
+
retired: boolean;
|
|
78
94
|
}
|
|
79
95
|
/**
|
|
80
96
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
@@ -84,49 +100,25 @@ export interface SessionState {
|
|
|
84
100
|
*/
|
|
85
101
|
export interface ISession {
|
|
86
102
|
readonly id: string;
|
|
87
|
-
userData: Dictionary;
|
|
88
|
-
errorScore: number;
|
|
89
|
-
usageCount: number;
|
|
90
|
-
maxErrorScore: number;
|
|
91
|
-
errorScoreDecrement: number;
|
|
92
|
-
expiresAt: Date;
|
|
93
|
-
createdAt: Date;
|
|
94
|
-
maxUsageCount: number;
|
|
95
103
|
cookieJar: CookieJar;
|
|
96
104
|
proxyInfo?: ProxyInfo;
|
|
97
|
-
|
|
98
|
-
* Indicates whether the session is blocked.
|
|
99
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
100
|
-
*/
|
|
101
|
-
isBlocked(): boolean;
|
|
102
|
-
/**
|
|
103
|
-
* Indicates whether the session is expired.
|
|
104
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
105
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
106
|
-
*/
|
|
107
|
-
isExpired(): boolean;
|
|
108
|
-
/**
|
|
109
|
-
* Indicates whether the session is used maximum number of times.
|
|
110
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
111
|
-
*/
|
|
112
|
-
isMaxUsageCountReached(): boolean;
|
|
105
|
+
fingerprint?: SessionFingerprint;
|
|
113
106
|
/**
|
|
114
107
|
* Indicates whether the session can be used for next requests.
|
|
115
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not
|
|
108
|
+
* Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
|
|
116
109
|
*/
|
|
117
110
|
isUsable(): boolean;
|
|
118
111
|
/**
|
|
119
|
-
*
|
|
120
|
-
*
|
|
112
|
+
* Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
|
|
113
|
+
* Unlike other reasons for not being usable, this means the identity itself is burned.
|
|
121
114
|
*/
|
|
122
|
-
|
|
115
|
+
isBlocked(): boolean;
|
|
123
116
|
/**
|
|
124
|
-
*
|
|
125
|
-
* @returns Represents session internal state.
|
|
117
|
+
* This method should be called after a successful session usage.
|
|
126
118
|
*/
|
|
127
|
-
|
|
119
|
+
markGood(): void;
|
|
128
120
|
/**
|
|
129
|
-
* Marks session as blocked
|
|
121
|
+
* Marks session as blocked.
|
|
130
122
|
* This method should be used if the session usage was unsuccessful
|
|
131
123
|
* and you are sure that it is because of the session configuration and not any external matters.
|
|
132
124
|
* For example when server returns 403 status code.
|
|
@@ -138,41 +130,29 @@ export interface ISession {
|
|
|
138
130
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
139
131
|
*/
|
|
140
132
|
markBad(): void;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
|
|
136
|
+
*
|
|
137
|
+
* Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
|
|
138
|
+
* `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
|
|
139
|
+
* is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
|
|
140
|
+
* the crawler, the crawler never tears it down.
|
|
141
|
+
*
|
|
142
|
+
* Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
|
|
143
|
+
* for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
|
|
144
|
+
* built-in `SessionPool` with different rotation rules.
|
|
145
|
+
*
|
|
146
|
+
* @category Scaling
|
|
147
|
+
*/
|
|
148
|
+
export interface ISessionPool {
|
|
141
149
|
/**
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* codes can be extended in the second parameter.
|
|
146
|
-
* @param statusCode HTTP status code.
|
|
147
|
-
* @returns Whether the session was retired.
|
|
148
|
-
*/
|
|
149
|
-
retireOnBlockedStatusCodes(statusCode: number): boolean;
|
|
150
|
-
/**
|
|
151
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
152
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
153
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
154
|
-
*
|
|
155
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
156
|
-
*/
|
|
157
|
-
setCookiesFromResponse(response: Response): void;
|
|
158
|
-
/**
|
|
159
|
-
* Saves an array with cookie objects to be used with the session.
|
|
160
|
-
* The objects should be in the format that
|
|
161
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
162
|
-
* but you can also use this function to set cookies manually:
|
|
150
|
+
* Returns a usable {@link ISession}. Without an id, the pool decides which session to return
|
|
151
|
+
* (creating a new one when appropriate). With an id, the pool returns the matching session if
|
|
152
|
+
* it is still usable.
|
|
163
153
|
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
167
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
168
|
-
* ]
|
|
169
|
-
* ```
|
|
170
|
-
*/
|
|
171
|
-
setCookies(cookies: Cookie[], url: string): void;
|
|
172
|
-
/**
|
|
173
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
174
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
154
|
+
* In case the `SessionPool` cannot provide a usable session given the configuration,
|
|
155
|
+
* this method may return `undefined`.
|
|
175
156
|
*/
|
|
176
|
-
|
|
157
|
+
getSession(sessionId?: string): Promise<ISession | undefined>;
|
|
177
158
|
}
|
|
178
|
-
//# sourceMappingURL=session.d.ts.map
|