@crawlee/types 4.0.0-beta.29 → 4.0.0-beta.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/browser.d.ts +103 -5
- package/browser.js +0 -1
- package/cookies.d.ts +82 -0
- package/cookies.js +1 -0
- package/http-client.d.ts +14 -22
- package/http-client.js +0 -1
- package/index.d.ts +8 -6
- package/index.js +1 -6
- package/logger.d.ts +73 -0
- package/logger.js +1 -0
- package/package.json +5 -6
- package/session.d.ts +53 -73
- package/session.js +0 -1
- package/status-message.d.ts +13 -0
- package/status-message.js +1 -0
- package/storages.d.ts +319 -176
- package/storages.js +0 -1
- package/utility-types.d.ts +0 -4
- package/utility-types.js +0 -1
- package/browser.d.ts.map +0 -1
- package/browser.js.map +0 -1
- package/http-client.d.ts.map +0 -1
- package/http-client.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/session.d.ts.map +0 -1
- package/session.js.map +0 -1
- package/storages.d.ts.map +0 -1
- package/storages.js.map +0 -1
- package/utility-types.d.ts.map +0 -1
- package/utility-types.js.map +0 -1
package/session.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import type { CookieJar, SerializedCookieJar } from '
|
|
2
|
-
import type { Cookie } from './browser.js';
|
|
3
|
-
import type { Dictionary } from './utility-types.js';
|
|
1
|
+
import type { CookieJar, SerializedCookieJar } from './cookies.js';
|
|
4
2
|
/**
|
|
5
3
|
* The main purpose of the ProxyInfo object is to provide information
|
|
6
4
|
* about the current proxy connection used by the crawler for the request.
|
|
@@ -49,25 +47,42 @@ export interface ProxyInfo {
|
|
|
49
47
|
* Proxy port.
|
|
50
48
|
*/
|
|
51
49
|
port: number | string;
|
|
52
|
-
/**
|
|
53
|
-
* Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
|
|
54
|
-
*/
|
|
55
|
-
proxyTier?: number;
|
|
56
50
|
/**
|
|
57
51
|
* When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
|
|
52
|
+
* The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
|
|
53
|
+
* requests when this is set.
|
|
58
54
|
*
|
|
59
55
|
* @default false
|
|
60
56
|
*/
|
|
61
57
|
ignoreTlsErrors?: boolean;
|
|
62
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Identifies the browser-like profile a {@link Session} is impersonating, so
|
|
61
|
+
* repeated requests with the same session look consistent to the target server.
|
|
62
|
+
*
|
|
63
|
+
* These fields are *hints* — `browser`, `platform`, `device`. Consumers
|
|
64
|
+
* (`@crawlee/browser-pool`, `@crawlee/impit-client`, …) derive their own rich
|
|
65
|
+
* state from them (e.g. a full browser fingerprint, a TLS impersonation profile)
|
|
66
|
+
* and cache it on their own; the session itself is read-only intent.
|
|
67
|
+
*/
|
|
68
|
+
export interface SessionFingerprint {
|
|
69
|
+
/** Browser family — consumed by HTTP clients that impersonate (e.g. `impit`). */
|
|
70
|
+
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
|
|
71
|
+
/** Platform hint — used by header generators and as a virtual session key. */
|
|
72
|
+
platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
|
|
73
|
+
/** Device class — drives header generation and viewport defaults. */
|
|
74
|
+
device?: 'desktop' | 'mobile';
|
|
75
|
+
}
|
|
63
76
|
/**
|
|
64
77
|
* Persistable {@link Session} state.
|
|
78
|
+
* @internal
|
|
65
79
|
*/
|
|
66
80
|
export interface SessionState {
|
|
67
81
|
id: string;
|
|
68
82
|
cookieJar: SerializedCookieJar;
|
|
69
83
|
proxyInfo?: ProxyInfo;
|
|
70
84
|
userData: object;
|
|
85
|
+
fingerprint?: SessionFingerprint;
|
|
71
86
|
errorScore: number;
|
|
72
87
|
maxErrorScore: number;
|
|
73
88
|
errorScoreDecrement: number;
|
|
@@ -75,6 +90,7 @@ export interface SessionState {
|
|
|
75
90
|
maxUsageCount: number;
|
|
76
91
|
expiresAt: string;
|
|
77
92
|
createdAt: string;
|
|
93
|
+
retired: boolean;
|
|
78
94
|
}
|
|
79
95
|
/**
|
|
80
96
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
@@ -84,49 +100,25 @@ export interface SessionState {
|
|
|
84
100
|
*/
|
|
85
101
|
export interface ISession {
|
|
86
102
|
readonly id: string;
|
|
87
|
-
userData: Dictionary;
|
|
88
|
-
errorScore: number;
|
|
89
|
-
usageCount: number;
|
|
90
|
-
maxErrorScore: number;
|
|
91
|
-
errorScoreDecrement: number;
|
|
92
|
-
expiresAt: Date;
|
|
93
|
-
createdAt: Date;
|
|
94
|
-
maxUsageCount: number;
|
|
95
103
|
cookieJar: CookieJar;
|
|
96
104
|
proxyInfo?: ProxyInfo;
|
|
97
|
-
|
|
98
|
-
* Indicates whether the session is blocked.
|
|
99
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
100
|
-
*/
|
|
101
|
-
isBlocked(): boolean;
|
|
102
|
-
/**
|
|
103
|
-
* Indicates whether the session is expired.
|
|
104
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
105
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
106
|
-
*/
|
|
107
|
-
isExpired(): boolean;
|
|
108
|
-
/**
|
|
109
|
-
* Indicates whether the session is used maximum number of times.
|
|
110
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
111
|
-
*/
|
|
112
|
-
isMaxUsageCountReached(): boolean;
|
|
105
|
+
fingerprint?: SessionFingerprint;
|
|
113
106
|
/**
|
|
114
107
|
* Indicates whether the session can be used for next requests.
|
|
115
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not
|
|
108
|
+
* Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
|
|
116
109
|
*/
|
|
117
110
|
isUsable(): boolean;
|
|
118
111
|
/**
|
|
119
|
-
*
|
|
120
|
-
*
|
|
112
|
+
* Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
|
|
113
|
+
* Unlike other reasons for not being usable, this means the identity itself is burned.
|
|
121
114
|
*/
|
|
122
|
-
|
|
115
|
+
isBlocked(): boolean;
|
|
123
116
|
/**
|
|
124
|
-
*
|
|
125
|
-
* @returns Represents session internal state.
|
|
117
|
+
* This method should be called after a successful session usage.
|
|
126
118
|
*/
|
|
127
|
-
|
|
119
|
+
markGood(): void;
|
|
128
120
|
/**
|
|
129
|
-
* Marks session as blocked
|
|
121
|
+
* Marks session as blocked.
|
|
130
122
|
* This method should be used if the session usage was unsuccessful
|
|
131
123
|
* and you are sure that it is because of the session configuration and not any external matters.
|
|
132
124
|
* For example when server returns 403 status code.
|
|
@@ -138,41 +130,29 @@ export interface ISession {
|
|
|
138
130
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
139
131
|
*/
|
|
140
132
|
markBad(): void;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
|
|
136
|
+
*
|
|
137
|
+
* Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
|
|
138
|
+
* `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
|
|
139
|
+
* is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
|
|
140
|
+
* the crawler, the crawler never tears it down.
|
|
141
|
+
*
|
|
142
|
+
* Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
|
|
143
|
+
* for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
|
|
144
|
+
* built-in `SessionPool` with different rotation rules.
|
|
145
|
+
*
|
|
146
|
+
* @category Scaling
|
|
147
|
+
*/
|
|
148
|
+
export interface ISessionPool {
|
|
141
149
|
/**
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* codes can be extended in the second parameter.
|
|
146
|
-
* @param statusCode HTTP status code.
|
|
147
|
-
* @returns Whether the session was retired.
|
|
148
|
-
*/
|
|
149
|
-
retireOnBlockedStatusCodes(statusCode: number): boolean;
|
|
150
|
-
/**
|
|
151
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
152
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
153
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
154
|
-
*
|
|
155
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
156
|
-
*/
|
|
157
|
-
setCookiesFromResponse(response: Response): void;
|
|
158
|
-
/**
|
|
159
|
-
* Saves an array with cookie objects to be used with the session.
|
|
160
|
-
* The objects should be in the format that
|
|
161
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
162
|
-
* but you can also use this function to set cookies manually:
|
|
150
|
+
* Returns a usable {@link ISession}. Without an id, the pool decides which session to return
|
|
151
|
+
* (creating a new one when appropriate). With an id, the pool returns the matching session if
|
|
152
|
+
* it is still usable.
|
|
163
153
|
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
167
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
168
|
-
* ]
|
|
169
|
-
* ```
|
|
170
|
-
*/
|
|
171
|
-
setCookies(cookies: Cookie[], url: string): void;
|
|
172
|
-
/**
|
|
173
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
174
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
154
|
+
* In case the `SessionPool` cannot provide a usable session given the configuration,
|
|
155
|
+
* this method may return `undefined`.
|
|
175
156
|
*/
|
|
176
|
-
|
|
157
|
+
getSession(sessionId?: string): Promise<ISession | undefined>;
|
|
177
158
|
}
|
|
178
|
-
//# sourceMappingURL=session.d.ts.map
|
package/session.js
CHANGED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
|
|
3
|
+
*
|
|
4
|
+
* Setting a status message is not a storage concern — the crawler broadcasts it through the event
|
|
5
|
+
* system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
|
|
6
|
+
* status-reporting backend.
|
|
7
|
+
*/
|
|
8
|
+
export interface SetStatusMessageOptions {
|
|
9
|
+
/** Whether this is the final status message of the run. */
|
|
10
|
+
isStatusMessageTerminal?: boolean;
|
|
11
|
+
/** The log level to log the message with. Defaults to `'DEBUG'`. */
|
|
12
|
+
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|