@crawlee/types 4.0.0-beta.21 → 4.0.0-beta.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/session.d.ts CHANGED
@@ -1,6 +1,4 @@
1
- import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
2
- import type { Cookie } from './browser.js';
3
- import type { Dictionary } from './utility-types.js';
1
+ import type { CookieJar, SerializedCookieJar } from './cookies.js';
4
2
  /**
5
3
  * The main purpose of the ProxyInfo object is to provide information
6
4
  * about the current proxy connection used by the crawler for the request.
@@ -49,25 +47,42 @@ export interface ProxyInfo {
49
47
  * Proxy port.
50
48
  */
51
49
  port: number | string;
52
- /**
53
- * Proxy tier for the current proxy, if applicable (only for `tieredProxyUrls`).
54
- */
55
- proxyTier?: number;
56
50
  /**
57
51
  * When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
52
+ * The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
53
+ * requests when this is set.
58
54
  *
59
55
  * @default false
60
56
  */
61
57
  ignoreTlsErrors?: boolean;
62
58
  }
59
+ /**
60
+ * Identifies the browser-like profile a {@link Session} is impersonating, so
61
+ * repeated requests with the same session look consistent to the target server.
62
+ *
63
+ * These fields are *hints* — `browser`, `platform`, `device`. Consumers
64
+ * (`@crawlee/browser-pool`, `@crawlee/impit-client`, …) derive their own rich
65
+ * state from them (e.g. a full browser fingerprint, a TLS impersonation profile)
66
+ * and cache it on their own; the session itself is read-only intent.
67
+ */
68
+ export interface SessionFingerprint {
69
+ /** Browser family — consumed by HTTP clients that impersonate (e.g. `impit`). */
70
+ browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
71
+ /** Platform hint — used by header generators and as a virtual session key. */
72
+ platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
73
+ /** Device class — drives header generation and viewport defaults. */
74
+ device?: 'desktop' | 'mobile';
75
+ }
63
76
  /**
64
77
  * Persistable {@link Session} state.
78
+ * @internal
65
79
  */
66
80
  export interface SessionState {
67
81
  id: string;
68
82
  cookieJar: SerializedCookieJar;
69
83
  proxyInfo?: ProxyInfo;
70
84
  userData: object;
85
+ fingerprint?: SessionFingerprint;
71
86
  errorScore: number;
72
87
  maxErrorScore: number;
73
88
  errorScoreDecrement: number;
@@ -75,6 +90,7 @@ export interface SessionState {
75
90
  maxUsageCount: number;
76
91
  expiresAt: string;
77
92
  createdAt: string;
93
+ retired: boolean;
78
94
  }
79
95
  /**
80
96
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
@@ -84,49 +100,25 @@ export interface SessionState {
84
100
  */
85
101
  export interface ISession {
86
102
  readonly id: string;
87
- userData: Dictionary;
88
- errorScore: number;
89
- usageCount: number;
90
- maxErrorScore: number;
91
- errorScoreDecrement: number;
92
- expiresAt: Date;
93
- createdAt: Date;
94
- maxUsageCount: number;
95
103
  cookieJar: CookieJar;
96
104
  proxyInfo?: ProxyInfo;
97
- /**
98
- * Indicates whether the session is blocked.
99
- * Session is blocked once it reaches the `maxErrorScore`.
100
- */
101
- isBlocked(): boolean;
102
- /**
103
- * Indicates whether the session is expired.
104
- * Session expiration is determined by the `maxAgeSecs`.
105
- * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
106
- */
107
- isExpired(): boolean;
108
- /**
109
- * Indicates whether the session is used maximum number of times.
110
- * Session maximum usage count can be changed by `maxUsageCount` parameter.
111
- */
112
- isMaxUsageCountReached(): boolean;
105
+ fingerprint?: SessionFingerprint;
113
106
  /**
114
107
  * Indicates whether the session can be used for next requests.
115
- * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
108
+ * Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
116
109
  */
117
110
  isUsable(): boolean;
118
111
  /**
119
- * This method should be called after a successful session usage.
120
- * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
112
+ * Indicates whether the session is blocked, i.e. its error score reached the maximum or it was retired via `retire()`.
113
+ * Unlike other reasons for not being usable, this means the identity itself is burned.
121
114
  */
122
- markGood(): void;
115
+ isBlocked(): boolean;
123
116
  /**
124
- * Gets session state for persistence in KeyValueStore.
125
- * @returns Represents session internal state.
117
+ * This method should be called after a successful session usage.
126
118
  */
127
- getState(): SessionState;
119
+ markGood(): void;
128
120
  /**
129
- * Marks session as blocked and emits event on the `SessionPool`
121
+ * Marks session as blocked.
130
122
  * This method should be used if the session usage was unsuccessful
131
123
  * and you are sure that it is because of the session configuration and not any external matters.
132
124
  * For example when server returns 403 status code.
@@ -138,41 +130,29 @@ export interface ISession {
138
130
  * Should be used when the session has been used unsuccessfully. For example because of timeouts.
139
131
  */
140
132
  markBad(): void;
133
+ }
134
+ /**
135
+ * Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
136
+ *
137
+ * Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
138
+ * `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
139
+ * is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
140
+ * the crawler, the crawler never tears it down.
141
+ *
142
+ * Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
143
+ * for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
144
+ * built-in `SessionPool` with different rotation rules.
145
+ *
146
+ * @category Scaling
147
+ */
148
+ export interface ISessionPool {
141
149
  /**
142
- * With certain status codes: `401`, `403` or `429` we can be certain
143
- * that the target website is blocking us. This function helps to do this conveniently
144
- * by retiring the session when such code is received. Optionally, the default status
145
- * codes can be extended in the second parameter.
146
- * @param statusCode HTTP status code.
147
- * @returns Whether the session was retired.
148
- */
149
- retireOnBlockedStatusCodes(statusCode: number): boolean;
150
- /**
151
- * Saves cookies from an HTTP response to be used with the session.
152
- * It expects an object with a `headers` property that's either an `Object`
153
- * (typical Node.js responses) or a `Function` (Puppeteer Response).
154
- *
155
- * It then parses and saves the cookies from the `set-cookie` header, if available.
156
- */
157
- setCookiesFromResponse(response: Response): void;
158
- /**
159
- * Saves an array with cookie objects to be used with the session.
160
- * The objects should be in the format that
161
- * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
162
- * but you can also use this function to set cookies manually:
150
+ * Returns a usable {@link ISession}. Without an id, the pool decides which session to return
151
+ * (creating a new one when appropriate). With an id, the pool returns the matching session if
152
+ * it is still usable.
163
153
  *
164
- * ```
165
- * [
166
- * { name: 'cookie1', value: 'my-cookie' },
167
- * { name: 'cookie2', value: 'your-cookie' }
168
- * ]
169
- * ```
170
- */
171
- setCookies(cookies: Cookie[], url: string): void;
172
- /**
173
- * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
174
- * @param url website url. Only cookies stored for this url will be returned
154
+ * In case the `SessionPool` cannot provide a usable session given the configuration,
155
+ * this method may return `undefined`.
175
156
  */
176
- getCookies(url: string): Cookie[];
157
+ getSession(sessionId?: string): Promise<ISession | undefined>;
177
158
  }
178
- //# sourceMappingURL=session.d.ts.map
package/session.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=session.js.map
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
3
+ *
4
+ * Setting a status message is not a storage concern — the crawler broadcasts it through the event
5
+ * system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
6
+ * status-reporting backend.
7
+ */
8
+ export interface SetStatusMessageOptions {
9
+ /** Whether this is the final status message of the run. */
10
+ isStatusMessageTerminal?: boolean;
11
+ /** The log level to log the message with. Defaults to `'DEBUG'`. */
12
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
13
+ }
@@ -0,0 +1 @@
1
+ export {};