@crawlee/types 4.0.0-beta.58 → 4.0.0-beta.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/session.d.ts +24 -58
- package/session.d.ts.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/types",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.59",
|
|
4
4
|
"description": "Shared types for the crawlee projects",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,5 +53,5 @@
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
-
"gitHead": "
|
|
56
|
+
"gitHead": "77e1c23edec732aadfd4e27bd12a8e1a8e0ab076"
|
|
57
57
|
}
|
package/session.d.ts
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
|
|
2
|
-
import type { Cookie } from './browser.js';
|
|
3
|
-
import type { Dictionary } from './utility-types.js';
|
|
4
2
|
/**
|
|
5
3
|
* The main purpose of the ProxyInfo object is to provide information
|
|
6
4
|
* about the current proxy connection used by the crawler for the request.
|
|
@@ -81,49 +79,19 @@ export interface SessionState {
|
|
|
81
79
|
*/
|
|
82
80
|
export interface ISession {
|
|
83
81
|
readonly id: string;
|
|
84
|
-
userData: Dictionary;
|
|
85
|
-
errorScore: number;
|
|
86
|
-
usageCount: number;
|
|
87
|
-
maxErrorScore: number;
|
|
88
|
-
errorScoreDecrement: number;
|
|
89
|
-
expiresAt: Date;
|
|
90
|
-
createdAt: Date;
|
|
91
|
-
maxUsageCount: number;
|
|
92
82
|
cookieJar: CookieJar;
|
|
93
83
|
proxyInfo?: ProxyInfo;
|
|
94
|
-
/**
|
|
95
|
-
* Indicates whether the session is blocked.
|
|
96
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
97
|
-
*/
|
|
98
|
-
isBlocked(): boolean;
|
|
99
|
-
/**
|
|
100
|
-
* Indicates whether the session is expired.
|
|
101
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
102
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
103
|
-
*/
|
|
104
|
-
isExpired(): boolean;
|
|
105
|
-
/**
|
|
106
|
-
* Indicates whether the session is used maximum number of times.
|
|
107
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
108
|
-
*/
|
|
109
|
-
isMaxUsageCountReached(): boolean;
|
|
110
84
|
/**
|
|
111
85
|
* Indicates whether the session can be used for next requests.
|
|
112
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not
|
|
86
|
+
* Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
|
|
113
87
|
*/
|
|
114
88
|
isUsable(): boolean;
|
|
115
89
|
/**
|
|
116
90
|
* This method should be called after a successful session usage.
|
|
117
|
-
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
118
91
|
*/
|
|
119
92
|
markGood(): void;
|
|
120
93
|
/**
|
|
121
|
-
*
|
|
122
|
-
* @returns Represents session internal state.
|
|
123
|
-
*/
|
|
124
|
-
getState(): SessionState;
|
|
125
|
-
/**
|
|
126
|
-
* Marks session as blocked and emits event on the `SessionPool`
|
|
94
|
+
* Marks session as blocked.
|
|
127
95
|
* This method should be used if the session usage was unsuccessful
|
|
128
96
|
* and you are sure that it is because of the session configuration and not any external matters.
|
|
129
97
|
* For example when server returns 403 status code.
|
|
@@ -135,32 +103,30 @@ export interface ISession {
|
|
|
135
103
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
136
104
|
*/
|
|
137
105
|
markBad(): void;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
|
|
109
|
+
*
|
|
110
|
+
* Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
|
|
111
|
+
* `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
|
|
112
|
+
* is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
|
|
113
|
+
* the crawler, the crawler never tears it down.
|
|
114
|
+
*
|
|
115
|
+
* Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
|
|
116
|
+
* for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
|
|
117
|
+
* built-in `SessionPool` with different rotation rules.
|
|
118
|
+
*
|
|
119
|
+
* @category Scaling
|
|
120
|
+
*/
|
|
121
|
+
export interface ISessionPool {
|
|
138
122
|
/**
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
144
|
-
*/
|
|
145
|
-
setCookiesFromResponse(response: Response): void;
|
|
146
|
-
/**
|
|
147
|
-
* Saves an array with cookie objects to be used with the session.
|
|
148
|
-
* The objects should be in the format that
|
|
149
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
150
|
-
* but you can also use this function to set cookies manually:
|
|
123
|
+
* Returns a usable {@link ISession}. Without an id, the pool decides which session to return
|
|
124
|
+
* (creating a new one when appropriate). With an id, the pool returns the matching session if
|
|
125
|
+
* it is still usable.
|
|
151
126
|
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
155
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
156
|
-
* ]
|
|
157
|
-
* ```
|
|
158
|
-
*/
|
|
159
|
-
setCookies(cookies: Cookie[], url: string): void;
|
|
160
|
-
/**
|
|
161
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
162
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
127
|
+
* In case the `SessionPool` cannot provide a usable session given the configuration,
|
|
128
|
+
* this method may return `undefined`.
|
|
163
129
|
*/
|
|
164
|
-
|
|
130
|
+
getSession(sessionId?: string): Promise<ISession | undefined>;
|
|
165
131
|
}
|
|
166
132
|
//# sourceMappingURL=session.d.ts.map
|
package/session.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnE
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;;OAGG;IACH,QAAQ,IAAI,OAAO,CAAC;IAEpB;;OAEG;IACH,QAAQ,IAAI,IAAI,CAAC;IAEjB;;;;;;OAMG;IACH,MAAM,IAAI,IAAI,CAAC;IAEf;;;OAGG;IACH,OAAO,IAAI,IAAI,CAAC;CACnB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,YAAY;IACzB;;;;;;;OAOG;IACH,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;CACjE"}
|