@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +9 -5
  2. package/autoscaling/autoscaled_pool.d.ts +3 -5
  3. package/autoscaling/autoscaled_pool.d.ts.map +1 -1
  4. package/autoscaling/autoscaled_pool.js +3 -9
  5. package/autoscaling/autoscaled_pool.js.map +1 -1
  6. package/autoscaling/snapshotter.d.ts +3 -13
  7. package/autoscaling/snapshotter.d.ts.map +1 -1
  8. package/autoscaling/snapshotter.js +18 -29
  9. package/autoscaling/snapshotter.js.map +1 -1
  10. package/autoscaling/system_status.d.ts +0 -3
  11. package/autoscaling/system_status.d.ts.map +1 -1
  12. package/autoscaling/system_status.js +2 -3
  13. package/autoscaling/system_status.js.map +1 -1
  14. package/configuration.d.ts +85 -227
  15. package/configuration.d.ts.map +1 -1
  16. package/configuration.js +159 -223
  17. package/configuration.js.map +1 -1
  18. package/cookie_utils.d.ts +4 -2
  19. package/cookie_utils.d.ts.map +1 -1
  20. package/cookie_utils.js +18 -12
  21. package/cookie_utils.js.map +1 -1
  22. package/crawlers/context_pipeline.d.ts +71 -0
  23. package/crawlers/context_pipeline.d.ts.map +1 -0
  24. package/crawlers/context_pipeline.js +123 -0
  25. package/crawlers/context_pipeline.js.map +1 -0
  26. package/crawlers/crawler_commons.d.ts +19 -28
  27. package/crawlers/crawler_commons.d.ts.map +1 -1
  28. package/crawlers/crawler_commons.js +12 -20
  29. package/crawlers/crawler_commons.js.map +1 -1
  30. package/crawlers/crawler_utils.d.ts +2 -2
  31. package/crawlers/crawler_utils.d.ts.map +1 -1
  32. package/crawlers/crawler_utils.js +1 -1
  33. package/crawlers/crawler_utils.js.map +1 -1
  34. package/crawlers/error_snapshotter.d.ts +3 -2
  35. package/crawlers/error_snapshotter.d.ts.map +1 -1
  36. package/crawlers/error_snapshotter.js +2 -2
  37. package/crawlers/error_snapshotter.js.map +1 -1
  38. package/crawlers/error_tracker.d.ts +2 -1
  39. package/crawlers/error_tracker.d.ts.map +1 -1
  40. package/crawlers/error_tracker.js.map +1 -1
  41. package/crawlers/index.d.ts +1 -1
  42. package/crawlers/index.d.ts.map +1 -1
  43. package/crawlers/index.js +1 -1
  44. package/crawlers/index.js.map +1 -1
  45. package/crawlers/internals/types.d.ts +8 -0
  46. package/crawlers/internals/types.d.ts.map +1 -0
  47. package/crawlers/internals/types.js +2 -0
  48. package/crawlers/internals/types.js.map +1 -0
  49. package/crawlers/statistics.d.ts +15 -15
  50. package/crawlers/statistics.d.ts.map +1 -1
  51. package/crawlers/statistics.js +21 -24
  52. package/crawlers/statistics.js.map +1 -1
  53. package/enqueue_links/enqueue_links.d.ts +32 -18
  54. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  55. package/enqueue_links/enqueue_links.js +45 -24
  56. package/enqueue_links/enqueue_links.js.map +1 -1
  57. package/enqueue_links/shared.d.ts +25 -8
  58. package/enqueue_links/shared.d.ts.map +1 -1
  59. package/enqueue_links/shared.js +69 -37
  60. package/enqueue_links/shared.js.map +1 -1
  61. package/errors.d.ts +33 -3
  62. package/errors.d.ts.map +1 -1
  63. package/errors.js +48 -4
  64. package/errors.js.map +1 -1
  65. package/events/event_manager.d.ts +8 -5
  66. package/events/event_manager.d.ts.map +1 -1
  67. package/events/event_manager.js +7 -9
  68. package/events/event_manager.js.map +1 -1
  69. package/events/local_event_manager.d.ts +14 -4
  70. package/events/local_event_manager.d.ts.map +1 -1
  71. package/events/local_event_manager.js +33 -39
  72. package/events/local_event_manager.js.map +1 -1
  73. package/index.d.ts +3 -2
  74. package/index.d.ts.map +1 -1
  75. package/index.js +2 -1
  76. package/index.js.map +1 -1
  77. package/log.d.ts +82 -2
  78. package/log.d.ts.map +1 -1
  79. package/log.js +102 -0
  80. package/log.js.map +1 -1
  81. package/package.json +9 -10
  82. package/proxy_configuration.d.ts +14 -148
  83. package/proxy_configuration.d.ts.map +1 -1
  84. package/proxy_configuration.js +19 -167
  85. package/proxy_configuration.js.map +1 -1
  86. package/recoverable_state.d.ts +121 -0
  87. package/recoverable_state.d.ts.map +1 -0
  88. package/recoverable_state.js +142 -0
  89. package/recoverable_state.js.map +1 -0
  90. package/request.d.ts +74 -10
  91. package/request.d.ts.map +1 -1
  92. package/request.js +85 -23
  93. package/request.js.map +1 -1
  94. package/router.d.ts.map +1 -1
  95. package/router.js.map +1 -1
  96. package/serialization.js +1 -1
  97. package/serialization.js.map +1 -1
  98. package/service_locator.d.ts +157 -0
  99. package/service_locator.d.ts.map +1 -0
  100. package/service_locator.js +234 -0
  101. package/service_locator.js.map +1 -0
  102. package/session_pool/index.d.ts +0 -1
  103. package/session_pool/index.d.ts.map +1 -1
  104. package/session_pool/index.js +0 -1
  105. package/session_pool/index.js.map +1 -1
  106. package/session_pool/session.d.ts +26 -72
  107. package/session_pool/session.d.ts.map +1 -1
  108. package/session_pool/session.js +36 -98
  109. package/session_pool/session.js.map +1 -1
  110. package/session_pool/session_pool.d.ts +65 -71
  111. package/session_pool/session_pool.d.ts.map +1 -1
  112. package/session_pool/session_pool.js +101 -100
  113. package/session_pool/session_pool.js.map +1 -1
  114. package/storages/dataset.d.ts +90 -46
  115. package/storages/dataset.d.ts.map +1 -1
  116. package/storages/dataset.js +149 -121
  117. package/storages/dataset.js.map +1 -1
  118. package/storages/index.d.ts +3 -1
  119. package/storages/index.d.ts.map +1 -1
  120. package/storages/index.js +3 -1
  121. package/storages/index.js.map +1 -1
  122. package/storages/key_value_store.d.ts +104 -22
  123. package/storages/key_value_store.d.ts.map +1 -1
  124. package/storages/key_value_store.js +166 -51
  125. package/storages/key_value_store.js.map +1 -1
  126. package/storages/request_list.d.ts +9 -9
  127. package/storages/request_list.d.ts.map +1 -1
  128. package/storages/request_list.js +13 -8
  129. package/storages/request_list.js.map +1 -1
  130. package/storages/request_list_adapter.d.ts +58 -0
  131. package/storages/request_list_adapter.d.ts.map +1 -0
  132. package/storages/request_list_adapter.js +81 -0
  133. package/storages/request_list_adapter.js.map +1 -0
  134. package/storages/request_manager_tandem.d.ts +68 -0
  135. package/storages/request_manager_tandem.d.ts.map +1 -0
  136. package/storages/request_manager_tandem.js +124 -0
  137. package/storages/request_manager_tandem.js.map +1 -0
  138. package/storages/request_provider.d.ts +87 -22
  139. package/storages/request_provider.d.ts.map +1 -1
  140. package/storages/request_provider.js +127 -77
  141. package/storages/request_provider.js.map +1 -1
  142. package/storages/request_queue.d.ts +1 -3
  143. package/storages/request_queue.d.ts.map +1 -1
  144. package/storages/request_queue.js +2 -4
  145. package/storages/request_queue.js.map +1 -1
  146. package/storages/request_queue_v2.d.ts +3 -3
  147. package/storages/request_queue_v2.d.ts.map +1 -1
  148. package/storages/request_queue_v2.js +4 -5
  149. package/storages/request_queue_v2.js.map +1 -1
  150. package/storages/sitemap_request_list.d.ts +5 -5
  151. package/storages/sitemap_request_list.d.ts.map +1 -1
  152. package/storages/sitemap_request_list.js +10 -7
  153. package/storages/sitemap_request_list.js.map +1 -1
  154. package/storages/storage_instance_manager.d.ts +91 -0
  155. package/storages/storage_instance_manager.d.ts.map +1 -0
  156. package/storages/storage_instance_manager.js +236 -0
  157. package/storages/storage_instance_manager.js.map +1 -0
  158. package/storages/utils.d.ts +47 -1
  159. package/storages/utils.d.ts.map +1 -1
  160. package/storages/utils.js +57 -5
  161. package/storages/utils.js.map +1 -1
  162. package/typedefs.d.ts +1 -1
  163. package/typedefs.d.ts.map +1 -1
  164. package/validators.d.ts +4 -0
  165. package/validators.d.ts.map +1 -1
  166. package/validators.js +4 -0
  167. package/validators.js.map +1 -1
  168. package/crawlers/crawler_extension.d.ts +0 -12
  169. package/crawlers/crawler_extension.d.ts.map +0 -1
  170. package/crawlers/crawler_extension.js +0 -14
  171. package/crawlers/crawler_extension.js.map +0 -1
  172. package/http_clients/base-http-client.d.ts +0 -134
  173. package/http_clients/base-http-client.d.ts.map +0 -1
  174. package/http_clients/base-http-client.js +0 -33
  175. package/http_clients/base-http-client.js.map +0 -1
  176. package/http_clients/form-data-like.d.ts +0 -67
  177. package/http_clients/form-data-like.d.ts.map +0 -1
  178. package/http_clients/form-data-like.js +0 -5
  179. package/http_clients/form-data-like.js.map +0 -1
  180. package/http_clients/got-scraping-http-client.d.ts +0 -15
  181. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  182. package/http_clients/got-scraping-http-client.js +0 -69
  183. package/http_clients/got-scraping-http-client.js.map +0 -1
  184. package/http_clients/index.d.ts +0 -3
  185. package/http_clients/index.d.ts.map +0 -1
  186. package/http_clients/index.js +0 -3
  187. package/http_clients/index.js.map +0 -1
  188. package/session_pool/events.d.ts +0 -3
  189. package/session_pool/events.d.ts.map +0 -1
  190. package/session_pool/events.js +0 -3
  191. package/session_pool/events.js.map +0 -1
  192. package/storages/storage_manager.d.ts +0 -58
  193. package/storages/storage_manager.d.ts.map +0 -1
  194. package/storages/storage_manager.js +0 -105
  195. package/storages/storage_manager.js.map +0 -1
  196. package/tsconfig.build.tsbuildinfo +0 -1
@@ -1,10 +1,8 @@
1
- import { EventEmitter } from 'node:events';
2
1
  import ow from 'ow';
3
2
  import { CookieJar } from 'tough-cookie';
4
3
  import { cryptoRandomObjectId } from '@apify/utilities';
5
- import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
6
- import { log as defaultLog } from '../log.js';
7
- import { EVENT_SESSION_RETIRED } from './events.js';
4
+ import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
5
+ import { serviceLocator } from '../service_locator.js';
8
6
  /**
9
7
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
10
8
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -13,7 +11,6 @@ import { EVENT_SESSION_RETIRED } from './events.js';
13
11
  */
14
12
  export class Session {
15
13
  id;
16
- maxAgeSecs;
17
14
  userData;
18
15
  _maxErrorScore;
19
16
  _errorScoreDecrement;
@@ -21,8 +18,9 @@ export class Session {
21
18
  _expiresAt;
22
19
  _usageCount;
23
20
  _maxUsageCount;
24
- sessionPool;
25
21
  _errorScore;
22
+ _retired = false;
23
+ _proxyInfo;
26
24
  _cookieJar;
27
25
  log;
28
26
  get errorScore() {
@@ -49,14 +47,24 @@ export class Session {
49
47
  get cookieJar() {
50
48
  return this._cookieJar;
51
49
  }
50
+ get proxyInfo() {
51
+ return this._proxyInfo;
52
+ }
53
+ /**
54
+ * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
55
+ * a retired session is never picked by the pool and cannot be revived via `markGood()`.
56
+ */
57
+ get retired() {
58
+ return this._retired;
59
+ }
52
60
  /**
53
61
  * Session configuration.
54
62
  */
55
- constructor(options) {
63
+ constructor(options = {}) {
56
64
  ow(options, ow.object.exactShape({
57
- sessionPool: ow.object.instanceOf(EventEmitter),
58
65
  id: ow.optional.string,
59
66
  cookieJar: ow.optional.object,
67
+ proxyInfo: ow.optional.object,
60
68
  maxAgeSecs: ow.optional.number,
61
69
  userData: ow.optional.object,
62
70
  maxErrorScore: ow.optional.number,
@@ -66,14 +74,15 @@ export class Session {
66
74
  usageCount: ow.optional.number,
67
75
  errorScore: ow.optional.number,
68
76
  maxUsageCount: ow.optional.number,
77
+ retired: ow.optional.boolean,
69
78
  log: ow.optional.object,
70
79
  }));
71
- const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
80
+ const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), } = options;
72
81
  const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
73
82
  this.log = log.child({ prefix: 'Session' });
74
83
  this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
84
+ this._proxyInfo = proxyInfo;
75
85
  this.id = id;
76
- this.maxAgeSecs = maxAgeSecs;
77
86
  this.userData = userData;
78
87
  this._maxErrorScore = maxErrorScore;
79
88
  this._errorScoreDecrement = errorScoreDecrement;
@@ -83,7 +92,7 @@ export class Session {
83
92
  this._usageCount = usageCount; // indicates how many times the session has been used
84
93
  this._errorScore = errorScore; // indicates number of markBaded request with the session
85
94
  this._maxUsageCount = maxUsageCount;
86
- this.sessionPool = sessionPool;
95
+ this._retired = retired;
87
96
  }
88
97
  /**
89
98
  * Indicates whether the session is blocked.
@@ -109,10 +118,10 @@ export class Session {
109
118
  }
110
119
  /**
111
120
  * Indicates whether the session can be used for next requests.
112
- * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
121
+ * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
113
122
  */
114
123
  isUsable() {
115
- return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
124
+ return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
116
125
  }
117
126
  /**
118
127
  * This method should be called after a successful session usage.
@@ -133,6 +142,7 @@ export class Session {
133
142
  return {
134
143
  id: this.id,
135
144
  cookieJar: this.cookieJar.toJSON(),
145
+ proxyInfo: this._proxyInfo,
136
146
  userData: this.userData,
137
147
  maxErrorScore: this.maxErrorScore,
138
148
  errorScoreDecrement: this.errorScoreDecrement,
@@ -141,21 +151,22 @@ export class Session {
141
151
  usageCount: this.usageCount,
142
152
  maxUsageCount: this.maxUsageCount,
143
153
  errorScore: this.errorScore,
154
+ retired: this._retired,
144
155
  };
145
156
  }
146
157
  /**
147
- * Marks session as blocked and emits event on the `SessionPool`
148
- * This method should be used if the session usage was unsuccessful
149
- * and you are sure that it is because of the session configuration and not any external matters.
150
- * For example when server returns 403 status code.
151
- * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
158
+ * Permanently retires the session `isUsable()` will return `false` from here on,
159
+ * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
160
+ *
161
+ * Use this when you're confident the session itself is the problem (e.g. a `403` response).
162
+ * For transient external failures (such as `5XX` responses), use `markBad()` instead.
152
163
  */
153
164
  retire() {
154
- // mark it as an invalid by increasing the error score count.
165
+ if (this._retired)
166
+ return;
155
167
  this._errorScore += this._maxErrorScore;
156
168
  this._usageCount += 1;
157
- // emit event so we can retire browser in puppeteer pool
158
- this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
169
+ this._retired = true;
159
170
  }
160
171
  /**
161
172
  * Increases usage and error count.
@@ -166,65 +177,6 @@ export class Session {
166
177
  this._usageCount += 1;
167
178
  this._maybeSelfRetire();
168
179
  }
169
- /**
170
- * With certain status codes: `401`, `403` or `429` we can be certain
171
- * that the target website is blocking us. This function helps to do this conveniently
172
- * by retiring the session when such code is received. Optionally, the default status
173
- * codes can be extended in the second parameter.
174
- * @param statusCode HTTP status code.
175
- * @returns Whether the session was retired.
176
- */
177
- retireOnBlockedStatusCodes(statusCode) {
178
- // eslint-disable-next-line dot-notation -- accessing private property
179
- const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
180
- if (isBlocked) {
181
- this.retire();
182
- }
183
- return isBlocked;
184
- }
185
- /**
186
- * Saves cookies from an HTTP response to be used with the session.
187
- * It expects an object with a `headers` property that's either an `Object`
188
- * (typical Node.js responses) or a `Function` (Puppeteer Response).
189
- *
190
- * It then parses and saves the cookies from the `set-cookie` header, if available.
191
- */
192
- setCookiesFromResponse(response) {
193
- try {
194
- const cookies = getCookiesFromResponse(response).filter((c) => c);
195
- this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
196
- }
197
- catch (e) {
198
- const err = e;
199
- // if invalid Cookie header is provided just log the exception.
200
- this.log.exception(err, 'Could not get cookies from response');
201
- }
202
- }
203
- /**
204
- * Saves an array with cookie objects to be used with the session.
205
- * The objects should be in the format that
206
- * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
207
- * but you can also use this function to set cookies manually:
208
- *
209
- * ```
210
- * [
211
- * { name: 'cookie1', value: 'my-cookie' },
212
- * { name: 'cookie2', value: 'your-cookie' }
213
- * ]
214
- * ```
215
- */
216
- setCookies(cookies, url) {
217
- const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
218
- this._setCookies(normalizedCookies, url);
219
- }
220
- /**
221
- * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
222
- * @param url website url. Only cookies stored for this url will be returned
223
- */
224
- getCookies(url) {
225
- const cookies = this.cookieJar.getCookiesSync(url);
226
- return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
227
- }
228
180
  /**
229
181
  * Returns cookies saved with the session in the typical
230
182
  * key1=value1; key2=value2 format, ready to be used in
@@ -238,25 +190,11 @@ export class Session {
238
190
  * Sets a cookie within this session for the specific URL.
239
191
  */
240
192
  setCookie(rawCookie, url) {
241
- this.cookieJar.setCookieSync(rawCookie, url);
242
- }
243
- /**
244
- * Sets cookies.
245
- */
246
- _setCookies(cookies, url) {
247
- const errorMessages = [];
248
- for (const cookie of cookies) {
249
- try {
250
- this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
251
- }
252
- catch (e) {
253
- const err = e;
254
- errorMessages.push(err.message);
255
- }
193
+ try {
194
+ this.cookieJar.setCookieSync(rawCookie, url);
256
195
  }
257
- // if invalid cookies are provided just log the exception. No need to retry the request automatically.
258
- if (errorMessages.length) {
259
- this.log.debug('Could not set cookies.', { errorMessages });
196
+ catch (e) {
197
+ this.log.warning('Could not set cookie.', { url, error: e.message });
260
198
  }
261
199
  }
262
200
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAGxD,OAAO,EACH,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,8BAA8B,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA2EpD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACZ,UAAU,CAAS;IAC3B,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAA0C;IACrD,WAAW,CAAS;IACpB,UAAU,CAAY;IACtB,GAAG,CAAM;IAEjB,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAC/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,UAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAkB;QACzC,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9E,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAsB;QACzC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;QACnG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
1
+ {"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,8BAA8B,EAAE,MAAM,oBAAoB,CAAC;AAEpE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAgEvD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACpB,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,QAAQ,GAAG,KAAK,CAAC;IACjB,UAAU,CAAa;IACvB,UAAU,CAAY;IACtB,GAAG,CAAgB;IAE3B,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,YAAY,UAA0B,EAAE;QACpC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC5B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,SAAS,GAAG,SAAS,EACrB,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,KAAK,EACf,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,GACnC,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACtG,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,QAAQ;SACzB,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
@@ -1,24 +1,30 @@
1
- import { EventEmitter } from 'node:events';
2
- import type { Log } from '@apify/log';
3
- import { Configuration } from '../configuration.js';
1
+ import type { ISessionPool } from '@crawlee/types';
4
2
  import type { PersistenceOptions } from '../crawlers/statistics.js';
5
3
  import type { EventManager } from '../events/event_manager.js';
4
+ import type { CrawleeLogger } from '../log.js';
6
5
  import { KeyValueStore } from '../storages/key_value_store.js';
7
6
  import type { SessionOptions } from './session.js';
8
7
  import { Session } from './session.js';
8
+ declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
9
+ export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
9
10
  /**
10
11
  * Factory user-function which creates customized {@link Session} instances.
11
12
  */
12
13
  export interface CreateSession {
13
14
  /**
14
- * @param sessionPool Pool requesting the new session.
15
- * @param options
15
+ * @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
16
16
  */
17
- (sessionPool: SessionPool, options?: {
17
+ (options?: {
18
18
  sessionOptions?: SessionOptions;
19
19
  }): Session | Promise<Session>;
20
20
  }
21
21
  export interface SessionPoolOptions {
22
+ /**
23
+ * Unique identifier for this session pool instance. Used to generate a unique
24
+ * persistence key when `persistStateKey` is not provided.
25
+ * If not provided, an auto-incrementing ID is used.
26
+ */
27
+ id?: string | number;
22
28
  /**
23
29
  * Maximum size of the pool. Indicates how many sessions are rotated.
24
30
  * @default 1000
@@ -29,24 +35,26 @@ export interface SessionPoolOptions {
29
35
  /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
30
36
  persistStateKeyValueStoreId?: string;
31
37
  /**
32
- * Session pool persists it's state under this key in Key value store.
33
- * @default SESSION_POOL_STATE
38
+ * Session pool persists its state under this key in Key value store.
39
+ * @default SDK_SESSION_POOL_STATE_{id}
34
40
  */
35
41
  persistStateKey?: string;
36
42
  /**
37
- * Custom function that should return `Session` instance.
38
- * Any error thrown from this function will terminate the process.
39
- * Function receives `SessionPool` instance as a parameter
43
+ * Custom function that should return a `Session` instance, or a promise resolving to such instance.
44
+ * Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
45
+ * already merged from the pool-wide defaults and the per-call overrides.
40
46
  */
41
47
  createSessionFunction?: CreateSession;
42
48
  /**
43
- * Specifies which response status codes are considered as blocked.
44
- * Session connected to such request will be marked as retired.
45
- * @default [401, 403, 429]
49
+ * Strategy for picking sessions from the pool.
50
+ * - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
51
+ * - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
52
+ * - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
53
+ * @default 'random'
46
54
  */
47
- blockedStatusCodes?: number[];
55
+ sessionReuseStrategy?: SessionReuseStrategy;
48
56
  /** @internal */
49
- log?: Log;
57
+ log?: CrawleeLogger;
50
58
  /**
51
59
  * Control how and when to persist the state of the session pool.
52
60
  */
@@ -58,20 +66,8 @@ export interface SessionPoolOptions {
58
66
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
59
67
  * Learn more in the {@doclink guides/session-management | Session management guide}.
60
68
  *
61
- * You can create one by calling the {@link SessionPool.open} function.
62
- *
63
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
64
- * performance with just 2 lines of code.
65
- *
66
- * **Example usage:**
67
- *
68
- * ```javascript
69
- * const crawler = new CheerioCrawler({
70
- * useSessionPool: true,
71
- * persistCookiesPerSession: true,
72
- * // ...
73
- * })
74
- * ```
69
+ * Session pool is already integrated into crawlers and is always active.
70
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
75
71
  *
76
72
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
77
73
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -81,7 +77,7 @@ export interface SessionPoolOptions {
81
77
  * **Advanced usage:**
82
78
  *
83
79
  * ```javascript
84
- * const sessionPool = await SessionPool.open({
80
+ * const sessionPool = new SessionPool({
85
81
  * maxPoolSize: 25,
86
82
  * sessionOptions:{
87
83
  * maxAgeSecs: 10,
@@ -117,40 +113,40 @@ export interface SessionPoolOptions {
117
113
  *
118
114
  * @category Scaling
119
115
  */
120
- export declare class SessionPool extends EventEmitter {
121
- readonly config: Configuration;
122
- protected log: Log;
116
+ export declare class SessionPool implements ISessionPool {
117
+ private static nextId;
118
+ readonly id: string;
119
+ protected log: CrawleeLogger;
123
120
  protected maxPoolSize: number;
124
121
  protected createSessionFunction: CreateSession;
125
- protected keyValueStore: KeyValueStore;
122
+ protected keyValueStore?: KeyValueStore;
126
123
  protected sessions: Session[];
127
124
  protected sessionMap: Map<string, Session>;
128
125
  protected sessionOptions: SessionOptions;
129
126
  protected persistStateKeyValueStoreId?: string;
130
127
  protected persistStateKey: string;
131
- protected _listener: () => Promise<void>;
128
+ protected _listener?: () => Promise<void>;
132
129
  protected events: EventManager;
133
- protected readonly blockedStatusCodes: number[];
134
130
  protected persistenceOptions: PersistenceOptions;
135
- protected isInitialized: boolean;
131
+ protected sessionReuseStrategy: SessionReuseStrategy;
132
+ private initPromise?;
136
133
  private queue;
137
- /**
138
- * @internal
139
- */
140
- constructor(options?: SessionPoolOptions, config?: Configuration);
134
+ private roundRobinIndex;
135
+ constructor(options?: SessionPoolOptions);
141
136
  /**
142
137
  * Gets count of usable sessions in the pool.
143
138
  */
144
- get usableSessionsCount(): number;
139
+ usableSessionsCount(): Promise<number>;
145
140
  /**
146
141
  * Gets count of retired sessions in the pool.
147
142
  */
148
- get retiredSessionsCount(): number;
143
+ retiredSessionsCount(): Promise<number>;
149
144
  /**
150
145
  * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
151
- * It is called automatically by the {@link SessionPool.open} function.
146
+ * Called automatically on first use of any public method.
152
147
  */
153
- initialize(): Promise<void>;
148
+ protected ensureInitialized(): Promise<void>;
149
+ private setupPool;
154
150
  /**
155
151
  * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
156
152
  * but this allows you to add more sessions once the max pool size is reached.
@@ -158,17 +154,21 @@ export declare class SessionPool extends EventEmitter {
158
154
  * @param [options] The configuration options for the session being added to the session pool.
159
155
  */
160
156
  addSession(options?: Session | SessionOptions): Promise<void>;
157
+ /**
158
+ * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
159
+ * but this allows you to add more sessions once the max pool size is reached.
160
+ * This also allows you to add session with overridden session options (e.g. with specific session id).
161
+ * @param [options] The configuration options for the session being added to the session pool.
162
+ */
163
+ newSession(sessionOptions?: SessionOptions): Promise<Session>;
161
164
  /**
162
165
  * Gets session.
163
166
  * If there is space for new session, it creates and returns new session.
164
167
  * If the session pool is full, it picks a session from the pool,
165
168
  * If the picked session is usable it is returned, otherwise it creates and returns a new one.
169
+ * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
166
170
  */
167
- getSession(): Promise<Session>;
168
- /**
169
- * Gets session based on the provided session id or `undefined.
170
- */
171
- getSession(sessionId: string): Promise<Session>;
171
+ getSession(sessionId?: string): Promise<Session | undefined>;
172
172
  /**
173
173
  * @param options - Override the persistence options provided in the constructor
174
174
  */
@@ -177,12 +177,12 @@ export declare class SessionPool extends EventEmitter {
177
177
  * Returns an object representing the internal state of the `SessionPool` instance.
178
178
  * Note that the object's fields can change in future releases.
179
179
  */
180
- getState(): {
180
+ getState(): Promise<{
181
181
  usableSessionsCount: number;
182
182
  retiredSessionsCount: number;
183
183
  // @ts-ignore optional peer dependency or compatibility with es2022
184
- sessions: import("./session.js").SessionState[];
185
- };
184
+ sessions: import("@crawlee/types").SessionState[];
185
+ }>;
186
186
  /**
187
187
  * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
188
188
  * The state is persisted automatically in regular intervals.
@@ -194,10 +194,6 @@ export declare class SessionPool extends EventEmitter {
194
194
  * This function should be called after you are done with using the `SessionPool` instance.
195
195
  */
196
196
  teardown(): Promise<void>;
197
- /**
198
- * SessionPool should not work before initialization.
199
- */
200
- protected _throwIfNotInitialized(): void;
201
197
  /**
202
198
  * Removes retired `Session` instances from `SessionPool`.
203
199
  */
@@ -213,14 +209,18 @@ export declare class SessionPool extends EventEmitter {
213
209
  protected _getRandomIndex(): number;
214
210
  /**
215
211
  * Creates new session without any extra behavior.
216
- * @param sessionPool
217
212
  * @param [options]
218
213
  * @param [options.sessionOptions] The configuration options for the session being created.
219
214
  * @returns New session.
220
215
  */
221
- protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
216
+ protected _defaultCreateSessionFunction(options?: {
222
217
  sessionOptions?: SessionOptions;
223
- }): Session;
218
+ }): Promise<Session>;
219
+ /**
220
+ * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
221
+ * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
222
+ */
223
+ private _invokeCreateSessionFunction;
224
224
  /**
225
225
  * Creates new session and adds it to the pool.
226
226
  * @returns Newly created `Session` instance.
@@ -231,21 +231,15 @@ export declare class SessionPool extends EventEmitter {
231
231
  */
232
232
  protected _hasSpaceForSession(): boolean;
233
233
  /**
234
- * Picks random session from the `SessionPool`.
235
- * @returns Picked `Session`.
234
+ * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
235
+ * Returns `undefined` when no session should be reused and a new one should be created instead.
236
236
  */
237
- protected _pickSession(): Session;
237
+ protected _pickSession(): Session | undefined;
238
238
  /**
239
239
  * Potentially loads `SessionPool`.
240
240
  * If the state was persisted it loads the `SessionPool` from the persisted state.
241
241
  */
242
242
  protected _maybeLoadSessionPool(): Promise<void>;
243
- /**
244
- * Opens a SessionPool and returns a promise resolving to an instance
245
- * of the {@link SessionPool} class that is already initialized.
246
- *
247
- * For more details and code examples, see the {@link SessionPool} class.
248
- */
249
- static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
250
243
  }
244
+ export {};
251
245
  //# sourceMappingURL=session_pool.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAuBrC,QAAQ,CAAC,MAAM;IAtBnB,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBAEC,OAAO,GAAE,kBAAuB,EACvB,MAAM,gBAAkC;IAsDrD;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO;IAUV;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC;CAKhG"}
1
+ {"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAI/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,QAAA,MAAM,wBAAwB,yDAA0D,CAAC;AACzF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,CAAC,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/E;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAE5C,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,qBAAa,WAAY,YAAW,YAAY;IAC5C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAK;IAE1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,CAAC;IAErD,OAAO,CAAC,WAAW,CAAC,CAAgB;IACpC,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,GAAE,kBAAuB;IAqD5C;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK7C;;;OAGG;cACa,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAOpC,SAAS;IAyBvB;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;;OAMG;IACG,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;IAyBlE;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAS7C;;;OAGG;IACG,QAAQ;;;;;IASd;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;cACa,6BAA6B,CAAC,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAOlH;;;OAGG;YACW,4BAA4B;IAK1C;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO,GAAG,SAAS;IAmB7C;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;CAuBzD"}