@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.140

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/README.md +14 -14
  2. package/autoscaling/autoscaled_pool.d.ts +67 -172
  3. package/autoscaling/autoscaled_pool.js +182 -329
  4. package/autoscaling/concurrency_system.d.ts +269 -0
  5. package/autoscaling/concurrency_system.js +365 -0
  6. package/autoscaling/cpu_load_signal.d.ts +43 -0
  7. package/autoscaling/cpu_load_signal.js +47 -0
  8. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  9. package/autoscaling/event_loop_load_signal.js +60 -0
  10. package/autoscaling/index.d.ts +6 -1
  11. package/autoscaling/index.js +6 -1
  12. package/autoscaling/load_signal.d.ts +100 -0
  13. package/autoscaling/load_signal.js +105 -0
  14. package/autoscaling/memory_load_signal.d.ts +47 -0
  15. package/autoscaling/memory_load_signal.js +106 -0
  16. package/autoscaling/snapshotter.d.ts +58 -163
  17. package/autoscaling/snapshotter.js +45 -263
  18. package/autoscaling/storage_backend_load_signal.d.ts +56 -0
  19. package/autoscaling/storage_backend_load_signal.js +73 -0
  20. package/autoscaling/system_status.d.ts +67 -89
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +3 -2
  29. package/cookie_utils.js +18 -7
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +90 -126
  33. package/crawlers/crawler_commons.js +1 -108
  34. package/crawlers/error_snapshotter.d.ts +2 -5
  35. package/crawlers/error_snapshotter.js +7 -8
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +187 -63
  43. package/crawlers/statistics.js +354 -164
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +60 -153
  47. package/enqueue_links/enqueue_links.js +38 -229
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +49 -30
  51. package/enqueue_links/shared.js +94 -68
  52. package/errors.d.ts +72 -4
  53. package/errors.js +89 -5
  54. package/events/event_manager.d.ts +35 -9
  55. package/events/event_manager.js +12 -13
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +39 -13
  60. package/http.d.ts +9 -0
  61. package/http.js +28 -0
  62. package/index.d.ts +7 -5
  63. package/index.js +8 -4
  64. package/iterables.d.ts +79 -0
  65. package/iterables.js +134 -0
  66. package/log.d.ts +78 -1
  67. package/log.js +106 -1
  68. package/memory-storage/consts.d.ts +4 -0
  69. package/memory-storage/consts.js +4 -0
  70. package/memory-storage/index.d.ts +1 -0
  71. package/memory-storage/index.js +1 -0
  72. package/memory-storage/memory-storage.d.ts +44 -0
  73. package/memory-storage/memory-storage.js +160 -0
  74. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  75. package/memory-storage/resource-clients/common/base-client.js +6 -0
  76. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  77. package/memory-storage/resource-clients/dataset.js +106 -0
  78. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  79. package/memory-storage/resource-clients/key-value-store.js +199 -0
  80. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  81. package/memory-storage/resource-clients/request-queue.js +407 -0
  82. package/memory-storage/utils.d.ts +16 -0
  83. package/memory-storage/utils.js +41 -0
  84. package/owned_or_injected.d.ts +58 -0
  85. package/owned_or_injected.js +98 -0
  86. package/package.json +13 -12
  87. package/proxy_configuration.d.ts +22 -128
  88. package/proxy_configuration.js +32 -144
  89. package/recoverable_state.d.ts +83 -51
  90. package/recoverable_state.js +163 -72
  91. package/request.d.ts +57 -16
  92. package/request.js +130 -69
  93. package/router.d.ts +193 -21
  94. package/router.js +188 -43
  95. package/serialization.d.ts +0 -1
  96. package/serialization.js +15 -15
  97. package/service_locator.d.ts +165 -0
  98. package/service_locator.js +253 -0
  99. package/session_pool/consts.d.ts +1 -2
  100. package/session_pool/consts.js +1 -2
  101. package/session_pool/errors.d.ts +0 -1
  102. package/session_pool/errors.js +0 -1
  103. package/session_pool/fingerprint.d.ts +9 -0
  104. package/session_pool/fingerprint.js +30 -0
  105. package/session_pool/index.d.ts +0 -2
  106. package/session_pool/index.js +0 -2
  107. package/session_pool/session.d.ts +35 -88
  108. package/session_pool/session.js +101 -159
  109. package/session_pool/session_pool.d.ts +74 -91
  110. package/session_pool/session_pool.js +175 -165
  111. package/storages/batched_adds.d.ts +37 -0
  112. package/storages/batched_adds.js +73 -0
  113. package/storages/dataset.d.ts +109 -56
  114. package/storages/dataset.js +283 -149
  115. package/storages/index.d.ts +9 -9
  116. package/storages/index.js +7 -9
  117. package/storages/key_value_store.d.ts +183 -48
  118. package/storages/key_value_store.js +445 -171
  119. package/storages/key_value_store_codec.d.ts +32 -0
  120. package/storages/key_value_store_codec.js +113 -0
  121. package/storages/request_dedup_cache.d.ts +22 -0
  122. package/storages/request_dedup_cache.js +48 -0
  123. package/storages/request_list.d.ts +52 -109
  124. package/storages/request_list.js +183 -152
  125. package/storages/request_loader.d.ts +101 -0
  126. package/storages/request_loader.js +1 -0
  127. package/storages/request_manager.d.ts +33 -0
  128. package/storages/request_manager.js +1 -0
  129. package/storages/request_manager_tandem.d.ts +48 -19
  130. package/storages/request_manager_tandem.js +118 -45
  131. package/storages/request_queue.d.ts +290 -47
  132. package/storages/request_queue.js +762 -216
  133. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +45 -89
  134. package/storages/sitemap_request_loader.js +438 -0
  135. package/storages/storage_instance_manager.d.ts +87 -0
  136. package/storages/storage_instance_manager.js +256 -0
  137. package/storages/storage_stats.d.ts +48 -0
  138. package/storages/storage_stats.js +29 -0
  139. package/storages/throttling_request_manager.d.ts +239 -0
  140. package/storages/throttling_request_manager.js +646 -0
  141. package/storages/transaction.d.ts +252 -0
  142. package/storages/transaction.js +251 -0
  143. package/storages/utils.d.ts +59 -11
  144. package/storages/utils.js +75 -15
  145. package/system-info/cpu-info.d.ts +67 -0
  146. package/system-info/cpu-info.js +216 -0
  147. package/system-info/memory-info.d.ts +31 -0
  148. package/system-info/memory-info.js +115 -0
  149. package/system-info/ps-tree.d.ts +17 -0
  150. package/system-info/ps-tree.js +144 -0
  151. package/system-info/runtime.d.ts +14 -0
  152. package/system-info/runtime.js +80 -0
  153. package/typedefs.d.ts +0 -6
  154. package/typedefs.js +0 -1
  155. package/url.d.ts +9 -0
  156. package/url.js +11 -0
  157. package/validators.d.ts +22 -18
  158. package/validators.js +13 -18
  159. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  160. package/autoscaling/autoscaled_pool.js.map +0 -1
  161. package/autoscaling/index.d.ts.map +0 -1
  162. package/autoscaling/index.js.map +0 -1
  163. package/autoscaling/snapshotter.d.ts.map +0 -1
  164. package/autoscaling/snapshotter.js.map +0 -1
  165. package/autoscaling/system_status.d.ts.map +0 -1
  166. package/autoscaling/system_status.js.map +0 -1
  167. package/configuration.d.ts.map +0 -1
  168. package/configuration.js.map +0 -1
  169. package/cookie_utils.d.ts.map +0 -1
  170. package/cookie_utils.js.map +0 -1
  171. package/crawlers/context_pipeline.d.ts.map +0 -1
  172. package/crawlers/context_pipeline.js.map +0 -1
  173. package/crawlers/crawler_commons.d.ts.map +0 -1
  174. package/crawlers/crawler_commons.js.map +0 -1
  175. package/crawlers/crawler_utils.d.ts +0 -10
  176. package/crawlers/crawler_utils.d.ts.map +0 -1
  177. package/crawlers/crawler_utils.js +0 -12
  178. package/crawlers/crawler_utils.js.map +0 -1
  179. package/crawlers/error_snapshotter.d.ts.map +0 -1
  180. package/crawlers/error_snapshotter.js.map +0 -1
  181. package/crawlers/error_tracker.d.ts.map +0 -1
  182. package/crawlers/error_tracker.js.map +0 -1
  183. package/crawlers/index.d.ts.map +0 -1
  184. package/crawlers/index.js.map +0 -1
  185. package/crawlers/internals/types.d.ts.map +0 -1
  186. package/crawlers/internals/types.js.map +0 -1
  187. package/crawlers/statistics.d.ts.map +0 -1
  188. package/crawlers/statistics.js.map +0 -1
  189. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  190. package/enqueue_links/enqueue_links.js.map +0 -1
  191. package/enqueue_links/index.d.ts.map +0 -1
  192. package/enqueue_links/index.js.map +0 -1
  193. package/enqueue_links/shared.d.ts.map +0 -1
  194. package/enqueue_links/shared.js.map +0 -1
  195. package/errors.d.ts.map +0 -1
  196. package/errors.js.map +0 -1
  197. package/events/event_manager.d.ts.map +0 -1
  198. package/events/event_manager.js.map +0 -1
  199. package/events/index.d.ts.map +0 -1
  200. package/events/index.js.map +0 -1
  201. package/events/local_event_manager.d.ts.map +0 -1
  202. package/events/local_event_manager.js.map +0 -1
  203. package/http_clients/base-http-client.d.ts +0 -140
  204. package/http_clients/base-http-client.d.ts.map +0 -1
  205. package/http_clients/base-http-client.js +0 -40
  206. package/http_clients/base-http-client.js.map +0 -1
  207. package/http_clients/form-data-like.d.ts +0 -67
  208. package/http_clients/form-data-like.d.ts.map +0 -1
  209. package/http_clients/form-data-like.js +0 -5
  210. package/http_clients/form-data-like.js.map +0 -1
  211. package/http_clients/got-scraping-http-client.d.ts +0 -20
  212. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  213. package/http_clients/got-scraping-http-client.js +0 -85
  214. package/http_clients/got-scraping-http-client.js.map +0 -1
  215. package/http_clients/index.d.ts +0 -3
  216. package/http_clients/index.d.ts.map +0 -1
  217. package/http_clients/index.js +0 -3
  218. package/http_clients/index.js.map +0 -1
  219. package/index.d.ts.map +0 -1
  220. package/index.js.map +0 -1
  221. package/log.d.ts.map +0 -1
  222. package/log.js.map +0 -1
  223. package/proxy_configuration.d.ts.map +0 -1
  224. package/proxy_configuration.js.map +0 -1
  225. package/recoverable_state.d.ts.map +0 -1
  226. package/recoverable_state.js.map +0 -1
  227. package/request.d.ts.map +0 -1
  228. package/request.js.map +0 -1
  229. package/router.d.ts.map +0 -1
  230. package/router.js.map +0 -1
  231. package/serialization.d.ts.map +0 -1
  232. package/serialization.js.map +0 -1
  233. package/session_pool/consts.d.ts.map +0 -1
  234. package/session_pool/consts.js.map +0 -1
  235. package/session_pool/errors.d.ts.map +0 -1
  236. package/session_pool/errors.js.map +0 -1
  237. package/session_pool/events.d.ts +0 -3
  238. package/session_pool/events.d.ts.map +0 -1
  239. package/session_pool/events.js +0 -3
  240. package/session_pool/events.js.map +0 -1
  241. package/session_pool/index.d.ts.map +0 -1
  242. package/session_pool/index.js.map +0 -1
  243. package/session_pool/session.d.ts.map +0 -1
  244. package/session_pool/session.js.map +0 -1
  245. package/session_pool/session_pool.d.ts.map +0 -1
  246. package/session_pool/session_pool.js.map +0 -1
  247. package/storages/access_checking.d.ts +0 -13
  248. package/storages/access_checking.d.ts.map +0 -1
  249. package/storages/access_checking.js +0 -14
  250. package/storages/access_checking.js.map +0 -1
  251. package/storages/dataset.d.ts.map +0 -1
  252. package/storages/dataset.js.map +0 -1
  253. package/storages/index.d.ts.map +0 -1
  254. package/storages/index.js.map +0 -1
  255. package/storages/key_value_store.d.ts.map +0 -1
  256. package/storages/key_value_store.js.map +0 -1
  257. package/storages/request_list.d.ts.map +0 -1
  258. package/storages/request_list.js.map +0 -1
  259. package/storages/request_list_adapter.d.ts +0 -58
  260. package/storages/request_list_adapter.d.ts.map +0 -1
  261. package/storages/request_list_adapter.js +0 -81
  262. package/storages/request_list_adapter.js.map +0 -1
  263. package/storages/request_manager_tandem.d.ts.map +0 -1
  264. package/storages/request_manager_tandem.js.map +0 -1
  265. package/storages/request_provider.d.ts +0 -371
  266. package/storages/request_provider.d.ts.map +0 -1
  267. package/storages/request_provider.js +0 -585
  268. package/storages/request_provider.js.map +0 -1
  269. package/storages/request_queue.d.ts.map +0 -1
  270. package/storages/request_queue.js.map +0 -1
  271. package/storages/request_queue_v2.d.ts +0 -87
  272. package/storages/request_queue_v2.d.ts.map +0 -1
  273. package/storages/request_queue_v2.js +0 -438
  274. package/storages/request_queue_v2.js.map +0 -1
  275. package/storages/sitemap_request_list.d.ts.map +0 -1
  276. package/storages/sitemap_request_list.js +0 -430
  277. package/storages/sitemap_request_list.js.map +0 -1
  278. package/storages/storage_manager.d.ts +0 -58
  279. package/storages/storage_manager.d.ts.map +0 -1
  280. package/storages/storage_manager.js +0 -105
  281. package/storages/storage_manager.js.map +0 -1
  282. package/storages/utils.d.ts.map +0 -1
  283. package/storages/utils.js.map +0 -1
  284. package/typedefs.d.ts.map +0 -1
  285. package/typedefs.js.map +0 -1
  286. package/validators.d.ts.map +0 -1
  287. package/validators.js.map +0 -1
@@ -1,10 +1,28 @@
1
- import { EventEmitter } from 'node:events';
2
- import ow from 'ow';
3
1
  import { CookieJar } from 'tough-cookie';
2
+ import { z } from 'zod';
4
3
  import { cryptoRandomObjectId } from '@apify/utilities';
5
- import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
6
- import { log as defaultLog } from '../log.js';
7
- import { EVENT_SESSION_RETIRED } from './events.js';
4
+ import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
5
+ import { serviceLocator } from '../service_locator.js';
6
+ import { parseArgument, schemas, validators } from '../validators.js';
7
+ // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
8
+ // copy), so class instances like cookie jars and loggers keep their prototype.
9
+ const sessionOptionsSchema = z.strictObject({
10
+ id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
11
+ cookieJar: schemas.anyObject.default(() => new CookieJar()),
12
+ proxyInfo: schemas.anyObject.optional(),
13
+ maxAgeSecs: schemas.anyNumber.default(3000),
14
+ userData: schemas.anyObject.default(() => ({})),
15
+ maxErrorScore: schemas.anyNumber.default(3),
16
+ errorScoreDecrement: schemas.anyNumber.default(0.5),
17
+ createdAt: z.date().default(() => new Date()),
18
+ expiresAt: z.date().optional(),
19
+ usageCount: schemas.anyNumber.default(0),
20
+ errorScore: schemas.anyNumber.default(0),
21
+ maxUsageCount: schemas.anyNumber.default(50),
22
+ retired: z.boolean().default(false),
23
+ log: validators.logger.default(() => serviceLocator.getLogger()),
24
+ fingerprint: schemas.anyObject.optional(),
25
+ });
8
26
  /**
9
27
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
10
28
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -13,83 +31,79 @@ import { EVENT_SESSION_RETIRED } from './events.js';
13
31
  */
14
32
  export class Session {
15
33
  id;
16
- maxAgeSecs;
17
34
  userData;
18
- _maxErrorScore;
19
- _errorScoreDecrement;
20
- _createdAt;
21
- _expiresAt;
22
- _usageCount;
23
- _maxUsageCount;
24
- sessionPool;
25
- _errorScore;
26
- _proxyInfo;
27
- _cookieJar;
28
- log;
35
+ #maxErrorScore;
36
+ #errorScoreDecrement;
37
+ #createdAt;
38
+ #expiresAt;
39
+ #usageCount;
40
+ #maxUsageCount;
41
+ #errorScore;
42
+ #retired = false;
43
+ #proxyInfo;
44
+ #cookieJar;
45
+ #fingerprint;
46
+ #log;
29
47
  get errorScore() {
30
- return this._errorScore;
48
+ return this.#errorScore;
31
49
  }
32
50
  get usageCount() {
33
- return this._usageCount;
51
+ return this.#usageCount;
34
52
  }
35
53
  get maxErrorScore() {
36
- return this._maxErrorScore;
54
+ return this.#maxErrorScore;
37
55
  }
38
56
  get errorScoreDecrement() {
39
- return this._errorScoreDecrement;
57
+ return this.#errorScoreDecrement;
40
58
  }
41
59
  get expiresAt() {
42
- return this._expiresAt;
60
+ return this.#expiresAt;
43
61
  }
44
62
  get createdAt() {
45
- return this._createdAt;
63
+ return this.#createdAt;
46
64
  }
47
65
  get maxUsageCount() {
48
- return this._maxUsageCount;
66
+ return this.#maxUsageCount;
49
67
  }
50
68
  get cookieJar() {
51
- return this._cookieJar;
69
+ return this.#cookieJar;
52
70
  }
53
71
  get proxyInfo() {
54
- return this._proxyInfo;
72
+ return this.#proxyInfo;
73
+ }
74
+ get fingerprint() {
75
+ return this.#fingerprint;
76
+ }
77
+ set fingerprint(fingerprint) {
78
+ this.#fingerprint = fingerprint;
79
+ }
80
+ /**
81
+ * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
82
+ * a retired session is never picked by the pool and cannot be revived via `markGood()`.
83
+ */
84
+ get retired() {
85
+ return this.#retired;
55
86
  }
56
87
  /**
57
88
  * Session configuration.
58
89
  */
59
- constructor(options) {
60
- ow(options, ow.object.exactShape({
61
- sessionPool: ow.object.instanceOf(EventEmitter),
62
- id: ow.optional.string,
63
- cookieJar: ow.optional.object,
64
- proxyInfo: ow.optional.object,
65
- maxAgeSecs: ow.optional.number,
66
- userData: ow.optional.object,
67
- maxErrorScore: ow.optional.number,
68
- errorScoreDecrement: ow.optional.number,
69
- createdAt: ow.optional.date,
70
- expiresAt: ow.optional.date,
71
- usageCount: ow.optional.number,
72
- errorScore: ow.optional.number,
73
- maxUsageCount: ow.optional.number,
74
- log: ow.optional.object,
75
- }));
76
- const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
77
- const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
78
- this.log = log.child({ prefix: 'Session' });
79
- this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
80
- this._proxyInfo = proxyInfo;
90
+ constructor(options = {}) {
91
+ const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
92
+ this.#log = log.child({ prefix: 'Session' });
93
+ this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
94
+ this.#proxyInfo = proxyInfo;
95
+ this.#fingerprint = fingerprint;
81
96
  this.id = id;
82
- this.maxAgeSecs = maxAgeSecs;
83
97
  this.userData = userData;
84
- this._maxErrorScore = maxErrorScore;
85
- this._errorScoreDecrement = errorScoreDecrement;
98
+ this.#maxErrorScore = maxErrorScore;
99
+ this.#errorScoreDecrement = errorScoreDecrement;
86
100
  // Internal
87
- this._expiresAt = expiresAt;
88
- this._createdAt = createdAt;
89
- this._usageCount = usageCount; // indicates how many times the session has been used
90
- this._errorScore = errorScore; // indicates number of markBaded request with the session
91
- this._maxUsageCount = maxUsageCount;
92
- this.sessionPool = sessionPool;
101
+ this.#expiresAt = expiresAt;
102
+ this.#createdAt = createdAt;
103
+ this.#usageCount = usageCount; // indicates how many times the session has been used
104
+ this.#errorScore = errorScore; // indicates number of markBaded request with the session
105
+ this.#maxUsageCount = maxUsageCount;
106
+ this.#retired = retired;
93
107
  }
94
108
  /**
95
109
  * Indicates whether the session is blocked.
@@ -115,21 +129,21 @@ export class Session {
115
129
  }
116
130
  /**
117
131
  * Indicates whether the session can be used for next requests.
118
- * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
132
+ * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
119
133
  */
120
134
  isUsable() {
121
- return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
135
+ return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
122
136
  }
123
137
  /**
124
138
  * This method should be called after a successful session usage.
125
139
  * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
126
140
  */
127
141
  markGood() {
128
- this._usageCount += 1;
129
- if (this._errorScore > 0) {
130
- this._errorScore -= this._errorScoreDecrement;
142
+ this.#usageCount += 1;
143
+ if (this.#errorScore > 0) {
144
+ this.#errorScore -= this.#errorScoreDecrement;
131
145
  }
132
- this._maybeSelfRetire();
146
+ this.maybeSelfRetire();
133
147
  }
134
148
  /**
135
149
  * Gets session state for persistence in KeyValueStore.
@@ -139,8 +153,9 @@ export class Session {
139
153
  return {
140
154
  id: this.id,
141
155
  cookieJar: this.cookieJar.toJSON(),
142
- proxyInfo: this._proxyInfo,
156
+ proxyInfo: this.#proxyInfo,
143
157
  userData: this.userData,
158
+ fingerprint: this.#fingerprint,
144
159
  maxErrorScore: this.maxErrorScore,
145
160
  errorScoreDecrement: this.errorScoreDecrement,
146
161
  expiresAt: this.expiresAt.toISOString(),
@@ -148,89 +163,31 @@ export class Session {
148
163
  usageCount: this.usageCount,
149
164
  maxUsageCount: this.maxUsageCount,
150
165
  errorScore: this.errorScore,
166
+ retired: this.#retired,
151
167
  };
152
168
  }
153
169
  /**
154
- * Marks session as blocked and emits event on the `SessionPool`
155
- * This method should be used if the session usage was unsuccessful
156
- * and you are sure that it is because of the session configuration and not any external matters.
157
- * For example when server returns 403 status code.
158
- * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
170
+ * Permanently retires the session `isUsable()` will return `false` from here on,
171
+ * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
172
+ *
173
+ * Use this when you're confident the session itself is the problem (e.g. a `403` response).
174
+ * For transient external failures (such as `5XX` responses), use `markBad()` instead.
159
175
  */
160
176
  retire() {
161
- // mark it as an invalid by increasing the error score count.
162
- this._errorScore += this._maxErrorScore;
163
- this._usageCount += 1;
164
- // emit event so we can retire browser in puppeteer pool
165
- this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
177
+ if (this.#retired)
178
+ return;
179
+ this.#errorScore += this.#maxErrorScore;
180
+ this.#usageCount += 1;
181
+ this.#retired = true;
166
182
  }
167
183
  /**
168
184
  * Increases usage and error count.
169
185
  * Should be used when the session has been used unsuccessfully. For example because of timeouts.
170
186
  */
171
187
  markBad() {
172
- this._errorScore += 1;
173
- this._usageCount += 1;
174
- this._maybeSelfRetire();
175
- }
176
- /**
177
- * With certain status codes: `401`, `403` or `429` we can be certain
178
- * that the target website is blocking us. This function helps to do this conveniently
179
- * by retiring the session when such code is received. Optionally, the default status
180
- * codes can be extended in the second parameter.
181
- * @param statusCode HTTP status code.
182
- * @returns Whether the session was retired.
183
- */
184
- retireOnBlockedStatusCodes(statusCode) {
185
- // eslint-disable-next-line dot-notation -- accessing private property
186
- const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
187
- if (isBlocked) {
188
- this.retire();
189
- }
190
- return isBlocked;
191
- }
192
- /**
193
- * Saves cookies from an HTTP response to be used with the session.
194
- * It expects an object with a `headers` property that's either an `Object`
195
- * (typical Node.js responses) or a `Function` (Puppeteer Response).
196
- *
197
- * It then parses and saves the cookies from the `set-cookie` header, if available.
198
- */
199
- setCookiesFromResponse(response) {
200
- try {
201
- const cookies = getCookiesFromResponse(response).filter((c) => c);
202
- this._setCookies(cookies, response.url);
203
- }
204
- catch (e) {
205
- const err = e;
206
- // if invalid Cookie header is provided just log the exception.
207
- this.log.exception(err, 'Could not get cookies from response');
208
- }
209
- }
210
- /**
211
- * Saves an array with cookie objects to be used with the session.
212
- * The objects should be in the format that
213
- * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
214
- * but you can also use this function to set cookies manually:
215
- *
216
- * ```
217
- * [
218
- * { name: 'cookie1', value: 'my-cookie' },
219
- * { name: 'cookie2', value: 'your-cookie' }
220
- * ]
221
- * ```
222
- */
223
- setCookies(cookies, url) {
224
- const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
225
- this._setCookies(normalizedCookies, url);
226
- }
227
- /**
228
- * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
229
- * @param url website url. Only cookies stored for this url will be returned
230
- */
231
- getCookies(url) {
232
- const cookies = this.cookieJar.getCookiesSync(url);
233
- return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
188
+ this.#errorScore += 1;
189
+ this.#usageCount += 1;
190
+ this.maybeSelfRetire();
234
191
  }
235
192
  /**
236
193
  * Returns cookies saved with the session in the typical
@@ -238,41 +195,26 @@ export class Session {
238
195
  * a cookie header or elsewhere.
239
196
  * @returns Represents `Cookie` header.
240
197
  */
241
- getCookieString(url) {
242
- return this.cookieJar.getCookieStringSync(url, {});
198
+ async getCookieString(url) {
199
+ return this.cookieJar.getCookieString(url, {});
243
200
  }
244
201
  /**
245
202
  * Sets a cookie within this session for the specific URL.
246
203
  */
247
- setCookie(rawCookie, url) {
248
- this.cookieJar.setCookieSync(rawCookie, url);
249
- }
250
- /**
251
- * Sets cookies.
252
- */
253
- _setCookies(cookies, url) {
254
- const errorMessages = [];
255
- for (const cookie of cookies) {
256
- try {
257
- this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
258
- }
259
- catch (e) {
260
- const err = e;
261
- errorMessages.push(err.message);
262
- }
204
+ async setCookie(rawCookie, url) {
205
+ try {
206
+ await this.cookieJar.setCookie(rawCookie, url);
263
207
  }
264
- // if invalid cookies are provided just log the exception. No need to retry the request automatically.
265
- if (errorMessages.length) {
266
- this.log.debug('Could not set cookies.', { errorMessages });
208
+ catch (e) {
209
+ this.#log.warning('Could not set cookie.', { url, error: e.message });
267
210
  }
268
211
  }
269
212
  /**
270
213
  * Checks if session is not usable. if it is not retires the session.
271
214
  */
272
- _maybeSelfRetire() {
215
+ maybeSelfRetire() {
273
216
  if (!this.isUsable()) {
274
217
  this.retire();
275
218
  }
276
219
  }
277
220
  }
278
- //# sourceMappingURL=session.js.map
@@ -1,24 +1,28 @@
1
- import { EventEmitter } from 'node:events';
2
- import type { Log } from '@apify/log';
3
- import { Configuration } from '../configuration.js';
1
+ import type { ISessionPool } from '@crawlee/types';
4
2
  import type { PersistenceOptions } from '../crawlers/statistics.js';
5
- import type { EventManager } from '../events/event_manager.js';
6
- import { KeyValueStore } from '../storages/key_value_store.js';
3
+ import type { CrawleeLogger } from '../log.js';
7
4
  import type { SessionOptions } from './session.js';
8
5
  import { Session } from './session.js';
6
+ declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
7
+ export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
9
8
  /**
10
9
  * Factory user-function which creates customized {@link Session} instances.
11
10
  */
12
11
  export interface CreateSession {
13
12
  /**
14
- * @param sessionPool Pool requesting the new session.
15
- * @param options
13
+ * @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
16
14
  */
17
- (sessionPool: SessionPool, options?: {
15
+ (options?: {
18
16
  sessionOptions?: SessionOptions;
19
17
  }): Session | Promise<Session>;
20
18
  }
21
19
  export interface SessionPoolOptions {
20
+ /**
21
+ * Unique identifier for this session pool instance. Used to generate a unique
22
+ * persistence key when `persistStateKey` is not provided.
23
+ * If not provided, an auto-incrementing ID is used.
24
+ */
25
+ id?: string | number;
22
26
  /**
23
27
  * Maximum size of the pool. Indicates how many sessions are rotated.
24
28
  * @default 1000
@@ -29,24 +33,26 @@ export interface SessionPoolOptions {
29
33
  /** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
30
34
  persistStateKeyValueStoreId?: string;
31
35
  /**
32
- * Session pool persists it's state under this key in Key value store.
33
- * @default SESSION_POOL_STATE
36
+ * Session pool persists its state under this key in Key value store.
37
+ * @default CRAWLEE_SESSION_POOL_STATE_{id}
34
38
  */
35
39
  persistStateKey?: string;
36
40
  /**
37
41
  * Custom function that should return a `Session` instance, or a promise resolving to such instance.
38
- * Any error thrown from this function will terminate the process.
39
- * Function receives `SessionPool` instance as a parameter
42
+ * Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
43
+ * already merged from the pool-wide defaults and the per-call overrides.
40
44
  */
41
45
  createSessionFunction?: CreateSession;
42
46
  /**
43
- * Specifies which response status codes are considered as blocked.
44
- * Session connected to such request will be marked as retired.
45
- * @default [401, 403, 429]
47
+ * Strategy for picking sessions from the pool.
48
+ * - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
49
+ * - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
50
+ * - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
51
+ * @default 'random'
46
52
  */
47
- blockedStatusCodes?: number[];
53
+ sessionReuseStrategy?: SessionReuseStrategy;
48
54
  /** @internal */
49
- log?: Log;
55
+ log?: CrawleeLogger;
50
56
  /**
51
57
  * Control how and when to persist the state of the session pool.
52
58
  */
@@ -58,20 +64,8 @@ export interface SessionPoolOptions {
58
64
  * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
59
65
  * Learn more in the {@doclink guides/session-management | Session management guide}.
60
66
  *
61
- * You can create one by calling the {@link SessionPool.open} function.
62
- *
63
- * Session pool is already integrated into crawlers, and it can significantly improve your scraper
64
- * performance with just 2 lines of code.
65
- *
66
- * **Example usage:**
67
- *
68
- * ```javascript
69
- * const crawler = new CheerioCrawler({
70
- * useSessionPool: true,
71
- * persistCookiesPerSession: true,
72
- * // ...
73
- * })
74
- * ```
67
+ * Session pool is already integrated into crawlers and is always active.
68
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
75
69
  *
76
70
  * You can configure the pool with many options. See the {@link SessionPoolOptions}.
77
71
  * Session pool is by default persisted in default {@link KeyValueStore}.
@@ -81,7 +75,7 @@ export interface SessionPoolOptions {
81
75
  * **Advanced usage:**
82
76
  *
83
77
  * ```javascript
84
- * const sessionPool = await SessionPool.open({
78
+ * const sessionPool = new SessionPool({
85
79
  * maxPoolSize: 25,
86
80
  * sessionOptions:{
87
81
  * maxAgeSecs: 10,
@@ -117,40 +111,31 @@ export interface SessionPoolOptions {
117
111
  *
118
112
  * @category Scaling
119
113
  */
120
- export declare class SessionPool extends EventEmitter {
121
- readonly config: Configuration;
122
- protected log: Log;
123
- protected maxPoolSize: number;
124
- protected createSessionFunction: CreateSession;
125
- protected keyValueStore: KeyValueStore;
126
- protected sessions: Session[];
127
- protected sessionMap: Map<string, Session>;
128
- protected sessionOptions: SessionOptions;
129
- protected persistStateKeyValueStoreId?: string;
130
- protected persistStateKey: string;
131
- protected _listener: () => Promise<void>;
132
- protected events: EventManager;
133
- protected readonly blockedStatusCodes: number[];
134
- protected persistenceOptions: PersistenceOptions;
135
- protected isInitialized: boolean;
136
- private queue;
137
- /**
138
- * @internal
139
- */
140
- constructor(options?: SessionPoolOptions, config?: Configuration);
114
+ export declare class SessionPool implements ISessionPool {
115
+ #private;
116
+ readonly id: string;
117
+ private maxPoolSize;
118
+ private createSessionFunction;
119
+ private keyValueStore?;
120
+ private sessionMap;
121
+ private sessionOptions;
122
+ private persistStateKeyValueStoreId?;
123
+ private persistStateKey;
124
+ constructor(options?: SessionPoolOptions);
141
125
  /**
142
126
  * Gets count of usable sessions in the pool.
143
127
  */
144
- get usableSessionsCount(): number;
128
+ usableSessionsCount(): Promise<number>;
145
129
  /**
146
130
  * Gets count of retired sessions in the pool.
147
131
  */
148
- get retiredSessionsCount(): number;
132
+ retiredSessionsCount(): Promise<number>;
149
133
  /**
150
134
  * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
151
- * It is called automatically by the {@link SessionPool.open} function.
135
+ * Called automatically on first use of any public method.
152
136
  */
153
- initialize(): Promise<void>;
137
+ private ensureInitialized;
138
+ private setupPool;
154
139
  /**
155
140
  * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
156
141
  * but this allows you to add more sessions once the max pool size is reached.
@@ -170,12 +155,9 @@ export declare class SessionPool extends EventEmitter {
170
155
  * If there is space for new session, it creates and returns new session.
171
156
  * If the session pool is full, it picks a session from the pool,
172
157
  * If the picked session is usable it is returned, otherwise it creates and returns a new one.
158
+ * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
173
159
  */
174
- getSession(): Promise<Session>;
175
- /**
176
- * Gets session based on the provided session id or `undefined.
177
- */
178
- getSession(sessionId: string): Promise<Session>;
160
+ getSession(sessionId?: string): Promise<Session | undefined>;
179
161
  /**
180
162
  * @param options - Override the persistence options provided in the constructor
181
163
  */
@@ -184,75 +166,76 @@ export declare class SessionPool extends EventEmitter {
184
166
  * Returns an object representing the internal state of the `SessionPool` instance.
185
167
  * Note that the object's fields can change in future releases.
186
168
  */
187
- getState(): {
169
+ getState(): Promise<{
188
170
  usableSessionsCount: number;
189
171
  retiredSessionsCount: number;
190
172
  // @ts-ignore optional peer dependency or compatibility with es2022
191
- sessions: import("./session.js").SessionState[];
192
- };
173
+ sessions: import("@crawlee/types").SessionState[];
174
+ }>;
193
175
  /**
194
176
  * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
195
177
  * The state is persisted automatically in regular intervals.
196
178
  * @param options - Override the persistence options provided in the constructor
197
179
  */
198
180
  persistState(options?: PersistenceOptions): Promise<void>;
181
+ [Symbol.asyncDispose](): Promise<void>;
199
182
  /**
200
183
  * Removes listener from `persistState` event.
201
184
  * This function should be called after you are done with using the `SessionPool` instance.
185
+ * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
202
186
  */
203
- teardown(): Promise<void>;
204
- /**
205
- * SessionPool should not work before initialization.
206
- */
207
- protected _throwIfNotInitialized(): void;
187
+ teardown({ persistState }?: {
188
+ persistState?: boolean;
189
+ }): Promise<void>;
208
190
  /**
209
191
  * Removes retired `Session` instances from `SessionPool`.
210
192
  */
211
- protected _removeRetiredSessions(): void;
193
+ private removeRetiredSessions;
212
194
  /**
213
195
  * Adds `Session` instance to `SessionPool`.
214
196
  * @param newSession `Session` instance to be added.
215
197
  */
216
- protected _addSession(newSession: Session): void;
198
+ private registerSession;
217
199
  /**
218
200
  * Gets random index.
219
201
  */
220
- protected _getRandomIndex(): number;
202
+ private getRandomIndex;
221
203
  /**
222
204
  * Creates new session without any extra behavior.
223
- * @param sessionPool
224
205
  * @param [options]
225
206
  * @param [options.sessionOptions] The configuration options for the session being created.
226
207
  * @returns New session.
227
208
  */
228
- protected _defaultCreateSessionFunction(sessionPool: SessionPool, options?: {
229
- sessionOptions?: SessionOptions;
230
- }): Promise<Session>;
209
+ private defaultCreateSessionFunction;
210
+ /**
211
+ * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
212
+ * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
213
+ *
214
+ * A default {@link SessionFingerprint} is generated up front (host OS as
215
+ * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
216
+ * and per-call options override it, and a persisted fingerprint coming
217
+ * through `maybeLoadSessionPool` naturally wins because it arrives in
218
+ * `perCallOptions`.
219
+ */
220
+ private invokeCreateSessionFunction;
231
221
  /**
232
222
  * Creates new session and adds it to the pool.
233
223
  * @returns Newly created `Session` instance.
234
224
  */
235
- protected _createSession(): Promise<Session>;
225
+ private createSession;
236
226
  /**
237
227
  * Decides whether there is enough space for creating new session.
238
228
  */
239
- protected _hasSpaceForSession(): boolean;
229
+ private hasSpaceForSession;
240
230
  /**
241
- * Picks random session from the `SessionPool`.
242
- * @returns Picked `Session`.
231
+ * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
232
+ * Returns `undefined` when no session should be reused and a new one should be created instead.
243
233
  */
244
- protected _pickSession(): Session;
234
+ private pickSession;
245
235
  /**
246
236
  * Potentially loads `SessionPool`.
247
237
  * If the state was persisted it loads the `SessionPool` from the persisted state.
248
238
  */
249
- protected _maybeLoadSessionPool(): Promise<void>;
250
- /**
251
- * Opens a SessionPool and returns a promise resolving to an instance
252
- * of the {@link SessionPool} class that is already initialized.
253
- *
254
- * For more details and code examples, see the {@link SessionPool} class.
255
- */
256
- static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
239
+ private maybeLoadSessionPool;
257
240
  }
258
- //# sourceMappingURL=session_pool.d.ts.map
241
+ export {};