@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. package/README.md +17 -13
  2. package/byte_utils.d.ts +17 -0
  3. package/byte_utils.js +42 -0
  4. package/configuration.d.ts +88 -234
  5. package/configuration.js +162 -230
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +43 -10
  9. package/errors.js +57 -12
  10. package/events/event_manager.d.ts +35 -9
  11. package/events/event_manager.js +12 -13
  12. package/events/index.d.ts +1 -1
  13. package/events/index.js +0 -1
  14. package/events/local_event_manager.d.ts +15 -5
  15. package/events/local_event_manager.js +39 -42
  16. package/events/system_info.d.ts +38 -0
  17. package/events/system_info.js +1 -0
  18. package/index.d.ts +8 -11
  19. package/index.js +9 -10
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +81 -3
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  35. package/memory-storage/resource-clients/dataset.js +106 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +56 -0
  37. package/memory-storage/resource-clients/key-value-store.js +180 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  39. package/memory-storage/resource-clients/request-queue.js +414 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned_or_injected.d.ts +58 -0
  43. package/owned_or_injected.js +98 -0
  44. package/package.json +19 -20
  45. package/proxy_configuration.d.ts +29 -155
  46. package/proxy_configuration.js +38 -183
  47. package/recoverable_state.d.ts +177 -0
  48. package/recoverable_state.js +268 -0
  49. package/request.d.ts +62 -81
  50. package/request.js +125 -220
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service_locator.d.ts +168 -0
  54. package/service_locator.js +255 -0
  55. package/storages/batched_adds.d.ts +37 -0
  56. package/storages/batched_adds.js +73 -0
  57. package/storages/dataset.d.ts +120 -56
  58. package/storages/dataset.js +299 -148
  59. package/storages/index.d.ts +9 -8
  60. package/storages/index.js +5 -8
  61. package/storages/key_value_store.d.ts +192 -65
  62. package/storages/key_value_store.js +453 -193
  63. package/storages/key_value_store_codec.d.ts +32 -0
  64. package/storages/key_value_store_codec.js +113 -0
  65. package/storages/request_dedup_cache.d.ts +22 -0
  66. package/storages/request_dedup_cache.js +48 -0
  67. package/storages/request_list.d.ts +55 -137
  68. package/storages/request_list.js +272 -245
  69. package/storages/request_loader.d.ts +127 -0
  70. package/storages/request_loader.js +36 -0
  71. package/storages/request_manager.d.ts +119 -0
  72. package/storages/request_manager.js +1 -0
  73. package/storages/request_manager_tandem.d.ts +94 -0
  74. package/storages/request_manager_tandem.js +200 -0
  75. package/storages/request_queue.d.ts +293 -48
  76. package/storages/request_queue.js +766 -216
  77. package/storages/storage_instance_manager.d.ts +86 -0
  78. package/storages/storage_instance_manager.js +256 -0
  79. package/storages/storage_stats.d.ts +48 -0
  80. package/storages/storage_stats.js +29 -0
  81. package/storages/transaction.d.ts +271 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +75 -15
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -292
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -113
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/crawler_commons.d.ts +0 -199
  122. package/crawlers/crawler_commons.d.ts.map +0 -1
  123. package/crawlers/crawler_commons.js +0 -116
  124. package/crawlers/crawler_commons.js.map +0 -1
  125. package/crawlers/crawler_extension.d.ts +0 -12
  126. package/crawlers/crawler_extension.d.ts.map +0 -1
  127. package/crawlers/crawler_extension.js +0 -14
  128. package/crawlers/crawler_extension.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -59
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -54
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/statistics.d.ts +0 -204
  146. package/crawlers/statistics.d.ts.map +0 -1
  147. package/crawlers/statistics.js +0 -346
  148. package/crawlers/statistics.js.map +0 -1
  149. package/enqueue_links/enqueue_links.d.ts +0 -242
  150. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  151. package/enqueue_links/enqueue_links.js +0 -250
  152. package/enqueue_links/enqueue_links.js.map +0 -1
  153. package/enqueue_links/index.d.ts +0 -3
  154. package/enqueue_links/index.d.ts.map +0 -1
  155. package/enqueue_links/index.js +0 -3
  156. package/enqueue_links/index.js.map +0 -1
  157. package/enqueue_links/shared.d.ts +0 -67
  158. package/enqueue_links/shared.d.ts.map +0 -1
  159. package/enqueue_links/shared.js +0 -190
  160. package/enqueue_links/shared.js.map +0 -1
  161. package/errors.d.ts.map +0 -1
  162. package/errors.js.map +0 -1
  163. package/events/event_manager.d.ts.map +0 -1
  164. package/events/event_manager.js.map +0 -1
  165. package/events/index.d.ts.map +0 -1
  166. package/events/index.js.map +0 -1
  167. package/events/local_event_manager.d.ts.map +0 -1
  168. package/events/local_event_manager.js.map +0 -1
  169. package/http_clients/base-http-client.d.ts +0 -134
  170. package/http_clients/base-http-client.d.ts.map +0 -1
  171. package/http_clients/base-http-client.js +0 -33
  172. package/http_clients/base-http-client.js.map +0 -1
  173. package/http_clients/form-data-like.d.ts +0 -67
  174. package/http_clients/form-data-like.d.ts.map +0 -1
  175. package/http_clients/form-data-like.js +0 -5
  176. package/http_clients/form-data-like.js.map +0 -1
  177. package/http_clients/got-scraping-http-client.d.ts +0 -15
  178. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  179. package/http_clients/got-scraping-http-client.js +0 -69
  180. package/http_clients/got-scraping-http-client.js.map +0 -1
  181. package/http_clients/index.d.ts +0 -3
  182. package/http_clients/index.d.ts.map +0 -1
  183. package/http_clients/index.js +0 -3
  184. package/http_clients/index.js.map +0 -1
  185. package/index.d.ts.map +0 -1
  186. package/index.js.map +0 -1
  187. package/log.d.ts.map +0 -1
  188. package/log.js.map +0 -1
  189. package/proxy_configuration.d.ts.map +0 -1
  190. package/proxy_configuration.js.map +0 -1
  191. package/request.d.ts.map +0 -1
  192. package/request.js.map +0 -1
  193. package/router.d.ts +0 -134
  194. package/router.d.ts.map +0 -1
  195. package/router.js +0 -164
  196. package/router.js.map +0 -1
  197. package/serialization.d.ts.map +0 -1
  198. package/serialization.js.map +0 -1
  199. package/session_pool/consts.d.ts +0 -4
  200. package/session_pool/consts.d.ts.map +0 -1
  201. package/session_pool/consts.js +0 -4
  202. package/session_pool/consts.js.map +0 -1
  203. package/session_pool/errors.d.ts +0 -8
  204. package/session_pool/errors.d.ts.map +0 -1
  205. package/session_pool/errors.js +0 -12
  206. package/session_pool/errors.js.map +0 -1
  207. package/session_pool/events.d.ts +0 -3
  208. package/session_pool/events.d.ts.map +0 -1
  209. package/session_pool/events.js +0 -3
  210. package/session_pool/events.js.map +0 -1
  211. package/session_pool/index.d.ts +0 -6
  212. package/session_pool/index.d.ts.map +0 -1
  213. package/session_pool/index.js +0 -6
  214. package/session_pool/index.js.map +0 -1
  215. package/session_pool/session.d.ts +0 -199
  216. package/session_pool/session.d.ts.map +0 -1
  217. package/session_pool/session.js +0 -271
  218. package/session_pool/session.js.map +0 -1
  219. package/session_pool/session_pool.d.ts +0 -251
  220. package/session_pool/session_pool.d.ts.map +0 -1
  221. package/session_pool/session_pool.js +0 -375
  222. package/session_pool/session_pool.js.map +0 -1
  223. package/storages/access_checking.d.ts +0 -13
  224. package/storages/access_checking.d.ts.map +0 -1
  225. package/storages/access_checking.js +0 -14
  226. package/storages/access_checking.js.map +0 -1
  227. package/storages/dataset.d.ts.map +0 -1
  228. package/storages/dataset.js.map +0 -1
  229. package/storages/index.d.ts.map +0 -1
  230. package/storages/index.js.map +0 -1
  231. package/storages/key_value_store.d.ts.map +0 -1
  232. package/storages/key_value_store.js.map +0 -1
  233. package/storages/request_list.d.ts.map +0 -1
  234. package/storages/request_list.js.map +0 -1
  235. package/storages/request_provider.d.ts +0 -307
  236. package/storages/request_provider.d.ts.map +0 -1
  237. package/storages/request_provider.js +0 -555
  238. package/storages/request_provider.js.map +0 -1
  239. package/storages/request_queue.d.ts.map +0 -1
  240. package/storages/request_queue.js.map +0 -1
  241. package/storages/request_queue_v2.d.ts +0 -87
  242. package/storages/request_queue_v2.d.ts.map +0 -1
  243. package/storages/request_queue_v2.js +0 -438
  244. package/storages/request_queue_v2.js.map +0 -1
  245. package/storages/sitemap_request_list.d.ts +0 -245
  246. package/storages/sitemap_request_list.d.ts.map +0 -1
  247. package/storages/sitemap_request_list.js +0 -430
  248. package/storages/sitemap_request_list.js.map +0 -1
  249. package/storages/storage_manager.d.ts +0 -58
  250. package/storages/storage_manager.d.ts.map +0 -1
  251. package/storages/storage_manager.js +0 -105
  252. package/storages/storage_manager.js.map +0 -1
  253. package/storages/utils.d.ts.map +0 -1
  254. package/storages/utils.js.map +0 -1
  255. package/tsconfig.build.tsbuildinfo +0 -1
  256. package/typedefs.d.ts.map +0 -1
  257. package/typedefs.js.map +0 -1
  258. package/validators.d.ts.map +0 -1
  259. package/validators.js.map +0 -1
@@ -1 +0,0 @@
1
- {"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,4CAA4C;IAC5C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,GAAE,mBAAwB;IAgC7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAkBlE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK3D;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK9D;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAKxD;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAKxE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,EAC9E,MAAM,EAAE,CAAC,EAAE,EACX,KAAK,EAAE,MAAM,GACd,UAAU;CA4BhB"}
@@ -1,169 +0,0 @@
1
- import { weightedAvg } from '@crawlee/utils';
2
- import ow from 'ow';
3
- import { Snapshotter } from './snapshotter.js';
4
- /**
5
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
6
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
7
- * and {@link SystemStatus.getHistoricalStatus}.
8
- * The system status is calculated using a weighted average of overloaded
9
- * messages in the snapshots, with the weights being the time intervals
10
- * between the snapshots. Each resource is calculated separately
11
- * and the system is overloaded whenever at least one resource is overloaded.
12
- * The class is used by the {@link AutoscaledPool} class.
13
- *
14
- * {@link SystemStatus.getCurrentStatus}
15
- * returns a boolean that represents the current status of the system.
16
- * The length of the current timeframe in seconds is configurable
17
- * by the `currentHistorySecs` option and represents the max age
18
- * of snapshots to be considered for the calculation.
19
- *
20
- * {@link SystemStatus.getHistoricalStatus}
21
- * returns a boolean that represents the long-term status
22
- * of the system. It considers the full snapshot history available
23
- * in the {@link Snapshotter} instance.
24
- * @category Scaling
25
- */
26
- export class SystemStatus {
27
- currentHistoryMillis;
28
- maxMemoryOverloadedRatio;
29
- maxEventLoopOverloadedRatio;
30
- maxCpuOverloadedRatio;
31
- maxClientOverloadedRatio;
32
- snapshotter;
33
- constructor(options = {}) {
34
- ow(options, ow.object.exactShape({
35
- currentHistorySecs: ow.optional.number,
36
- maxMemoryOverloadedRatio: ow.optional.number,
37
- maxEventLoopOverloadedRatio: ow.optional.number,
38
- maxCpuOverloadedRatio: ow.optional.number,
39
- maxClientOverloadedRatio: ow.optional.number,
40
- snapshotter: ow.optional.object,
41
- config: ow.optional.object,
42
- }));
43
- const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
44
- this.currentHistoryMillis = currentHistorySecs * 1000;
45
- this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
46
- this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
47
- this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
48
- this.maxClientOverloadedRatio = maxClientOverloadedRatio;
49
- this.snapshotter = snapshotter || new Snapshotter({ config });
50
- }
51
- /**
52
- * Returns an {@link SystemInfo} object with the following structure:
53
- *
54
- * ```javascript
55
- * {
56
- * isSystemIdle: Boolean,
57
- * memInfo: Object,
58
- * eventLoopInfo: Object,
59
- * cpuInfo: Object
60
- * }
61
- * ```
62
- *
63
- * Where the `isSystemIdle` property is set to `false` if the system
64
- * has been overloaded in the last `options.currentHistorySecs` seconds,
65
- * and `true` otherwise.
66
- */
67
- getCurrentStatus() {
68
- return this._isSystemIdle(this.currentHistoryMillis);
69
- }
70
- /**
71
- * Returns an {@link SystemInfo} object with the following structure:
72
- *
73
- * ```javascript
74
- * {
75
- * isSystemIdle: Boolean,
76
- * memInfo: Object,
77
- * eventLoopInfo: Object,
78
- * cpuInfo: Object
79
- * }
80
- * ```
81
- *
82
- * Where the `isSystemIdle` property is set to `false` if the system
83
- * has been overloaded in the full history of the {@link Snapshotter}
84
- * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
85
- */
86
- getHistoricalStatus() {
87
- return this._isSystemIdle();
88
- }
89
- /**
90
- * Returns a system status object.
91
- */
92
- _isSystemIdle(sampleDurationMillis) {
93
- const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
94
- const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
95
- const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
96
- const clientInfo = this._isClientOverloaded(sampleDurationMillis);
97
- return {
98
- isSystemIdle: !memInfo.isOverloaded &&
99
- !eventLoopInfo.isOverloaded &&
100
- !cpuInfo.isOverloaded &&
101
- !clientInfo.isOverloaded,
102
- memInfo,
103
- eventLoopInfo,
104
- cpuInfo,
105
- clientInfo,
106
- };
107
- }
108
- /**
109
- * Returns an object with an isOverloaded property set to true
110
- * if the memory has been overloaded in the last sampleDurationMillis.
111
- */
112
- _isMemoryOverloaded(sampleDurationMillis) {
113
- const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
114
- return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
115
- }
116
- /**
117
- * Returns an object with an isOverloaded property set to true
118
- * if the event loop has been overloaded in the last sampleDurationMillis.
119
- */
120
- _isEventLoopOverloaded(sampleDurationMillis) {
121
- const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
122
- return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
123
- }
124
- /**
125
- * Returns an object with an isOverloaded property set to true
126
- * if the CPU has been overloaded in the last sampleDurationMillis.
127
- */
128
- _isCpuOverloaded(sampleDurationMillis) {
129
- const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
130
- return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
131
- }
132
- /**
133
- * Returns an object with an isOverloaded property set to true
134
- * if the client has been overloaded in the last sampleDurationMillis.
135
- */
136
- _isClientOverloaded(sampleDurationMillis) {
137
- const sample = this.snapshotter.getClientSample(sampleDurationMillis);
138
- return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
139
- }
140
- /**
141
- * Returns an object with sample information and an isOverloaded property
142
- * set to true if at least the ratio of snapshots in the sample are overloaded.
143
- */
144
- _isSampleOverloaded(sample, ratio) {
145
- if (sample.length === 0) {
146
- return {
147
- isOverloaded: false,
148
- limitRatio: ratio,
149
- actualRatio: 0,
150
- };
151
- }
152
- const weights = [];
153
- const values = [];
154
- for (let i = 1; i < sample.length; i++) {
155
- const previous = sample[i - 1];
156
- const current = sample[i];
157
- const weight = +current.createdAt - +previous.createdAt;
158
- weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
159
- values.push(+current.isOverloaded);
160
- }
161
- const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
162
- return {
163
- isOverloaded: wAvg > ratio,
164
- limitRatio: ratio,
165
- actualRatio: Math.round(wAvg * 1000) / 1000,
166
- };
167
- }
168
- }
169
- //# sourceMappingURL=system_status.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"system_status.js","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AA6F/C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,YAAY;IACJ,oBAAoB,CAAS;IAC7B,wBAAwB,CAAS;IACjC,2BAA2B,CAAS;IACpC,qBAAqB,CAAS;IAC9B,wBAAwB,CAAS;IACjC,WAAW,CAAc;IAE1C,YAAY,UAA+B,EAAE;QACzC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,kBAAkB,GAAG,CAAC,EACtB,wBAAwB,GAAG,GAAG,EAC9B,2BAA2B,GAAG,GAAG,EACjC,qBAAqB,GAAG,GAAG,EAC3B,wBAAwB,GAAG,GAAG,EAC9B,WAAW,EACX,MAAM,GACT,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,oBAAoB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACtD,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QACzD,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,WAAW,IAAI,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB;QACf,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,oBAA6B;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;QAClE,OAAO;YACH,YAAY,EACR,CAAC,OAAO,CAAC,YAAY;gBACrB,CAAC,aAAa,CAAC,YAAY;gBAC3B,CAAC,OAAO,CAAC,YAAY;gBACrB,CAAC,UAAU,CAAC,YAAY;YAC5B,OAAO;YACP,aAAa;YACb,OAAO;YACP,UAAU;SACb,CAAC;IACN,CAAC;IAED;;;OAGG;IACO,mBAAmB,CAAC,oBAA6B;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACO,sBAAsB,CAAC,oBAA6B;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC9E,CAAC;IAED;;;OAGG;IACO,gBAAgB,CAAC,oBAA6B;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACxE,CAAC;IAED;;;OAGG;IACO,mBAAmB,CAAC,oBAA6B;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACO,mBAAmB,CACzB,MAAW,EACX,KAAa;QAEb,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACH,YAAY,EAAE,KAAK;gBACnB,UAAU,EAAE,KAAK;gBACjB,WAAW,EAAE,CAAC;aACjB,CAAC;QACN,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,mEAAmE;YAC9F,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACvC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE1F,OAAO;YACH,YAAY,EAAE,IAAI,GAAG,KAAK;YAC1B,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;SAC9C,CAAC;IACN,CAAC;CACJ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,KAAK,WAAW,EAAW,MAAM,eAAe,CAAC;AAE1D,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE/C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAkBlC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAQ3B;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,yBAAyB,EAAE,MAAM,EAAE,CAAM;IAE1D,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAcnC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC,SAAgB,eAAe,qFAA0C;IAEzE;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAiB9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAoBzF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAI9D;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAiBtE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAIpD,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAoB,OAAO,EAAE,MAAM,eAAe,CAAC;AAuK1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,MAAM,OAAO,aAAa;IACtB;;OAEG;IACO,MAAM,CAAC,OAAO,GAAe;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,gBAAgB;QACzC,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,eAAe;KACzC,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG;QAC5B,cAAc;QACd,UAAU;QACV,MAAM;QACN,uBAAuB;QACvB,gBAAgB;QAChB,cAAc;QACd,eAAe;KAClB,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;IAEjG,MAAM,CAAC,yBAAyB,GAAa,EAAE,CAAC;IAEhD,MAAM,CAAC,QAAQ,GAAe;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,MAAM;QAClC,wBAAwB,EAAE,KAAK;QAC/B,cAAc,EAAE,IAAI;QACpB,YAAY,EAAE,KAAK;KACtB,CAAC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,GAAG,IAAI,iBAAiB,EAAiB,CAAC;IAE9C,OAAO,CAAqF;IAC5F,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAiB;IAEpB,eAAe,GAAG,IAAI,GAAG,EAA+B,CAAC;IAEzE;;OAEG;IACH,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,YAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;gBACpC,CAAC,CAAC,CAAC,QAAQ;gBACX,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACtE,GAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACZ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;QAClD,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACtC,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,KAAK,CAAC;QAClB,CAAC;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,aAAa,CAAC,yBAAyB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,KAAK;gBAAE,OAAO,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,KAAK,CAAC;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAA+B,EAAE,KAAW;QACnD,IAAI,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;QAC9D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC7D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;YAC9B,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAC1C,uEAAuE;YACvE,GAAG,OAAO;SACb,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,MAAqB;QACzC,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;QAC7C,CAAC;QAED,aAAa,CAAC,YAAY,KAAK,IAAI,aAAa,EAAE,CAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;QACL,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC"}
package/cookie_utils.d.ts DELETED
@@ -1,43 +0,0 @@
1
- import type { Cookie as CookieObject } from '@crawlee/types';
2
- import { Cookie } from 'tough-cookie';
3
- export interface ResponseLike {
4
- url?: string | (() => string);
5
- headers?: Record<string, string | string[] | undefined> | (() => Record<string, string | string[] | undefined>);
6
- }
7
- /**
8
- * @internal
9
- */
10
- export declare function getCookiesFromResponse(response: ResponseLike): Cookie[];
11
- /**
12
- * Calculate cookie expiration date
13
- * @param maxAgeSecs
14
- * @returns Calculated date by session max age seconds.
15
- * @internal
16
- */
17
- export declare function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
18
- /**
19
- * Transforms tough-cookie to puppeteer cookie.
20
- * @param toughCookie Cookie from CookieJar
21
- * @return Cookie compatible with browser pool
22
- * @internal
23
- */
24
- export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject;
25
- /**
26
- * Transforms browser-pool cookie to tough-cookie.
27
- * @param cookieObject Cookie object (for instance from the `page.cookies` method).
28
- * @internal
29
- */
30
- export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number): Cookie;
31
- /**
32
- * @internal
33
- * @param cookieString The cookie string to attempt parsing
34
- * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
35
- */
36
- export declare function cookieStringToToughCookie(cookieString: string): CookieObject | null;
37
- /**
38
- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
39
- * if we see two cookies with same keys but different casing.
40
- * @internal
41
- */
42
- export declare function mergeCookies(url: string, sourceCookies: string[]): string;
43
- //# sourceMappingURL=cookie_utils.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,YAAY;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,GAAG,MAAM,EAAE,CAWvE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAehF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAiB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA8BzE"}
package/cookie_utils.js DELETED
@@ -1,113 +0,0 @@
1
- import { Cookie, CookieJar } from 'tough-cookie';
2
- import { log } from './log.js';
3
- import { CookieParseError } from './session_pool/errors.js';
4
- /**
5
- * @internal
6
- */
7
- export function getCookiesFromResponse(response) {
8
- const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
9
- const cookieHeader = headers?.['set-cookie'] || '';
10
- try {
11
- return Array.isArray(cookieHeader)
12
- ? cookieHeader.map((cookie) => Cookie.parse(cookie))
13
- : [Cookie.parse(cookieHeader)];
14
- }
15
- catch (e) {
16
- throw new CookieParseError(cookieHeader);
17
- }
18
- }
19
- /**
20
- * Calculate cookie expiration date
21
- * @param maxAgeSecs
22
- * @returns Calculated date by session max age seconds.
23
- * @internal
24
- */
25
- export function getDefaultCookieExpirationDate(maxAgeSecs) {
26
- return new Date(Date.now() + maxAgeSecs * 1000);
27
- }
28
- /**
29
- * Transforms tough-cookie to puppeteer cookie.
30
- * @param toughCookie Cookie from CookieJar
31
- * @return Cookie compatible with browser pool
32
- * @internal
33
- */
34
- export function toughCookieToBrowserPoolCookie(toughCookie) {
35
- return {
36
- name: toughCookie.key,
37
- value: toughCookie.value,
38
- // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
39
- // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
40
- expires: toughCookie.expires == null || toughCookie.expires === 'Infinity'
41
- ? undefined
42
- : toughCookie.expires.getTime() / 1000,
43
- domain: toughCookie.domain ? `${toughCookie.hostOnly ? '' : '.'}${toughCookie.domain}` : undefined,
44
- path: toughCookie.path ?? undefined,
45
- secure: toughCookie.secure,
46
- httpOnly: toughCookie.httpOnly,
47
- };
48
- }
49
- /**
50
- * Transforms browser-pool cookie to tough-cookie.
51
- * @param cookieObject Cookie object (for instance from the `page.cookies` method).
52
- * @internal
53
- */
54
- export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
55
- const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
56
- const expires = isExpiresValid
57
- ? new Date(cookieObject.expires * 1000)
58
- : getDefaultCookieExpirationDate(maxAgeSecs);
59
- const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
60
- const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
61
- return new Cookie({
62
- key: cookieObject.name,
63
- value: cookieObject.value,
64
- expires,
65
- domain,
66
- path: cookieObject.path,
67
- secure: cookieObject.secure,
68
- httpOnly: cookieObject.httpOnly,
69
- hostOnly: !domainHasLeadingDot,
70
- });
71
- }
72
- /**
73
- * @internal
74
- * @param cookieString The cookie string to attempt parsing
75
- * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
76
- */
77
- export function cookieStringToToughCookie(cookieString) {
78
- const parsed = Cookie.parse(cookieString);
79
- if (parsed) {
80
- return toughCookieToBrowserPoolCookie(parsed);
81
- }
82
- return null;
83
- }
84
- /**
85
- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
86
- * if we see two cookies with same keys but different casing.
87
- * @internal
88
- */
89
- export function mergeCookies(url, sourceCookies) {
90
- const jar = new CookieJar();
91
- // ignore empty cookies
92
- for (const sourceCookieString of sourceCookies) {
93
- // ignore empty cookies
94
- if (!sourceCookieString)
95
- continue;
96
- const cookies = sourceCookieString.split(/ *; */);
97
- for (const cookieString of cookies) {
98
- // ignore extra spaces
99
- if (!cookieString)
100
- continue;
101
- const cookie = Cookie.parse(cookieString);
102
- const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
103
- return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
104
- });
105
- if (similarKeyCookie) {
106
- log.warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
107
- }
108
- jar.setCookieSync(cookie, url);
109
- }
110
- }
111
- return jar.getCookieStringSync(url);
112
- }
113
- //# sourceMappingURL=cookie_utils.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAO5D;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAsB;IACzD,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEnD,IAAI,CAAC;QACD,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,gBAAgB,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EACH,WAAW,CAAC,OAAO,IAAI,IAAI,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU;YAC7D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI;QAC9C,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;QAClG,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc;QAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAC3F,OAAO,IAAI,MAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,QAAQ,EAAE,CAAC,mBAAmB;KACjC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;QAC7C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACnB,GAAG,CAAC,WAAW,CACX,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CACzG,CAAC;YACN,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC"}
@@ -1,199 +0,0 @@
1
- import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
2
- // @ts-ignore optional peer dependency or compatibility with es2022
3
- import type { OptionsInit, Response as GotResponse } from 'got-scraping';
4
- import type { ReadonlyDeep } from 'type-fest';
5
- import type { Configuration } from '../configuration.js';
6
- import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
7
- import type { Log } from '../log.js';
8
- import type { ProxyInfo } from '../proxy_configuration.js';
9
- import type { Request, Source } from '../request.js';
10
- import type { Session } from '../session_pool/session.js';
11
- import type { Dataset } from '../storages/dataset.js';
12
- import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
13
- import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
14
- /** @internal */
15
- export type IsAny<T> = 0 extends 1 & T ? true : false;
16
- /** @internal */
17
- export type WithRequired<T, K extends keyof T> = T & {
18
- [P in K]-?: T[P];
19
- };
20
- export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl'>;
21
- /** @internal */
22
- export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
23
- request: LoadedRequest<Context['request']>;
24
- } & Omit<Context, 'request'>;
25
- export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
26
- id: string;
27
- session?: Session;
28
- /**
29
- * An object with information about currently used proxy by the crawler
30
- * and configured by the {@link ProxyConfiguration} class.
31
- */
32
- proxyInfo?: ProxyInfo;
33
- /**
34
- * The original {@link Request} object.
35
- */
36
- request: Request<UserData>;
37
- /**
38
- * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
39
- *
40
- * Shortcut for `crawler.pushData()`.
41
- *
42
- * @param [data] Data to be pushed to the default dataset.
43
- */
44
- pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdOrName?: string): Promise<void>;
45
- /**
46
- * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
47
- * currently used by the crawler.
48
- *
49
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
50
- * and override settings of the enqueued {@link Request} objects.
51
- *
52
- * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
53
- * for more details regarding its usage.
54
- *
55
- * **Example usage**
56
- *
57
- * ```ts
58
- * async requestHandler({ enqueueLinks }) {
59
- * await enqueueLinks({
60
- * globs: [
61
- * 'https://www.example.com/handbags/*',
62
- * ],
63
- * });
64
- * },
65
- * ```
66
- *
67
- * @param [options] All `enqueueLinks()` parameters are passed via an options object.
68
- */
69
- enqueueLinks: (options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>>) => Promise<unknown>;
70
- /**
71
- * Add requests directly to the request queue.
72
- *
73
- * @param requests The requests to add
74
- * @param options Options for the request queue
75
- */
76
- addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<RequestQueueOperationOptions>) => Promise<void>;
77
- /**
78
- * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
79
- */
80
- useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
81
- /**
82
- * Get a key-value store with given name or id, or the default one for the crawler.
83
- */
84
- getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
85
- /**
86
- * A preconfigured logger for the request handler.
87
- */
88
- log: Log;
89
- }
90
- export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
91
- crawler: Crawler;
92
- /**
93
- * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
94
- * currently used by the crawler.
95
- *
96
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
97
- * and override settings of the enqueued {@link Request} objects.
98
- *
99
- * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
100
- * for more details regarding its usage.
101
- *
102
- * **Example usage**
103
- *
104
- * ```ts
105
- * async requestHandler({ enqueueLinks }) {
106
- * await enqueueLinks({
107
- * globs: [
108
- * 'https://www.example.com/handbags/*',
109
- * ],
110
- * });
111
- * },
112
- * ```
113
- *
114
- * @param [options] All `enqueueLinks()` parameters are passed via an options object.
115
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
116
- */
117
- enqueueLinks(options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>): Promise<BatchAddRequestsResult>;
118
- /**
119
- * Get a key-value store with given name or id, or the default one for the crawler.
120
- */
121
- getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;
122
- /**
123
- * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
124
- * options on the fly.
125
- *
126
- * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
127
- * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
128
- * more detailed explanation of how to do that.
129
- *
130
- * ```ts
131
- * async requestHandler({ sendRequest }) {
132
- * const { body } = await sendRequest({
133
- * // override headers only
134
- * headers: { ... },
135
- * });
136
- * },
137
- * ```
138
- */
139
- sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;
140
- }
141
- /**
142
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
143
- *
144
- * @experimental
145
- */
146
- export declare class RequestHandlerResult {
147
- private config;
148
- private crawleeStateKey;
149
- private _keyValueStoreChanges;
150
- private pushDataCalls;
151
- private addRequestsCalls;
152
- private enqueueLinksCalls;
153
- constructor(config: Configuration, crawleeStateKey: string);
154
- /**
155
- * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
156
- */
157
- get calls(): ReadonlyDeep<{
158
- pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
159
- addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
160
- enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
161
- }>;
162
- /**
163
- * A record of changes made to key-value stores by a request handler.
164
- */
165
- get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
166
- changedValue: unknown;
167
- options?: RecordOptions;
168
- }>>>;
169
- /**
170
- * Items added to datasets by a request handler.
171
- */
172
- get datasetItems(): ReadonlyDeep<{
173
- item: Dictionary;
174
- datasetIdOrName?: string;
175
- }[]>;
176
- /**
177
- * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
178
- */
179
- get enqueuedUrls(): ReadonlyDeep<{
180
- url: string;
181
- label?: string;
182
- }[]>;
183
- /**
184
- * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
185
- */
186
- get enqueuedUrlLists(): ReadonlyDeep<{
187
- listUrl: string;
188
- label?: string;
189
- }[]>;
190
- pushData: RestrictedCrawlingContext['pushData'];
191
- enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
192
- addRequests: RestrictedCrawlingContext['addRequests'];
193
- useState: RestrictedCrawlingContext['useState'];
194
- getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
195
- private idOrDefault;
196
- private getKeyValueStoreChangedValue;
197
- private setKeyValueStoreChangedValue;
198
- }
199
- //# sourceMappingURL=crawler_commons.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAEpF,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAE/E,SAAQ,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE,OAAO,CAAC;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtG;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,SAAS,UAAU,GAAG,UAAU,CACxF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAC9G,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAEhE;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,QAAQ,GAAG,MAAM,EAAE,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;CAC1G;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IAWzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAX3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;IAEtF,OAAO,CAAC,iBAAiB,CAA+D;gBAG5E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,YAAY,EAAE,UAAU,CAAC,yBAAyB,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;KACzE,CAAC,CAMD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAsBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,YAAY,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAErD;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}