@crawlee/core 4.0.0-beta.19 → 4.0.0-beta.190

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/README.md +14 -14
  2. package/byte_utils.d.ts +17 -0
  3. package/byte_utils.js +42 -0
  4. package/configuration.d.ts +90 -226
  5. package/configuration.js +164 -225
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +40 -19
  9. package/errors.js +53 -28
  10. package/events/event_manager.d.ts +35 -9
  11. package/events/event_manager.js +12 -13
  12. package/events/index.d.ts +1 -1
  13. package/events/index.js +0 -1
  14. package/events/local_event_manager.d.ts +15 -3
  15. package/events/local_event_manager.js +39 -13
  16. package/events/system_info.d.ts +38 -0
  17. package/events/system_info.js +1 -0
  18. package/index.d.ts +7 -11
  19. package/index.js +8 -10
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +44 -0
  31. package/memory-storage/memory-storage.js +160 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  35. package/memory-storage/resource-clients/dataset.js +106 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  37. package/memory-storage/resource-clients/key-value-store.js +199 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  39. package/memory-storage/resource-clients/request-queue.js +414 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned_or_injected.d.ts +58 -0
  43. package/owned_or_injected.js +98 -0
  44. package/package.json +18 -19
  45. package/proxy_configuration.d.ts +22 -134
  46. package/proxy_configuration.js +34 -146
  47. package/recoverable_state.d.ts +85 -51
  48. package/recoverable_state.js +178 -72
  49. package/request.d.ts +48 -93
  50. package/request.js +116 -217
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +15 -15
  53. package/service_locator.d.ts +165 -0
  54. package/service_locator.js +254 -0
  55. package/storages/batched_adds.d.ts +37 -0
  56. package/storages/batched_adds.js +73 -0
  57. package/storages/dataset.d.ts +114 -56
  58. package/storages/dataset.js +292 -149
  59. package/storages/index.d.ts +7 -9
  60. package/storages/index.js +5 -9
  61. package/storages/key_value_store.d.ts +187 -50
  62. package/storages/key_value_store.js +453 -172
  63. package/storages/key_value_store_codec.d.ts +32 -0
  64. package/storages/key_value_store_codec.js +113 -0
  65. package/storages/request_dedup_cache.d.ts +22 -0
  66. package/storages/request_dedup_cache.js +48 -0
  67. package/storages/request_list.d.ts +54 -129
  68. package/storages/request_list.js +272 -245
  69. package/storages/request_loader.d.ts +127 -0
  70. package/storages/request_loader.js +36 -0
  71. package/storages/request_manager.d.ts +109 -0
  72. package/storages/request_manager.js +1 -0
  73. package/storages/request_manager_tandem.d.ts +50 -23
  74. package/storages/request_manager_tandem.js +132 -57
  75. package/storages/request_queue.d.ts +286 -48
  76. package/storages/request_queue.js +756 -218
  77. package/storages/storage_instance_manager.d.ts +87 -0
  78. package/storages/storage_instance_manager.js +256 -0
  79. package/storages/storage_stats.d.ts +48 -0
  80. package/storages/storage_stats.js +29 -0
  81. package/storages/transaction.d.ts +260 -0
  82. package/storages/transaction.js +287 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +75 -15
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -195
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -204
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -346
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js.map +0 -1
  173. package/http_clients/base-http-client.d.ts +0 -91
  174. package/http_clients/base-http-client.d.ts.map +0 -1
  175. package/http_clients/base-http-client.js +0 -44
  176. package/http_clients/base-http-client.js.map +0 -1
  177. package/http_clients/form-data-like.d.ts +0 -67
  178. package/http_clients/form-data-like.d.ts.map +0 -1
  179. package/http_clients/form-data-like.js +0 -5
  180. package/http_clients/form-data-like.js.map +0 -1
  181. package/http_clients/got-scraping-http-client.d.ts +0 -20
  182. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  183. package/http_clients/got-scraping-http-client.js +0 -102
  184. package/http_clients/got-scraping-http-client.js.map +0 -1
  185. package/http_clients/index.d.ts +0 -3
  186. package/http_clients/index.d.ts.map +0 -1
  187. package/http_clients/index.js +0 -3
  188. package/http_clients/index.js.map +0 -1
  189. package/index.d.ts.map +0 -1
  190. package/index.js.map +0 -1
  191. package/log.d.ts.map +0 -1
  192. package/log.js.map +0 -1
  193. package/proxy_configuration.d.ts.map +0 -1
  194. package/proxy_configuration.js.map +0 -1
  195. package/recoverable_state.d.ts.map +0 -1
  196. package/recoverable_state.js.map +0 -1
  197. package/request.d.ts.map +0 -1
  198. package/request.js.map +0 -1
  199. package/router.d.ts +0 -134
  200. package/router.d.ts.map +0 -1
  201. package/router.js +0 -164
  202. package/router.js.map +0 -1
  203. package/serialization.d.ts.map +0 -1
  204. package/serialization.js.map +0 -1
  205. package/session_pool/consts.d.ts +0 -4
  206. package/session_pool/consts.d.ts.map +0 -1
  207. package/session_pool/consts.js +0 -4
  208. package/session_pool/consts.js.map +0 -1
  209. package/session_pool/errors.d.ts +0 -8
  210. package/session_pool/errors.d.ts.map +0 -1
  211. package/session_pool/errors.js +0 -12
  212. package/session_pool/errors.js.map +0 -1
  213. package/session_pool/events.d.ts +0 -3
  214. package/session_pool/events.d.ts.map +0 -1
  215. package/session_pool/events.js +0 -3
  216. package/session_pool/events.js.map +0 -1
  217. package/session_pool/index.d.ts +0 -6
  218. package/session_pool/index.d.ts.map +0 -1
  219. package/session_pool/index.js +0 -6
  220. package/session_pool/index.js.map +0 -1
  221. package/session_pool/session.d.ts +0 -203
  222. package/session_pool/session.d.ts.map +0 -1
  223. package/session_pool/session.js +0 -278
  224. package/session_pool/session.js.map +0 -1
  225. package/session_pool/session_pool.d.ts +0 -258
  226. package/session_pool/session_pool.d.ts.map +0 -1
  227. package/session_pool/session_pool.js +0 -387
  228. package/session_pool/session_pool.js.map +0 -1
  229. package/storages/access_checking.d.ts +0 -13
  230. package/storages/access_checking.d.ts.map +0 -1
  231. package/storages/access_checking.js +0 -14
  232. package/storages/access_checking.js.map +0 -1
  233. package/storages/dataset.d.ts.map +0 -1
  234. package/storages/dataset.js.map +0 -1
  235. package/storages/index.d.ts.map +0 -1
  236. package/storages/index.js.map +0 -1
  237. package/storages/key_value_store.d.ts.map +0 -1
  238. package/storages/key_value_store.js.map +0 -1
  239. package/storages/request_list.d.ts.map +0 -1
  240. package/storages/request_list.js.map +0 -1
  241. package/storages/request_list_adapter.d.ts +0 -58
  242. package/storages/request_list_adapter.d.ts.map +0 -1
  243. package/storages/request_list_adapter.js +0 -81
  244. package/storages/request_list_adapter.js.map +0 -1
  245. package/storages/request_manager_tandem.d.ts.map +0 -1
  246. package/storages/request_manager_tandem.js.map +0 -1
  247. package/storages/request_provider.d.ts +0 -371
  248. package/storages/request_provider.d.ts.map +0 -1
  249. package/storages/request_provider.js +0 -585
  250. package/storages/request_provider.js.map +0 -1
  251. package/storages/request_queue.d.ts.map +0 -1
  252. package/storages/request_queue.js.map +0 -1
  253. package/storages/request_queue_v2.d.ts +0 -87
  254. package/storages/request_queue_v2.d.ts.map +0 -1
  255. package/storages/request_queue_v2.js +0 -438
  256. package/storages/request_queue_v2.js.map +0 -1
  257. package/storages/sitemap_request_list.d.ts +0 -245
  258. package/storages/sitemap_request_list.d.ts.map +0 -1
  259. package/storages/sitemap_request_list.js +0 -430
  260. package/storages/sitemap_request_list.js.map +0 -1
  261. package/storages/storage_manager.d.ts +0 -58
  262. package/storages/storage_manager.d.ts.map +0 -1
  263. package/storages/storage_manager.js +0 -105
  264. package/storages/storage_manager.js.map +0 -1
  265. package/storages/utils.d.ts.map +0 -1
  266. package/storages/utils.js.map +0 -1
  267. package/typedefs.d.ts.map +0 -1
  268. package/typedefs.js.map +0 -1
  269. package/validators.d.ts.map +0 -1
  270. package/validators.js.map +0 -1
@@ -1,533 +0,0 @@
1
- import ow from 'ow';
2
- import { addTimeoutToPromise } from '@apify/timeout';
3
- import { betterClearInterval, betterSetInterval } from '@apify/utilities';
4
- import { Configuration } from '../configuration.js';
5
- import { CriticalError } from '../errors.js';
6
- import { log as defaultLog } from '../log.js';
7
- import { Snapshotter } from './snapshotter.js';
8
- import { SystemStatus } from './system_status.js';
9
- /**
10
- * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
11
- * The pool only starts new tasks if there is enough free CPU and memory available
12
- * and the Javascript event loop is not blocked.
13
- *
14
- * The information about the CPU and memory usage is obtained by the {@link Snapshotter} class,
15
- * which makes regular snapshots of system resources that may be either local
16
- * or from the Apify cloud infrastructure in case the process is running on the Apify platform.
17
- * Meaningful data gathered from these snapshots is provided to `AutoscaledPool` by the {@link SystemStatus} class.
18
- *
19
- * Before running the pool, you need to implement the following three functions:
20
- * {@link AutoscaledPoolOptions.runTaskFunction},
21
- * {@link AutoscaledPoolOptions.isTaskReadyFunction} and
22
- * {@link AutoscaledPoolOptions.isFinishedFunction}.
23
- *
24
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
25
- * The pool periodically queries the {@link AutoscaledPoolOptions.isTaskReadyFunction} function
26
- * for more tasks, managing optimal concurrency, until the function resolves to `false`. The pool then queries
27
- * the {@link AutoscaledPoolOptions.isFinishedFunction}. If it resolves to `true`, the run finishes after all running tasks complete.
28
- * If it resolves to `false`, it assumes there will be more tasks available later and keeps periodically querying for tasks.
29
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
30
- *
31
- * The pool evaluates whether it should start a new task every time one of the tasks finishes
32
- * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
33
- *
34
- * **Example usage:**
35
- *
36
- * ```javascript
37
- * const pool = new AutoscaledPool({
38
- * maxConcurrency: 50,
39
- * runTaskFunction: async () => {
40
- * // Run some resource-intensive asynchronous operation here.
41
- * },
42
- * isTaskReadyFunction: async () => {
43
- * // Tell the pool whether more tasks are ready to be processed.
44
- * // Return true or false
45
- * },
46
- * isFinishedFunction: async () => {
47
- * // Tell the pool whether it should finish
48
- * // or wait for more tasks to become available.
49
- * // Return true or false
50
- * }
51
- * });
52
- *
53
- * await pool.run();
54
- * ```
55
- * @category Scaling
56
- */
57
- export class AutoscaledPool {
58
- config;
59
- log;
60
- // Configurable properties.
61
- desiredConcurrencyRatio;
62
- scaleUpStepRatio;
63
- scaleDownStepRatio;
64
- maybeRunIntervalMillis;
65
- loggingIntervalMillis;
66
- autoscaleIntervalMillis;
67
- taskTimeoutMillis;
68
- runTaskFunction;
69
- isFinishedFunction;
70
- isTaskReadyFunction;
71
- maxTasksPerMinute;
72
- // Internal properties.
73
- _minConcurrency;
74
- _maxConcurrency;
75
- _desiredConcurrency;
76
- _currentConcurrency = 0;
77
- isStopped = false;
78
- lastLoggingTime;
79
- resolve = null;
80
- reject = null;
81
- snapshotter;
82
- systemStatus;
83
- autoscaleInterval;
84
- maybeRunInterval;
85
- queryingIsTaskReady;
86
- queryingIsFinished;
87
- tasksDonePerSecondInterval;
88
- _tasksPerMinute = Array.from({ length: 60 }, () => 0);
89
- constructor(options, config = Configuration.getGlobalConfig()) {
90
- this.config = config;
91
- ow(options, ow.object.exactShape({
92
- runTaskFunction: ow.function,
93
- isFinishedFunction: ow.function,
94
- isTaskReadyFunction: ow.function,
95
- maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
96
- minConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
97
- desiredConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
98
- desiredConcurrencyRatio: ow.optional.number.greaterThan(0).lessThan(1),
99
- scaleUpStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
100
- scaleDownStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
101
- maybeRunIntervalSecs: ow.optional.number.greaterThan(0),
102
- loggingIntervalSecs: ow.any(ow.number.greaterThan(0), ow.nullOrUndefined),
103
- autoscaleIntervalSecs: ow.optional.number.greaterThan(0),
104
- taskTimeoutSecs: ow.optional.number.greaterThanOrEqual(0),
105
- systemStatusOptions: ow.optional.object,
106
- snapshotterOptions: ow.optional.object,
107
- log: ow.optional.object,
108
- maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
109
- }));
110
- const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = defaultLog, maxTasksPerMinute = Infinity, } = options;
111
- this.log = log.child({ prefix: 'AutoscaledPool' });
112
- // Configurable properties.
113
- this.desiredConcurrencyRatio = desiredConcurrencyRatio;
114
- this.scaleUpStepRatio = scaleUpStepRatio;
115
- this.scaleDownStepRatio = scaleDownStepRatio;
116
- this.maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;
117
- this.loggingIntervalMillis = loggingIntervalSecs * 1000;
118
- this.autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
119
- this.taskTimeoutMillis = taskTimeoutSecs * 1000;
120
- this.runTaskFunction = runTaskFunction;
121
- this.isFinishedFunction = isFinishedFunction;
122
- this.isTaskReadyFunction = isTaskReadyFunction;
123
- this.maxTasksPerMinute = maxTasksPerMinute;
124
- // Internal properties.
125
- this._minConcurrency = minConcurrency;
126
- this._maxConcurrency = maxConcurrency;
127
- this._desiredConcurrency = Math.min(desiredConcurrency ?? minConcurrency, maxConcurrency);
128
- this._currentConcurrency = 0;
129
- this.isStopped = false;
130
- this.resolve = null;
131
- this.reject = null;
132
- this._autoscale = this._autoscale.bind(this);
133
- this._maybeRunTask = this._maybeRunTask.bind(this);
134
- this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
135
- // Create instances with correct options.
136
- const ssoCopy = { ...systemStatusOptions };
137
- ssoCopy.snapshotter ??= new Snapshotter({
138
- ...snapshotterOptions,
139
- log: this.log,
140
- config: this.config,
141
- client: this.config.getStorageClient(),
142
- });
143
- ssoCopy.config ??= this.config;
144
- this.snapshotter = ssoCopy.snapshotter;
145
- this.systemStatus = new SystemStatus(ssoCopy);
146
- }
147
- /**
148
- * Gets the minimum number of tasks running in parallel.
149
- */
150
- get minConcurrency() {
151
- return this._minConcurrency;
152
- }
153
- /**
154
- * Sets the minimum number of tasks running in parallel.
155
- *
156
- * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
157
- * If you're not sure, just keep the default value and the concurrency will scale up automatically.
158
- */
159
- set minConcurrency(value) {
160
- ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
161
- this._minConcurrency = value;
162
- }
163
- /**
164
- * Gets the maximum number of tasks running in parallel.
165
- */
166
- get maxConcurrency() {
167
- return this._maxConcurrency;
168
- }
169
- /**
170
- * Sets the maximum number of tasks running in parallel.
171
- */
172
- set maxConcurrency(value) {
173
- ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
174
- this._maxConcurrency = value;
175
- }
176
- /**
177
- * Gets the desired concurrency for the pool,
178
- * which is an estimated number of parallel tasks that the system can currently support.
179
- */
180
- get desiredConcurrency() {
181
- return this._desiredConcurrency;
182
- }
183
- /**
184
- * Sets the desired concurrency for the pool, i.e. the number of tasks that should be running
185
- * in parallel if there's large enough supply of tasks.
186
- */
187
- set desiredConcurrency(value) {
188
- ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
189
- this._desiredConcurrency = value;
190
- }
191
- /**
192
- * Gets the number of parallel tasks currently running in the pool.
193
- */
194
- get currentConcurrency() {
195
- return this._currentConcurrency;
196
- }
197
- /**
198
- * Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
199
- * all the tasks are finished or one of them fails.
200
- */
201
- async run() {
202
- const poolPromise = new Promise((resolve, reject) => {
203
- this.resolve = resolve;
204
- this.reject = reject;
205
- });
206
- await this.snapshotter.start();
207
- // This interval checks the system status and updates the desired concurrency accordingly.
208
- this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
209
- // This is here because if we scale down to let's say 1, then after each promise is finished
210
- // this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
211
- // in the crawler getting stuck and even after scaling up it never triggers another promise.
212
- this.maybeRunInterval = betterSetInterval(this._maybeRunTask, this.maybeRunIntervalMillis);
213
- if (this.maxTasksPerMinute !== Infinity) {
214
- // Start the interval that resets the counter of tasks per minute.
215
- this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);
216
- }
217
- try {
218
- await poolPromise;
219
- }
220
- finally {
221
- // If resolve is null, the pool is already destroyed.
222
- if (this.resolve)
223
- await this._destroy();
224
- }
225
- }
226
- /**
227
- * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
228
- * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
229
- * will be spawned and all running tasks will be left in their current state.
230
- *
231
- * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
232
- * of all the running tasks, therefore, no abortion is attempted and some of the tasks
233
- * may finish, while others may not. Essentially, auto-scaled pool doesn't care about
234
- * their state after the invocation of `.abort()`, but that does not mean that some
235
- * parts of their asynchronous chains of commands will not execute.
236
- */
237
- async abort() {
238
- this.isStopped = true;
239
- if (this.resolve) {
240
- this.resolve();
241
- await this._destroy();
242
- }
243
- }
244
- /**
245
- * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
246
- * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
247
- *
248
- * The function's promise will resolve once all running tasks have completed and the pool
249
- * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
250
- * with a timeout error after the `timeoutSecs` seconds.
251
- *
252
- * The promise returned from the {@link AutoscaledPool.run} function will not resolve
253
- * when `.pause()` is invoked (unlike abort, which resolves it).
254
- */
255
- async pause(timeoutSecs) {
256
- if (this.isStopped)
257
- return;
258
- this.isStopped = true;
259
- await new Promise((resolve, reject) => {
260
- let timeout;
261
- if (timeoutSecs) {
262
- timeout = setTimeout(() => {
263
- const err = new Error("The pool's running tasks did not finish" +
264
- `in ${timeoutSecs} secs after pool.pause() invocation.`);
265
- reject(err);
266
- }, timeoutSecs);
267
- }
268
- const interval = setInterval(() => {
269
- if (this._currentConcurrency <= 0) {
270
- // Clean up timeout and interval to prevent process hanging.
271
- if (timeout)
272
- clearTimeout(timeout);
273
- clearInterval(interval);
274
- resolve();
275
- }
276
- }, this.maybeRunIntervalMillis);
277
- });
278
- }
279
- /**
280
- * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
281
- * Used together with {@link AutoscaledPool.pause}
282
- *
283
- * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
284
- */
285
- resume() {
286
- this.isStopped = false;
287
- }
288
- /**
289
- * Explicitly check the queue for new tasks. The AutoscaledPool checks the queue for new tasks periodically,
290
- * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
291
- */
292
- async notify() {
293
- setImmediate(this._maybeRunTask);
294
- }
295
- /**
296
- * Starts a new task
297
- * if the number of running tasks (current concurrency) is lower than desired concurrency
298
- * and the system is not currently overloaded
299
- * and this.isTaskReadyFunction() returns true.
300
- *
301
- * It doesn't allow multiple concurrent runs of this method.
302
- */
303
- async _maybeRunTask(intervalCallback) {
304
- this.log.perf('Attempting to run a task.');
305
- // Check if the function was invoked by the maybeRunInterval and use an empty function if not.
306
- const done = intervalCallback || (() => { });
307
- // Prevent starting a new task if:
308
- // - the pool is paused or aborted
309
- if (this.isStopped) {
310
- this.log.perf('Task will not run. AutoscaledPool is stopped.');
311
- return done();
312
- }
313
- // - we are already querying for a task.
314
- if (this.queryingIsTaskReady) {
315
- this.log.perf('Task will not run. Waiting for a ready task.');
316
- return done();
317
- }
318
- // - we would exceed desired concurrency.
319
- if (this._currentConcurrency >= this._desiredConcurrency) {
320
- this.log.perf('Task will not run. Desired concurrency achieved.');
321
- return done();
322
- }
323
- // - system is overloaded now and we are at or above minConcurrency
324
- const currentStatus = this.systemStatus.getCurrentStatus();
325
- const { isSystemIdle } = currentStatus;
326
- if (!isSystemIdle && this._currentConcurrency >= this._minConcurrency) {
327
- this.log.perf('Task will not be run. System is overloaded.', currentStatus);
328
- return done();
329
- }
330
- // - a task is ready.
331
- this.queryingIsTaskReady = true;
332
- let isTaskReady;
333
- try {
334
- this.log.perf('Checking for ready tasks.');
335
- isTaskReady = await this.isTaskReadyFunction();
336
- }
337
- catch (e) {
338
- const err = e;
339
- this.log.perf('Checking for ready tasks failed.');
340
- // We might have already rejected this promise.
341
- if (this.reject) {
342
- // No need to log all concurrent errors.
343
- this.log.exception(err, 'isTaskReadyFunction failed');
344
- this.reject(err);
345
- }
346
- }
347
- finally {
348
- this.queryingIsTaskReady = false;
349
- }
350
- if (!isTaskReady) {
351
- this.log.perf('Task will not run. No tasks are ready.');
352
- done();
353
- // No tasks could mean that we're finished with all tasks.
354
- return this._maybeFinish();
355
- }
356
- // - we have already reached the maximum tasks per minute
357
- // we need to check this *after* checking if a task is ready to prevent hanging the pool
358
- // for an extra minute if there are no more tasks
359
- if (this._isOverMaxRequestLimit) {
360
- this.log.perf('Task will not run. Maximum tasks per minute reached.');
361
- return done();
362
- }
363
- try {
364
- // Everything's fine. Run task.
365
- this._currentConcurrency++;
366
- this._tasksPerMinute[0]++;
367
- // Try to run next task to build up concurrency,
368
- // but defer it so it doesn't create a cycle.
369
- setImmediate(this._maybeRunTask);
370
- // We need to restart interval here, so that it doesn't get blocked by a stalled task.
371
- done();
372
- // Execute the current task.
373
- this.log.perf('Running a task.');
374
- if (this.taskTimeoutMillis > 0) {
375
- await addTimeoutToPromise(async () => this.runTaskFunction(), this.taskTimeoutMillis, `runTaskFunction timed out after ${this.taskTimeoutMillis / 1000} seconds.`);
376
- }
377
- else {
378
- await this.runTaskFunction();
379
- }
380
- this.log.perf('Task finished.');
381
- this._currentConcurrency--;
382
- // Run task after the previous one finished.
383
- setImmediate(this._maybeRunTask);
384
- }
385
- catch (e) {
386
- const err = e;
387
- this.log.perf('Running a task failed.');
388
- // We might have already rejected this promise.
389
- if (this.reject) {
390
- // No need to log all concurrent errors.
391
- if (
392
- // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
393
- !(e instanceof CriticalError)) {
394
- this.log.exception(err, 'runTaskFunction failed.');
395
- }
396
- this.reject(err);
397
- }
398
- }
399
- return undefined;
400
- }
401
- /**
402
- * Gets called every autoScaleIntervalSecs and evaluates the current system status.
403
- * If the system IS NOT overloaded and the settings allow it, it scales up.
404
- * If the system IS overloaded and the settings allow it, it scales down.
405
- */
406
- _autoscale(intervalCallback) {
407
- // Don't scale if paused.
408
- if (this.isStopped)
409
- return intervalCallback();
410
- // Don't scale if we've hit the maximum requests per minute
411
- if (this._isOverMaxRequestLimit)
412
- return intervalCallback();
413
- // Only scale up if:
414
- // - system has not been overloaded lately.
415
- const systemStatus = this.systemStatus.getHistoricalStatus();
416
- const { isSystemIdle } = systemStatus;
417
- // - we're not already at max concurrency.
418
- const weAreNotAtMax = this._desiredConcurrency < this._maxConcurrency;
419
- // - current concurrency reaches at least the given ratio of desired concurrency.
420
- const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);
421
- const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
422
- if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
423
- this._scaleUp(systemStatus);
424
- // Always scale down if:
425
- // - the system has been overloaded lately.
426
- const isSystemOverloaded = !isSystemIdle;
427
- // - we're over min concurrency.
428
- const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;
429
- if (isSystemOverloaded && weAreNotAtMin)
430
- this._scaleDown(systemStatus);
431
- // On periodic intervals, print comprehensive log information
432
- if (this.loggingIntervalMillis > 0) {
433
- const now = Date.now();
434
- if (this.lastLoggingTime == null) {
435
- this.lastLoggingTime = now;
436
- }
437
- else if (now > this.lastLoggingTime + this.loggingIntervalMillis) {
438
- this.lastLoggingTime = now;
439
- this.log.info('state', {
440
- currentConcurrency: this._currentConcurrency,
441
- desiredConcurrency: this._desiredConcurrency,
442
- systemStatus,
443
- });
444
- }
445
- }
446
- // Start a new interval cycle.
447
- return intervalCallback();
448
- }
449
- /**
450
- * Scales the pool up by increasing
451
- * the desired concurrency by the scaleUpStepRatio.
452
- *
453
- * @param systemStatus for logging
454
- */
455
- _scaleUp(systemStatus) {
456
- const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
457
- this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);
458
- this.log.debug('scaling up', {
459
- oldConcurrency: this._desiredConcurrency - step,
460
- newConcurrency: this._desiredConcurrency,
461
- systemStatus,
462
- });
463
- }
464
- /**
465
- * Scales the pool down by decreasing
466
- * the desired concurrency by the scaleDownStepRatio.
467
- *
468
- * @param systemStatus for logging
469
- */
470
- _scaleDown(systemStatus) {
471
- const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);
472
- this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);
473
- this.log.debug('scaling down', {
474
- oldConcurrency: this._desiredConcurrency + step,
475
- newConcurrency: this._desiredConcurrency,
476
- systemStatus,
477
- });
478
- }
479
- /**
480
- * If there are no running tasks and this.isFinishedFunction() returns true then closes
481
- * the pool and resolves the pool's promise returned by the run() method.
482
- *
483
- * It doesn't allow multiple concurrent runs of this method.
484
- */
485
- async _maybeFinish() {
486
- if (this.queryingIsFinished)
487
- return;
488
- if (this._currentConcurrency > 0)
489
- return;
490
- this.queryingIsFinished = true;
491
- try {
492
- const isFinished = await this.isFinishedFunction();
493
- if (isFinished && this.resolve)
494
- this.resolve();
495
- }
496
- catch (e) {
497
- const err = e;
498
- if (this.reject) {
499
- // No need to log all concurrent errors.
500
- this.log.exception(err, 'isFinishedFunction failed.');
501
- this.reject(err);
502
- }
503
- }
504
- finally {
505
- this.queryingIsFinished = false;
506
- }
507
- }
508
- /**
509
- * Cleans up resources.
510
- */
511
- async _destroy() {
512
- this.resolve = null;
513
- this.reject = null;
514
- betterClearInterval(this.autoscaleInterval);
515
- betterClearInterval(this.maybeRunInterval);
516
- if (this.tasksDonePerSecondInterval)
517
- betterClearInterval(this.tasksDonePerSecondInterval);
518
- if (this.snapshotter)
519
- await this.snapshotter.stop();
520
- }
521
- _incrementTasksDonePerSecond(intervalCallback) {
522
- this._tasksPerMinute.unshift(0);
523
- this._tasksPerMinute.pop();
524
- return intervalCallback();
525
- }
526
- get _isOverMaxRequestLimit() {
527
- if (this.maxTasksPerMinute === Infinity) {
528
- return false;
529
- }
530
- return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
531
- }
532
- }
533
- //# sourceMappingURL=autoscaled_pool.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"autoscaled_pool.js","sourceRoot":"","sources":["../../src/autoscaling/autoscaled_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAErD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1E,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAsHlD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAM,OAAO,cAAc;IAoCF;IAnCJ,GAAG,CAAM;IAE1B,2BAA2B;IACV,uBAAuB,CAAS;IAChC,gBAAgB,CAAS;IACzB,kBAAkB,CAAS;IAC3B,sBAAsB,CAAS;IAC/B,qBAAqB,CAAS;IAC9B,uBAAuB,CAAS;IAChC,iBAAiB,CAAS;IAC1B,eAAe,CAAyB;IACxC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,iBAAiB,CAAS;IAE3C,uBAAuB;IACf,eAAe,CAAS;IACxB,eAAe,CAAS;IACxB,mBAAmB,CAAS;IAC5B,mBAAmB,GAAG,CAAC,CAAC;IACxB,SAAS,GAAG,KAAK,CAAC;IAClB,eAAe,CAAU;IACzB,OAAO,GAAqC,IAAI,CAAC;IACjD,MAAM,GAAwC,IAAI,CAAC;IACnD,WAAW,CAAc;IACzB,YAAY,CAAe;IAC3B,iBAAiB,CAAoB;IACrC,gBAAgB,CAAoB;IACpC,mBAAmB,CAAW;IAC9B,kBAAkB,CAAW;IAC7B,0BAA0B,CAAoB;IAC9C,eAAe,GAAa,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAExE,YACI,OAA8B,EACb,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEzD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,EAAE,CAAC,QAAQ;YAC5B,kBAAkB,EAAE,EAAE,CAAC,QAAQ;YAC/B,mBAAmB,EAAE,EAAE,CAAC,QAAQ;YAChC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAChE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAChE,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACpE,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YACtE,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjE,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACvD,mBAAmB,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;YACzE,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACxD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACzD,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC,CAAC;SAChF,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GAAG,GAAG,EACpB,cAAc,GAAG,CAAC,EAClB,kBAAkB,EAClB,uBAAuB,GAAG,GAAG,EAC7B,gBAAgB,GAAG,IAAI,EACvB,kBAAkB,GAAG,IAAI,EACzB,oBAAoB,GAAG,GAAG,EAC1B,mBAAmB,GAAG,EAAE,EACxB,eAAe,GAAG,CAAC,EACnB,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,EACnB,kBAAkB,EAClB,GAAG,GAAG,UAAU,EAChB,iBAAiB,GAAG,QAAQ,GAC/B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,sBAAsB,GAAG,oBAAoB,GAAG,IAAI,CAAC;QAC1D,IAAI,CAAC,qBAAqB,GAAG,mBAAoB,GAAG,IAAI,CAAC;QACzD,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,uBAAuB;QACvB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB,IAAI,cAAc,EAAE,cAAc,CAAC,CAAC;QAC1F,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,4BAA4B,GAAG,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjF,yCAAyC;QACzC,MAAM,OAAO,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC3C,OAAO,CAAC,WAAW,KAAK,IAAI,WAAW,CAAC;YACpC,GAAG,kBAAkB;YACrB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE;SACzC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc,CAAC,KAAa;QAC5B,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,kBAAkB,CAAC,KAAa;QAChC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAI,kBAAkB;QAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,GAAG;QACL,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAChD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAE/B,0FAA0F;QAC1F,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAE1F,4FAA4F;QAC5F,gGAAgG;QAChG,4FAA4F;QAC5F,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3F,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE,CAAC;YACtC,kEAAkE;YAClE,IAAI,CAAC,0BAA0B,GAAG,iBAAiB,CAAC,IAAI,CAAC,4BAA4B,EAAE,IAAI,CAAC,CAAC;QACjG,CAAC;QAED,IAAI,CAAC;YACD,MAAM,WAAW,CAAC;QACtB,CAAC;gBAAS,CAAC;YACP,qDAAqD;YACrD,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5C,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,WAAoB;QAC5B,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,OAAuB,CAAC;YAC5B,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBACtB,MAAM,GAAG,GAAG,IAAI,KAAK,CACjB,yCAAyC;wBACrC,MAAM,WAAW,sCAAsC,CAC9D,CAAC;oBACF,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,EAAE,WAAW,CAAC,CAAC;YACpB,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC9B,IAAI,IAAI,CAAC,mBAAmB,IAAI,CAAC,EAAE,CAAC;oBAChC,4DAA4D;oBAC5D,IAAI,OAAO;wBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACnC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;gBACd,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM;QACR,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrC,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,aAAa,CAAC,gBAA6B;QACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC3C,8FAA8F;QAC9F,MAAM,IAAI,GAAG,gBAAgB,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE5C,kCAAkC;QAClC,kCAAkC;QAClC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;YAC/D,OAAO,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,wCAAwC;QACxC,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,yCAAyC;QACzC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACvD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YAClE,OAAO,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,mEAAmE;QACnE,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC;QACvC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACpE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE,aAAa,CAAC,CAAC;YAC5E,OAAO,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,qBAAqB;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,WAAW,CAAC;QAChB,IAAI,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YAC3C,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClD,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,IAAI,EAAE,CAAC;YACP,0DAA0D;YAC1D,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/B,CAAC;QAED,yDAAyD;QACzD,wFAAwF;QACxF,iDAAiD;QACjD,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YACtE,OAAO,IAAI,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,CAAC;YACD,+BAA+B;YAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1B,gDAAgD;YAChD,6CAA6C;YAC7C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAEjC,sFAAsF;YACtF,IAAI,EAAE,CAAC;YAEP,4BAA4B;YAC5B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,EAClC,IAAI,CAAC,iBAAiB,EACtB,mCAAmC,IAAI,CAAC,iBAAiB,GAAG,IAAI,WAAW,CAC9E,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YACjC,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAChC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,4CAA4C;YAC5C,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACxC,+CAA+C;YAC/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,wCAAwC;gBACxC;gBACI,6GAA6G;gBAC7G,CAAC,CAAC,CAAC,YAAY,aAAa,CAAC,EAC/B,CAAC;oBACC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;gBACvD,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;OAIG;IACO,UAAU,CAAC,gBAA4B;QAC7C,yBAAyB;QACzB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE9C,2DAA2D;QAC3D,IAAI,IAAI,CAAC,sBAAsB;YAAE,OAAO,gBAAgB,EAAE,CAAC;QAE3D,oBAAoB;QACpB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,CAAC;QAC7D,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,CAAC;QACtC,0CAA0C;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QACtE,iFAAiF;QACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAClG,MAAM,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,IAAI,qBAAqB,CAAC;QAE1F,IAAI,YAAY,IAAI,aAAa,IAAI,+BAA+B;YAAE,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElG,wBAAwB;QACxB,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC;QACzC,gCAAgC;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,eAAe,CAAC;QAEtE,IAAI,kBAAkB,IAAI,aAAa;YAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAEvE,6DAA6D;QAC7D,IAAI,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC/B,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC/B,CAAC;iBAAM,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBACjE,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;gBAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE;oBACnB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;oBAC5C,YAAY;iBACf,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,8BAA8B;QAC9B,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACO,QAAQ,CAAC,YAAwB;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACzE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;YACzB,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,UAAU,CAAC,YAAwB;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC3E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE;YAC3B,cAAc,EAAE,IAAI,CAAC,mBAAmB,GAAG,IAAI;YAC/C,cAAc,EAAE,IAAI,CAAC,mBAAmB;YACxC,YAAY;SACf,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,IAAI,CAAC,mBAAmB,GAAG,CAAC;YAAE,OAAO;QAEzC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,UAAU,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,EAAE,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,wCAAwC;gBACxC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACtD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,mBAAmB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,0BAA0B;YAAE,mBAAmB,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC1F,IAAI,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAES,4BAA4B,CAAC,gBAA4B;QAC/D,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;QAE3B,OAAO,gBAAgB,EAAE,CAAC;IAC9B,CAAC;IAED,IAAc,sBAAsB;QAChC,IAAI,IAAI,CAAC,iBAAiB,KAAK,QAAQ,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC;IAC/F,CAAC;CACJ"}
@@ -1,4 +0,0 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './snapshotter.js';
3
- export * from './system_status.js';
4
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/autoscaling/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC"}
@@ -1,4 +0,0 @@
1
- export * from './autoscaled_pool.js';
2
- export * from './snapshotter.js';
3
- export * from './system_status.js';
4
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/autoscaling/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC"}