apify 2.3.1-beta.4 → 3.0.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/README.md +6 -5
  2. package/package.json +69 -128
  3. package/build/actor.d.ts +0 -113
  4. package/build/actor.d.ts.map +0 -1
  5. package/build/actor.js +0 -582
  6. package/build/actor.js.map +0 -1
  7. package/build/apify.d.ts +0 -752
  8. package/build/apify.d.ts.map +0 -1
  9. package/build/apify.js +0 -877
  10. package/build/apify.js.map +0 -1
  11. package/build/autoscaling/autoscaled_pool.d.ts +0 -384
  12. package/build/autoscaling/autoscaled_pool.d.ts.map +0 -1
  13. package/build/autoscaling/autoscaled_pool.js +0 -557
  14. package/build/autoscaling/autoscaled_pool.js.map +0 -1
  15. package/build/autoscaling/snapshotter.d.ts +0 -278
  16. package/build/autoscaling/snapshotter.d.ts.map +0 -1
  17. package/build/autoscaling/snapshotter.js +0 -447
  18. package/build/autoscaling/snapshotter.js.map +0 -1
  19. package/build/autoscaling/system_status.d.ts +0 -224
  20. package/build/autoscaling/system_status.d.ts.map +0 -1
  21. package/build/autoscaling/system_status.js +0 -228
  22. package/build/autoscaling/system_status.js.map +0 -1
  23. package/build/browser_launchers/browser_launcher.d.ts +0 -154
  24. package/build/browser_launchers/browser_launcher.d.ts.map +0 -1
  25. package/build/browser_launchers/browser_launcher.js +0 -160
  26. package/build/browser_launchers/browser_launcher.js.map +0 -1
  27. package/build/browser_launchers/browser_plugin.d.ts +0 -23
  28. package/build/browser_launchers/browser_plugin.d.ts.map +0 -1
  29. package/build/browser_launchers/browser_plugin.js +0 -25
  30. package/build/browser_launchers/browser_plugin.js.map +0 -1
  31. package/build/browser_launchers/playwright_launcher.d.ts +0 -131
  32. package/build/browser_launchers/playwright_launcher.d.ts.map +0 -1
  33. package/build/browser_launchers/playwright_launcher.js +0 -150
  34. package/build/browser_launchers/playwright_launcher.js.map +0 -1
  35. package/build/browser_launchers/puppeteer_launcher.d.ts +0 -153
  36. package/build/browser_launchers/puppeteer_launcher.d.ts.map +0 -1
  37. package/build/browser_launchers/puppeteer_launcher.js +0 -197
  38. package/build/browser_launchers/puppeteer_launcher.js.map +0 -1
  39. package/build/cache_container.d.ts +0 -31
  40. package/build/cache_container.d.ts.map +0 -1
  41. package/build/cache_container.js +0 -48
  42. package/build/cache_container.js.map +0 -1
  43. package/build/configuration.d.ts +0 -226
  44. package/build/configuration.d.ts.map +0 -1
  45. package/build/configuration.js +0 -325
  46. package/build/configuration.js.map +0 -1
  47. package/build/constants.d.ts +0 -37
  48. package/build/constants.d.ts.map +0 -1
  49. package/build/constants.js +0 -41
  50. package/build/constants.js.map +0 -1
  51. package/build/crawlers/basic_crawler.d.ts +0 -443
  52. package/build/crawlers/basic_crawler.d.ts.map +0 -1
  53. package/build/crawlers/basic_crawler.js +0 -664
  54. package/build/crawlers/basic_crawler.js.map +0 -1
  55. package/build/crawlers/browser_crawler.d.ts +0 -512
  56. package/build/crawlers/browser_crawler.d.ts.map +0 -1
  57. package/build/crawlers/browser_crawler.js +0 -540
  58. package/build/crawlers/browser_crawler.js.map +0 -1
  59. package/build/crawlers/cheerio_crawler.d.ts +0 -931
  60. package/build/crawlers/cheerio_crawler.d.ts.map +0 -1
  61. package/build/crawlers/cheerio_crawler.js +0 -913
  62. package/build/crawlers/cheerio_crawler.js.map +0 -1
  63. package/build/crawlers/crawler_extension.d.ts +0 -10
  64. package/build/crawlers/crawler_extension.d.ts.map +0 -1
  65. package/build/crawlers/crawler_extension.js +0 -19
  66. package/build/crawlers/crawler_extension.js.map +0 -1
  67. package/build/crawlers/crawler_utils.d.ts +0 -34
  68. package/build/crawlers/crawler_utils.d.ts.map +0 -1
  69. package/build/crawlers/crawler_utils.js +0 -87
  70. package/build/crawlers/crawler_utils.js.map +0 -1
  71. package/build/crawlers/playwright_crawler.d.ts +0 -448
  72. package/build/crawlers/playwright_crawler.d.ts.map +0 -1
  73. package/build/crawlers/playwright_crawler.js +0 -299
  74. package/build/crawlers/playwright_crawler.js.map +0 -1
  75. package/build/crawlers/puppeteer_crawler.d.ts +0 -425
  76. package/build/crawlers/puppeteer_crawler.d.ts.map +0 -1
  77. package/build/crawlers/puppeteer_crawler.js +0 -299
  78. package/build/crawlers/puppeteer_crawler.js.map +0 -1
  79. package/build/crawlers/statistics.d.ts +0 -185
  80. package/build/crawlers/statistics.d.ts.map +0 -1
  81. package/build/crawlers/statistics.js +0 -331
  82. package/build/crawlers/statistics.js.map +0 -1
  83. package/build/enqueue_links/click_elements.d.ts +0 -179
  84. package/build/enqueue_links/click_elements.d.ts.map +0 -1
  85. package/build/enqueue_links/click_elements.js +0 -434
  86. package/build/enqueue_links/click_elements.js.map +0 -1
  87. package/build/enqueue_links/enqueue_links.d.ts +0 -117
  88. package/build/enqueue_links/enqueue_links.d.ts.map +0 -1
  89. package/build/enqueue_links/enqueue_links.js +0 -163
  90. package/build/enqueue_links/enqueue_links.js.map +0 -1
  91. package/build/enqueue_links/shared.d.ts +0 -42
  92. package/build/enqueue_links/shared.d.ts.map +0 -1
  93. package/build/enqueue_links/shared.js +0 -121
  94. package/build/enqueue_links/shared.js.map +0 -1
  95. package/build/errors.d.ts +0 -29
  96. package/build/errors.d.ts.map +0 -1
  97. package/build/errors.js +0 -38
  98. package/build/errors.js.map +0 -1
  99. package/build/events.d.ts +0 -11
  100. package/build/events.d.ts.map +0 -1
  101. package/build/events.js +0 -147
  102. package/build/events.js.map +0 -1
  103. package/build/index.d.ts +0 -4
  104. package/build/index.d.ts.map +0 -1
  105. package/build/index.js +0 -7
  106. package/build/index.js.map +0 -1
  107. package/build/main.d.ts +0 -179
  108. package/build/main.d.ts.map +0 -1
  109. package/build/main.js +0 -81
  110. package/build/main.js.map +0 -1
  111. package/build/playwright_utils.d.ts +0 -9
  112. package/build/playwright_utils.d.ts.map +0 -1
  113. package/build/playwright_utils.js +0 -90
  114. package/build/playwright_utils.js.map +0 -1
  115. package/build/proxy_configuration.d.ts +0 -411
  116. package/build/proxy_configuration.d.ts.map +0 -1
  117. package/build/proxy_configuration.js +0 -517
  118. package/build/proxy_configuration.js.map +0 -1
  119. package/build/pseudo_url.d.ts +0 -86
  120. package/build/pseudo_url.d.ts.map +0 -1
  121. package/build/pseudo_url.js +0 -153
  122. package/build/pseudo_url.js.map +0 -1
  123. package/build/puppeteer_request_interception.d.ts +0 -8
  124. package/build/puppeteer_request_interception.d.ts.map +0 -1
  125. package/build/puppeteer_request_interception.js +0 -235
  126. package/build/puppeteer_request_interception.js.map +0 -1
  127. package/build/puppeteer_utils.d.ts +0 -250
  128. package/build/puppeteer_utils.d.ts.map +0 -1
  129. package/build/puppeteer_utils.js +0 -551
  130. package/build/puppeteer_utils.js.map +0 -1
  131. package/build/request.d.ts +0 -180
  132. package/build/request.d.ts.map +0 -1
  133. package/build/request.js +0 -261
  134. package/build/request.js.map +0 -1
  135. package/build/request_list.d.ts +0 -581
  136. package/build/request_list.d.ts.map +0 -1
  137. package/build/request_list.js +0 -826
  138. package/build/request_list.js.map +0 -1
  139. package/build/serialization.d.ts +0 -5
  140. package/build/serialization.d.ts.map +0 -1
  141. package/build/serialization.js +0 -139
  142. package/build/serialization.js.map +0 -1
  143. package/build/session_pool/errors.d.ts +0 -11
  144. package/build/session_pool/errors.d.ts.map +0 -1
  145. package/build/session_pool/errors.js +0 -18
  146. package/build/session_pool/errors.js.map +0 -1
  147. package/build/session_pool/events.d.ts +0 -5
  148. package/build/session_pool/events.d.ts.map +0 -1
  149. package/build/session_pool/events.js +0 -6
  150. package/build/session_pool/events.js.map +0 -1
  151. package/build/session_pool/session.d.ts +0 -286
  152. package/build/session_pool/session.d.ts.map +0 -1
  153. package/build/session_pool/session.js +0 -355
  154. package/build/session_pool/session.js.map +0 -1
  155. package/build/session_pool/session_pool.d.ts +0 -280
  156. package/build/session_pool/session_pool.d.ts.map +0 -1
  157. package/build/session_pool/session_pool.js +0 -393
  158. package/build/session_pool/session_pool.js.map +0 -1
  159. package/build/session_pool/session_utils.d.ts +0 -4
  160. package/build/session_pool/session_utils.d.ts.map +0 -1
  161. package/build/session_pool/session_utils.js +0 -24
  162. package/build/session_pool/session_utils.js.map +0 -1
  163. package/build/stealth/hiding_tricks.d.ts +0 -22
  164. package/build/stealth/hiding_tricks.d.ts.map +0 -1
  165. package/build/stealth/hiding_tricks.js +0 -308
  166. package/build/stealth/hiding_tricks.js.map +0 -1
  167. package/build/stealth/stealth.d.ts +0 -56
  168. package/build/stealth/stealth.d.ts.map +0 -1
  169. package/build/stealth/stealth.js +0 -125
  170. package/build/stealth/stealth.js.map +0 -1
  171. package/build/storages/dataset.d.ts +0 -288
  172. package/build/storages/dataset.d.ts.map +0 -1
  173. package/build/storages/dataset.js +0 -480
  174. package/build/storages/dataset.js.map +0 -1
  175. package/build/storages/key_value_store.d.ts +0 -243
  176. package/build/storages/key_value_store.d.ts.map +0 -1
  177. package/build/storages/key_value_store.js +0 -462
  178. package/build/storages/key_value_store.js.map +0 -1
  179. package/build/storages/request_queue.d.ts +0 -318
  180. package/build/storages/request_queue.d.ts.map +0 -1
  181. package/build/storages/request_queue.js +0 -636
  182. package/build/storages/request_queue.js.map +0 -1
  183. package/build/storages/storage_manager.d.ts +0 -87
  184. package/build/storages/storage_manager.d.ts.map +0 -1
  185. package/build/storages/storage_manager.js +0 -150
  186. package/build/storages/storage_manager.js.map +0 -1
  187. package/build/tsconfig.tsbuildinfo +0 -1
  188. package/build/typedefs.d.ts +0 -146
  189. package/build/typedefs.d.ts.map +0 -1
  190. package/build/typedefs.js +0 -88
  191. package/build/typedefs.js.map +0 -1
  192. package/build/utils.d.ts +0 -175
  193. package/build/utils.d.ts.map +0 -1
  194. package/build/utils.js +0 -731
  195. package/build/utils.js.map +0 -1
  196. package/build/utils_log.d.ts +0 -41
  197. package/build/utils_log.d.ts.map +0 -1
  198. package/build/utils_log.js +0 -192
  199. package/build/utils_log.js.map +0 -1
  200. package/build/utils_request.d.ts +0 -77
  201. package/build/utils_request.d.ts.map +0 -1
  202. package/build/utils_request.js +0 -385
  203. package/build/utils_request.js.map +0 -1
  204. package/build/utils_social.d.ts +0 -210
  205. package/build/utils_social.d.ts.map +0 -1
  206. package/build/utils_social.js +0 -787
  207. package/build/utils_social.js.map +0 -1
  208. package/build/validators.d.ts +0 -23
  209. package/build/validators.d.ts.map +0 -1
  210. package/build/validators.js +0 -29
  211. package/build/validators.js.map +0 -1
@@ -1,447 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const tslib_1 = require("tslib");
4
- const os_1 = (0, tslib_1.__importDefault)(require("os"));
5
- const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
6
- const utilities_1 = require("@apify/utilities");
7
- const consts_1 = require("@apify/consts");
8
- const utils_1 = require("../utils");
9
- const events_1 = (0, tslib_1.__importDefault)(require("../events"));
10
- const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log")); // eslint-disable-line no-unused-vars
11
- const RESERVE_MEMORY_RATIO = 0.5;
12
- const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
13
- const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
14
- /**
15
- * @typedef SnapshotterOptions
16
- * @property {number} [eventLoopSnapshotIntervalSecs=0.5]
17
- * Defines the interval of measuring the event loop response time.
18
- * @property {number} [clientSnapshotIntervalSecs=1]
19
- * Defines the interval of checking the current state
20
- * of the remote API client.
21
- * @property {number} [maxBlockedMillis=50]
22
- * Maximum allowed delay of the event loop in milliseconds.
23
- * Exceeding this limit overloads the event loop.
24
- * @property {number} [cpuSnapshotIntervalSecs=1]
25
- * Defines the interval of measuring CPU usage.
26
- * This is only used when running locally. On the Apify platform,
27
- * the statistics are provided externally at a fixed interval.
28
- * @property {number} [maxUsedCpuRatio=0.95]
29
- * Defines the maximum usage of CPU.
30
- * Exceeding this limit overloads the CPU.
31
- * @property {number} [memorySnapshotIntervalSecs=1]
32
- * Defines the interval of measuring memory consumption.
33
- * This is only used when running locally. On the Apify platform,
34
- * the statistics are provided externally at a fixed interval.
35
- * The measurement itself is resource intensive (25 - 50ms async).
36
- * Therefore, setting this interval below 1 second is not recommended.
37
- * @property {number} [maxUsedMemoryRatio=0.7]
38
- * Defines the maximum ratio of total memory that can be used.
39
- * Exceeding this limit overloads the memory.
40
- * @property {number} [maxClientErrors=1]
41
- * Defines the maximum number of new rate limit errors within
42
- * the given interval.
43
- * @property {number} [snapshotHistorySecs=60]
44
- * Sets the interval in seconds for which a history of resource snapshots
45
- * will be kept. Increasing this to very high numbers will affect performance.
46
- */
47
- /**
48
- * Creates snapshots of system resources at given intervals and marks the resource
49
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
50
- * It tracks the following resources: Memory, EventLoop, API and CPU.
51
- * The class is used by the {@link AutoscaledPool} class.
52
- *
53
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
54
- * as collected from the running Docker container. When running locally, `Snapshotter`
55
- * makes its own statistics by querying the OS.
56
- *
57
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
58
- * when Apify platform marks it as overloaded.
59
- *
60
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
61
- * It's computed using the total memory available to the container when running on
62
- * the Apify platform and a quarter of total system memory when running locally.
63
- * Max total memory when running locally may be overridden by using the `APIFY_MEMORY_MBYTES`
64
- * environment variable.
65
- *
66
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
67
- *
68
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
69
- * typically received from the request queue, exceed the set limit within the set interval.
70
- */
71
- class Snapshotter {
72
- /**
73
- * @param {SnapshotterOptions} [options] All `Snapshotter` configuration options.
74
- */
75
- constructor(options = {}) {
76
- (0, ow_1.default)(options, ow_1.default.object.exactShape({
77
- eventLoopSnapshotIntervalSecs: ow_1.default.optional.number,
78
- cpuSnapshotIntervalSecs: ow_1.default.optional.number,
79
- memorySnapshotIntervalSecs: ow_1.default.optional.number,
80
- clientSnapshotIntervalSecs: ow_1.default.optional.number,
81
- snapshotHistorySecs: ow_1.default.optional.number,
82
- maxBlockedMillis: ow_1.default.optional.number,
83
- maxUsedMemoryRatio: ow_1.default.optional.number,
84
- maxUsedCpuRatio: ow_1.default.optional.number,
85
- maxClientErrors: ow_1.default.optional.number,
86
- log: ow_1.default.optional.object,
87
- }));
88
- const { eventLoopSnapshotIntervalSecs = 0.5, cpuSnapshotIntervalSecs = 1, memorySnapshotIntervalSecs = 1, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.7, maxUsedCpuRatio = 0.95, maxClientErrors = 3, log = utils_log_1.default, } = options;
89
- /** @type {Log} */
90
- this.log = log.child({ prefix: 'Snapshotter' });
91
- this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
92
- this.memorySnapshotIntervalMillis = memorySnapshotIntervalSecs * 1000;
93
- this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
94
- this.cpuSnapshotIntervalMillis = cpuSnapshotIntervalSecs * 1000;
95
- this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
96
- this.maxBlockedMillis = maxBlockedMillis;
97
- this.maxUsedMemoryRatio = maxUsedMemoryRatio;
98
- this.maxUsedCpuRatio = maxUsedCpuRatio;
99
- this.maxClientErrors = maxClientErrors;
100
- this.maxMemoryBytes = (parseInt(process.env[consts_1.ENV_VARS.MEMORY_MBYTES], 10) * 1024 * 1024) || null;
101
- this.cpuSnapshots = [];
102
- this.eventLoopSnapshots = [];
103
- this.memorySnapshots = [];
104
- this.clientSnapshots = [];
105
- this.eventLoopInterval = null;
106
- this.memoryInterval = null;
107
- this.clientInterval = null;
108
- this.cpuInterval = null;
109
- this.lastLoggedCriticalMemoryOverloadAt = null;
110
- // We need to pre-bind those functions to be able to successfully remove listeners.
111
- this._snapshotCpuOnPlatform = this._snapshotCpuOnPlatform.bind(this);
112
- this._snapshotMemoryOnPlatform = this._snapshotMemoryOnPlatform.bind(this);
113
- }
114
- /**
115
- * Starts capturing snapshots at configured intervals.
116
- * @return {Promise<void>}
117
- */
118
- async start() {
119
- await this._ensureCorrectMaxMemory();
120
- // Start snapshotting.
121
- this.eventLoopInterval = (0, utilities_1.betterSetInterval)(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
122
- this.clientInterval = (0, utilities_1.betterSetInterval)(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
123
- if ((0, utils_1.isAtHome)()) {
124
- events_1.default.on(consts_1.ACTOR_EVENT_NAMES.SYSTEM_INFO, this._snapshotCpuOnPlatform);
125
- events_1.default.on(consts_1.ACTOR_EVENT_NAMES.SYSTEM_INFO, this._snapshotMemoryOnPlatform);
126
- }
127
- else {
128
- this.cpuInterval = (0, utilities_1.betterSetInterval)(this._snapshotCpuOnLocal.bind(this), this.cpuSnapshotIntervalMillis);
129
- this.memoryInterval = (0, utilities_1.betterSetInterval)(this._snapshotMemoryOnLocal.bind(this), this.memorySnapshotIntervalMillis);
130
- }
131
- }
132
- /**
133
- * Stops all resource capturing.
134
- * @return {Promise<void>}
135
- */
136
- async stop() {
137
- (0, utilities_1.betterClearInterval)(this.eventLoopInterval);
138
- (0, utilities_1.betterClearInterval)(this.memoryInterval);
139
- (0, utilities_1.betterClearInterval)(this.cpuInterval);
140
- (0, utilities_1.betterClearInterval)(this.clientInterval);
141
- events_1.default.removeListener(consts_1.ACTOR_EVENT_NAMES.SYSTEM_INFO, this._snapshotCpuOnPlatform);
142
- events_1.default.removeListener(consts_1.ACTOR_EVENT_NAMES.SYSTEM_INFO, this._snapshotMemoryOnPlatform);
143
- // Allow microtask queue to unwind before stop returns.
144
- await new Promise((resolve) => setImmediate(resolve));
145
- }
146
- /**
147
- * Returns a sample of latest memory snapshots, with the size of the sample defined
148
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
149
- * @param {number} [sampleDurationMillis]
150
- * @return {Array<*>}
151
- */
152
- getMemorySample(sampleDurationMillis) {
153
- return this._getSample(this.memorySnapshots, sampleDurationMillis);
154
- }
155
- /**
156
- * Returns a sample of latest event loop snapshots, with the size of the sample defined
157
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
158
- * @param {number} [sampleDurationMillis]
159
- * @return {Array<*>}
160
- */
161
- getEventLoopSample(sampleDurationMillis) {
162
- return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
163
- }
164
- /**
165
- * Returns a sample of latest CPU snapshots, with the size of the sample defined
166
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
167
- * @param {number} [sampleDurationMillis]
168
- * @return {Array<*>}
169
- */
170
- getCpuSample(sampleDurationMillis) {
171
- return this._getSample(this.cpuSnapshots, sampleDurationMillis);
172
- }
173
- /**
174
- * Returns a sample of latest Client snapshots, with the size of the sample defined
175
- * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
176
- * @param {number} sampleDurationMillis
177
- * @return {Array<*>}
178
- */
179
- getClientSample(sampleDurationMillis) {
180
- return this._getSample(this.clientSnapshots, sampleDurationMillis);
181
- }
182
- /**
183
- * Finds the latest snapshots by sampleDurationMillis in the provided array.
184
- * @param {Array<*>} snapshots
185
- * @param {number} [sampleDurationMillis]
186
- * @return {Array<*>}
187
- * @ignore
188
- * @protected
189
- * @internal
190
- */
191
- _getSample(snapshots, sampleDurationMillis) {
192
- if (!sampleDurationMillis)
193
- return snapshots;
194
- const sample = [];
195
- let idx = snapshots.length;
196
- if (!idx)
197
- return sample;
198
- const latestTime = snapshots[idx - 1].createdAt;
199
- while (idx--) {
200
- const snapshot = snapshots[idx];
201
- if (latestTime - snapshot.createdAt <= sampleDurationMillis)
202
- sample.unshift(snapshot);
203
- else
204
- break;
205
- }
206
- return sample;
207
- }
208
- /**
209
- * Creates a snapshot of current memory usage
210
- * using the Apify platform `systemInfo` event.
211
- * @param {*} systemInfo
212
- * @ignore
213
- * @protected
214
- * @internal
215
- */
216
- _snapshotMemoryOnPlatform(systemInfo) {
217
- const now = new Date();
218
- this._pruneSnapshots(this.memorySnapshots, now);
219
- const { memCurrentBytes } = systemInfo;
220
- const snapshot = {
221
- createdAt: now,
222
- isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
223
- usedBytes: memCurrentBytes,
224
- };
225
- this.memorySnapshots.push(snapshot);
226
- this._memoryOverloadWarning(systemInfo);
227
- }
228
- /**
229
- * Creates a snapshot of current memory usage
230
- * using the Apify platform `systemInfo` event.
231
- * @param {Function} intervalCallback
232
- * @return {Promise<void>}
233
- * @ignore
234
- * @protected
235
- * @internal
236
- */
237
- async _snapshotMemoryOnLocal(intervalCallback) {
238
- try {
239
- const now = new Date();
240
- const memInfo = await (0, utils_1.getMemoryInfo)();
241
- const { mainProcessBytes, childProcessesBytes } = memInfo;
242
- this._pruneSnapshots(this.memorySnapshots, now);
243
- const usedBytes = mainProcessBytes + childProcessesBytes;
244
- const snapshot = {
245
- createdAt: now,
246
- isOverloaded: usedBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
247
- usedBytes,
248
- };
249
- this.memorySnapshots.push(snapshot);
250
- }
251
- catch (err) {
252
- this.log.exception(err, 'Memory snapshot failed.');
253
- }
254
- finally {
255
- intervalCallback();
256
- }
257
- }
258
- /**
259
- * Checks for critical memory overload and logs it to the console.
260
- * @param {*} systemInfo
261
- * @ignore
262
- * @protected
263
- * @internal
264
- */
265
- _memoryOverloadWarning({ memCurrentBytes }) {
266
- const now = new Date();
267
- if (this.lastLoggedCriticalMemoryOverloadAt && now.getTime() < this.lastLoggedCriticalMemoryOverloadAt.getTime()
268
- + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
269
- return;
270
- const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
271
- const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
272
- const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
273
- const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
274
- if (isCriticalOverload) {
275
- const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
276
- const toMb = (bytes) => Math.round(bytes / (1024 ** 2));
277
- this.log.warning('Memory is critically overloaded. '
278
- + `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing the actor memory.`);
279
- this.lastLoggedCriticalMemoryOverloadAt = now;
280
- }
281
- }
282
- /**
283
- * Creates a snapshot of current event loop delay.
284
- * @param {Function} intervalCallback
285
- * @ignore
286
- * @protected
287
- * @internal
288
- */
289
- _snapshotEventLoop(intervalCallback) {
290
- const now = new Date();
291
- this._pruneSnapshots(this.eventLoopSnapshots, now);
292
- const snapshot = {
293
- createdAt: now,
294
- isOverloaded: false,
295
- exceededMillis: 0,
296
- };
297
- const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
298
- if (previousSnapshot) {
299
- const { createdAt } = previousSnapshot;
300
- const delta = now.getTime() - createdAt - this.eventLoopSnapshotIntervalMillis;
301
- if (delta > this.maxBlockedMillis)
302
- snapshot.isOverloaded = true;
303
- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
304
- }
305
- this.eventLoopSnapshots.push(snapshot);
306
- intervalCallback();
307
- }
308
- /**
309
- * Creates a snapshot of current CPU usage
310
- * using the Apify platform `systemInfo` event.
311
- * @param {Object} systemInfo
312
- * @ignore
313
- * @protected
314
- * @internal
315
- */
316
- _snapshotCpuOnPlatform(systemInfo) {
317
- const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
318
- const createdAt = (new Date(systemInfo.createdAt));
319
- this._pruneSnapshots(this.cpuSnapshots, createdAt);
320
- this.cpuSnapshots.push({
321
- createdAt,
322
- isOverloaded: isCpuOverloaded,
323
- usedRatio: Math.ceil(cpuCurrentUsage / 100),
324
- });
325
- }
326
- /**
327
- * Creates a snapshot of current CPU usage
328
- * using OS provided metrics.
329
- * @param {Function} intervalCallback
330
- * @ignore
331
- * @protected
332
- * @internal
333
- */
334
- _snapshotCpuOnLocal(intervalCallback) {
335
- const now = new Date();
336
- this._pruneSnapshots(this.cpuSnapshots, now);
337
- const ticks = this._getCurrentCpuTicks();
338
- const snapshot = {
339
- createdAt: now,
340
- isOverloaded: false,
341
- ticks,
342
- usedRatio: 0,
343
- };
344
- const previousSnapshot = this.cpuSnapshots[this.cpuSnapshots.length - 1];
345
- if (previousSnapshot) {
346
- const { ticks: prevTicks } = previousSnapshot;
347
- const idleTicksDelta = ticks.idle - prevTicks.idle;
348
- const totalTicksDelta = ticks.total - prevTicks.total;
349
- const usedCpuRatio = 1 - (idleTicksDelta / totalTicksDelta);
350
- if (usedCpuRatio > this.maxUsedCpuRatio)
351
- snapshot.isOverloaded = true;
352
- snapshot.usedRatio = Math.ceil(usedCpuRatio);
353
- }
354
- this.cpuSnapshots.push(snapshot);
355
- intervalCallback();
356
- }
357
- /**
358
- * @ignore
359
- * @protected
360
- * @internal
361
- */
362
- _getCurrentCpuTicks() {
363
- const cpus = os_1.default.cpus();
364
- return cpus.reduce((acc, cpu) => {
365
- const cpuTimes = Object.values(cpu.times);
366
- return {
367
- idle: acc.idle + cpu.times.idle,
368
- total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
369
- };
370
- }, { idle: 0, total: 0 });
371
- }
372
- /**
373
- * Creates a snapshot of current API state by checking for
374
- * rate limit errors. Only errors produced by a 2nd retry
375
- * of the API call are considered for snapshotting since
376
- * earlier errors may just be caused by a random spike in
377
- * number of requests and do not necessarily signify API
378
- * overloading.
379
- *
380
- * @param intervalCallback
381
- * @ignore
382
- * @protected
383
- * @internal
384
- */
385
- _snapshotClient(intervalCallback) {
386
- const now = new Date();
387
- this._pruneSnapshots(this.clientSnapshots, now);
388
- const allErrorCounts = utils_1.apifyClient.stats.rateLimitErrors;
389
- const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
390
- // Handle empty snapshots array
391
- const snapshot = {
392
- createdAt: now,
393
- isOverloaded: false,
394
- rateLimitErrorCount: currentErrCount,
395
- };
396
- const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
397
- if (previousSnapshot) {
398
- const { rateLimitErrorCount } = previousSnapshot;
399
- const delta = currentErrCount - rateLimitErrorCount;
400
- if (delta > this.maxClientErrors)
401
- snapshot.isOverloaded = true;
402
- }
403
- this.clientSnapshots.push(snapshot);
404
- intervalCallback();
405
- }
406
- /**
407
- * Removes snapshots that are older than the snapshotHistorySecs option
408
- * from the array (destructively - in place).
409
- * @param {Array<*>} snapshots
410
- * @param {Date} now
411
- * @ignore
412
- * @protected
413
- * @internal
414
- */
415
- _pruneSnapshots(snapshots, now) {
416
- let oldCount = 0;
417
- for (let i = 0; i < snapshots.length; i++) {
418
- const { createdAt } = snapshots[i];
419
- if (now.getTime() - createdAt > this.snapshotHistoryMillis)
420
- oldCount++;
421
- else
422
- break;
423
- }
424
- snapshots.splice(0, oldCount);
425
- }
426
- /**
427
- * Calculate max memory for platform or local usage.
428
- * @ignore
429
- * @protected
430
- * @internal
431
- */
432
- async _ensureCorrectMaxMemory() {
433
- if (this.maxMemoryBytes)
434
- return;
435
- const { totalBytes } = await (0, utils_1.getMemoryInfo)();
436
- if ((0, utils_1.isAtHome)()) {
437
- this.maxMemoryBytes = totalBytes;
438
- }
439
- else {
440
- this.maxMemoryBytes = Math.ceil(totalBytes / 4);
441
- // NOTE: Log as AutoscaledPool, so that users are not confused what "Snapshotter" is
442
- this.log.info(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the ${consts_1.ENV_VARS.MEMORY_MBYTES} environment variable to override it.`); // eslint-disable-line max-len
443
- }
444
- }
445
- }
446
- exports.default = Snapshotter;
447
- //# sourceMappingURL=snapshotter.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,yDAAoB;AACpB,gDAA0E;AAC1E,0CAA4D;AAC5D,oCAAgE;AAChE,oEAA+B;AAC/B,0EAA+C,CAAC,qCAAqC;AAErF,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AAElD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW;IACb;;OAEG;IACH,YAAY,OAAO,GAAG,EAAE;QACpB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC3C,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,uBAAuB,GAAG,CAAC,EAC3B,0BAA0B,GAAG,CAAC,EAC9B,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,IAAI,EACtB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,mBAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,kBAAkB;QAClB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,yBAAyB,GAAG,uBAAuB,GAAG,IAAI,CAAC;QAChE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC;QAEhG,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAE1B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,IAAI,CAAC,kCAAkC,GAAG,IAAI,CAAC;QAE/C,mFAAmF;QACnF,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/E,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAErC,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,IAAA,gBAAQ,GAAE,EAAE;YACZ,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;YACtE,gBAAM,CAAC,EAAE,CAAC,0BAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,yBAAyB,CAAC,CAAC;SAC5E;aAAM;YACH,IAAI,CAAC,WAAW,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,yBAAyB,CAAC,CAAC;YAC1G,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAA,+BAAmB,EAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,gBAAM,CAAC,cAAc,CAAC,0BAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAClF,gBAAM,CAAC,cAAc,CAAC,0BAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACrF,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,oBAAoB;QAChC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,oBAAoB;QACnC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,oBAAoB;QAC7B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,oBAAoB;QAChC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,SAAS,EAAE,oBAAoB;QACtC,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,UAAU,GAAG,QAAQ,CAAC,SAAS,IAAI,oBAAoB;gBAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;;gBACjF,MAAM;SACd;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;OAOG;IACH,yBAAyB,CAAC,UAAU;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAChD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,eAAe,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,kBAAkB;YAC7E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,sBAAsB,CAAC,gBAAgB;QACzC,IAAI;YACA,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,MAAM,IAAA,qBAAa,GAAE,CAAC;YACtC,MAAM,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC;YAC1D,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YAEhD,MAAM,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;YACzD,MAAM,QAAQ,GAAG;gBACb,SAAS,EAAE,GAAG;gBACd,YAAY,EAAE,SAAS,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,kBAAkB;gBACvE,SAAS;aACZ,CAAC;YAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SACvC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yBAAyB,CAAC,CAAC;SACtD;gBAAS;YACN,gBAAgB,EAAE,CAAC;SACtB;IACL,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,EAAE,eAAe,EAAE;QACtC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,kCAAkC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,kCAAkC,CAAC,OAAO,EAAE;cAC1G,mCAAmC;YAAE,OAAO;QAElD,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC;QAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QACjG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAe,GAAG,qBAAqB,CAAC;QACnE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC;YACjF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC1I,IAAI,CAAC,kCAAkC,GAAG,GAAG,CAAC;SACjD;IACL,CAAC;IAED;;;;;;OAMG;IACH,kBAAkB,CAAC,gBAAgB;QAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAE/E,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;OAOG;IACH,sBAAsB,CAAC,UAAU;QAC7B,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAe;YAC7B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;SAC9C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACH,mBAAmB,CAAC,gBAAgB;QAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QAE7C,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,KAAK;YACL,SAAS,EAAE,CAAC;SACf,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzE,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;YACnD,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;YACtD,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC;YAE5D,IAAI,YAAY,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YACtE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SAChD;QAED,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,mBAAmB;QACf,MAAM,IAAI,GAAG,YAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,gBAAgB;QAC5B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,mBAAW,CAAC,KAAK,CAAC,eAAe,CAAC;QACzD,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;OAQG;IACH,eAAe,CAAC,SAAS,EAAE,GAAG;QAC1B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBAClE,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,uBAAuB;QACzB,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO;QAChC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAA,qBAAa,GAAE,CAAC;QAC7C,IAAI,IAAA,gBAAQ,GAAE,EAAE;YACZ,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;SACpC;aAAM;YACH,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;YAChD,oFAAoF;YACpF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,gBAAgB,iBAAQ,CAAC,aAAa,uCAAuC,CAAC,CAAC,CAAC,8BAA8B;SACjN;IACL,CAAC;CACJ;AAED,kBAAe,WAAW,CAAC"}
@@ -1,224 +0,0 @@
1
- export default SystemStatus;
2
- /**
3
- * Represents the current status of the system.
4
- */
5
- export type SystemInfo = {
6
- /**
7
- * If true, system is being overloaded.
8
- */
9
- isSystemIdle: boolean;
10
- /**
11
- * Memory
12
- */
13
- memInfo: {
14
- [x: string]: any;
15
- };
16
- eventLoopInfo: {
17
- [x: string]: any;
18
- };
19
- cpuInfo: {
20
- [x: string]: any;
21
- };
22
- };
23
- export type SystemStatusOptions = {
24
- /**
25
- * Defines max age of snapshots used in the
26
- * {@link SystemStatusgetCurrentStatus } measurement.
27
- */
28
- currentHistorySecs?: number | undefined;
29
- /**
30
- * Sets the maximum ratio of overloaded snapshots in a memory sample.
31
- * If the sample exceeds this ratio, the system will be overloaded.
32
- */
33
- maxMemoryOverloadedRatio?: number | undefined;
34
- /**
35
- * Sets the maximum ratio of overloaded snapshots in an event loop sample.
36
- * If the sample exceeds this ratio, the system will be overloaded.
37
- */
38
- maxEventLoopOverloadedRatio?: number | undefined;
39
- /**
40
- * Sets the maximum ratio of overloaded snapshots in a CPU sample.
41
- * If the sample exceeds this ratio, the system will be overloaded.
42
- */
43
- maxCpuOverloadedRatio?: number | undefined;
44
- /**
45
- * Sets the maximum ratio of overloaded snapshots in a Client sample.
46
- * If the sample exceeds this ratio, the system will be overloaded.
47
- */
48
- maxClientOverloadedRatio?: number | undefined;
49
- /**
50
- * The `Snapshotter` instance to be queried for `SystemStatus`.
51
- */
52
- snapshotter?: Snapshotter | undefined;
53
- };
54
- /**
55
- * Represents the current status of the system.
56
- *
57
- * @typedef SystemInfo
58
- * @property {boolean} isSystemIdle
59
- * If true, system is being overloaded.
60
- * @property {Object<string, *>} memInfo
61
- * Memory
62
- * @property {Object<string, *>} eventLoopInfo
63
- * @property {Object<string, *>} cpuInfo
64
- */
65
- /**
66
- * @typedef SystemStatusOptions
67
- * @property {number} [currentHistorySecs=5]
68
- * Defines max age of snapshots used in the
69
- * {@link SystemStatus#getCurrentStatus} measurement.
70
- * @property {number} [maxMemoryOverloadedRatio=0.2]
71
- * Sets the maximum ratio of overloaded snapshots in a memory sample.
72
- * If the sample exceeds this ratio, the system will be overloaded.
73
- * @property {number} [maxEventLoopOverloadedRatio=0.6]
74
- * Sets the maximum ratio of overloaded snapshots in an event loop sample.
75
- * If the sample exceeds this ratio, the system will be overloaded.
76
- * @property {number} [maxCpuOverloadedRatio=0.4]
77
- * Sets the maximum ratio of overloaded snapshots in a CPU sample.
78
- * If the sample exceeds this ratio, the system will be overloaded.
79
- * @property {number} [maxClientOverloadedRatio=0.3]
80
- * Sets the maximum ratio of overloaded snapshots in a Client sample.
81
- * If the sample exceeds this ratio, the system will be overloaded.
82
- * @property {Snapshotter} [snapshotter]
83
- * The `Snapshotter` instance to be queried for `SystemStatus`.
84
- */
85
- /**
86
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
87
- * It only exposes two functions {@link SystemStatus#getCurrentStatus}
88
- * and {@link SystemStatus#getHistoricalStatus}.
89
- * The system status is calculated using a weighted average of overloaded
90
- * messages in the snapshots, with the weights being the time intervals
91
- * between the snapshots. Each resource is calculated separately
92
- * and the system is overloaded whenever at least one resource is overloaded.
93
- * The class is used by the {@link AutoscaledPool} class.
94
- *
95
- * {@link SystemStatus#getCurrentStatus}
96
- * returns a boolean that represents the current status of the system.
97
- * The length of the current timeframe in seconds is configurable
98
- * by the `currentHistorySecs` option and represents the max age
99
- * of snapshots to be considered for the calculation.
100
- *
101
- * {@link SystemStatus#getHistoricalStatus}
102
- * returns a boolean that represents the long-term status
103
- * of the system. It considers the full snapshot history available
104
- * in the {@link Snapshotter} instance.
105
- */
106
- declare class SystemStatus {
107
- /**
108
- * @param {SystemStatusOptions} [options] All `SystemStatus` configuration options.
109
- */
110
- constructor(options?: SystemStatusOptions | undefined);
111
- currentHistorySecs: number;
112
- maxMemoryOverloadedRatio: number;
113
- maxEventLoopOverloadedRatio: number;
114
- maxCpuOverloadedRatio: number;
115
- maxClientOverloadedRatio: number;
116
- snapshotter: Snapshotter;
117
- /**
118
- * Returns an {@link SystemInfo} object with the following structure:
119
- *
120
- * ```javascript
121
- * {
122
- * isSystemIdle: Boolean,
123
- * memInfo: Object,
124
- * eventLoopInfo: Object,
125
- * cpuInfo: Object
126
- * }
127
- * ```
128
- *
129
- * Where the `isSystemIdle` property is set to `false` if the system
130
- * has been overloaded in the last `options.currentHistorySecs` seconds,
131
- * and `true` otherwise.
132
- * @return {SystemInfo}
133
- */
134
- getCurrentStatus(): SystemInfo;
135
- /**
136
- * Returns an {@link SystemInfo} object with the following structure:
137
- *
138
- * ```javascript
139
- * {
140
- * isSystemIdle: Boolean,
141
- * memInfo: Object,
142
- * eventLoopInfo: Object,
143
- * cpuInfo: Object
144
- * }
145
- * ```
146
- *
147
- * Where the `isSystemIdle` property is set to `false` if the system
148
- * has been overloaded in the full history of the {@link Snapshotter}
149
- * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
150
- * @return {SystemInfo}
151
- */
152
- getHistoricalStatus(): SystemInfo;
153
- /**
154
- * Returns a system status object.
155
- *
156
- * @param {number} [sampleDurationMillis]
157
- * @return {SystemInfo}
158
- * @ignore
159
- * @protected
160
- * @internal
161
- */
162
- protected _isSystemIdle(sampleDurationMillis?: number | undefined): SystemInfo;
163
- /**
164
- * Returns an object with an isOverloaded property set to true
165
- * if the memory has been overloaded in the last sampleDurationMillis.
166
- *
167
- * @param {number} sampleDurationMillis
168
- * @return {object}
169
- * @ignore
170
- * @protected
171
- * @internal
172
- */
173
- protected _isMemoryOverloaded(sampleDurationMillis: number): object;
174
- /**
175
- * Returns an object with an isOverloaded property set to true
176
- * if the event loop has been overloaded in the last sampleDurationMillis.
177
- *
178
- * @param {number} sampleDurationMillis
179
- * @return {object}
180
- * @ignore
181
- * @protected
182
- * @internal
183
- */
184
- protected _isEventLoopOverloaded(sampleDurationMillis: number): object;
185
- /**
186
- * Returns an object with an isOverloaded property set to true
187
- * if the CPU has been overloaded in the last sampleDurationMillis.
188
- *
189
- * @param {number} sampleDurationMillis
190
- * @return {object}
191
- * @ignore
192
- * @protected
193
- * @internal
194
- */
195
- protected _isCpuOverloaded(sampleDurationMillis: number): object;
196
- /**
197
- * Returns an object with an isOverloaded property set to true
198
- * if the client has been overloaded in the last sampleDurationMillis.
199
- * @param {number} sampleDurationMillis
200
- * @return {{isOverloaded: boolean, maxOverloadedRatio: number, actualRatio: number}}
201
- * @ignore
202
- * @protected
203
- * @internal
204
- */
205
- protected _isClientOverloaded(sampleDurationMillis: number): {
206
- isOverloaded: boolean;
207
- maxOverloadedRatio: number;
208
- actualRatio: number;
209
- };
210
- /**
211
- * Returns an object with sample information and an isOverloaded property
212
- * set to true if at least the ratio of snapshots in the sample are overloaded.
213
- *
214
- * @param {Array<*>} sample
215
- * @param {number} ratio
216
- * @return {object}
217
- * @ignore
218
- * @protected
219
- * @internal
220
- */
221
- protected _isSampleOverloaded(sample: Array<any>, ratio: number): object;
222
- }
223
- import Snapshotter from "./snapshotter";
224
- //# sourceMappingURL=system_status.d.ts.map