@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +3 -21
  2. package/autoscaling/autoscaled_pool.js +85 -85
  3. package/autoscaling/client_load_signal.d.ts +1 -5
  4. package/autoscaling/client_load_signal.js +20 -20
  5. package/autoscaling/concurrency_system.d.ts +5 -20
  6. package/autoscaling/concurrency_system.js +81 -80
  7. package/autoscaling/cpu_load_signal.d.ts +1 -2
  8. package/autoscaling/cpu_load_signal.js +10 -10
  9. package/autoscaling/event_loop_load_signal.d.ts +1 -4
  10. package/autoscaling/event_loop_load_signal.js +18 -18
  11. package/autoscaling/load_signal.d.ts +1 -1
  12. package/autoscaling/load_signal.js +12 -11
  13. package/autoscaling/memory_load_signal.d.ts +3 -12
  14. package/autoscaling/memory_load_signal.js +40 -41
  15. package/autoscaling/snapshotter.d.ts +1 -4
  16. package/autoscaling/snapshotter.js +12 -12
  17. package/autoscaling/system_status.d.ts +1 -3
  18. package/autoscaling/system_status.js +11 -11
  19. package/configuration.d.ts +1 -1
  20. package/configuration.js +3 -3
  21. package/crawlers/context_pipeline.js +6 -6
  22. package/crawlers/crawler_commons.d.ts +6 -56
  23. package/crawlers/crawler_commons.js +1 -107
  24. package/crawlers/index.d.ts +1 -1
  25. package/crawlers/index.js +0 -1
  26. package/crawlers/statistics.d.ts +1 -8
  27. package/crawlers/statistics.js +45 -44
  28. package/events/event_manager.d.ts +1 -1
  29. package/events/event_manager.js +3 -3
  30. package/events/local_event_manager.d.ts +1 -1
  31. package/events/local_event_manager.js +3 -3
  32. package/log.js +5 -1
  33. package/memory-storage/memory-storage.d.ts +1 -5
  34. package/memory-storage/memory-storage.js +2 -2
  35. package/memory-storage/resource-clients/dataset.d.ts +1 -1
  36. package/memory-storage/resource-clients/dataset.js +6 -5
  37. package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
  38. package/memory-storage/resource-clients/key-value-store.js +13 -12
  39. package/memory-storage/resource-clients/request-queue.d.ts +4 -23
  40. package/memory-storage/resource-clients/request-queue.js +59 -58
  41. package/owned_or_injected.d.ts +1 -3
  42. package/owned_or_injected.js +17 -17
  43. package/package.json +5 -5
  44. package/proxy_configuration.d.ts +1 -3
  45. package/proxy_configuration.js +8 -8
  46. package/recoverable_state.d.ts +1 -10
  47. package/recoverable_state.js +41 -41
  48. package/request.d.ts +1 -2
  49. package/request.js +10 -13
  50. package/router.d.ts +1 -4
  51. package/router.js +23 -23
  52. package/serialization.js +8 -9
  53. package/service_locator.d.ts +1 -10
  54. package/service_locator.js +48 -48
  55. package/session_pool/session.d.ts +1 -12
  56. package/session_pool/session.js +50 -50
  57. package/session_pool/session_pool.d.ts +2 -11
  58. package/session_pool/session_pool.js +59 -58
  59. package/storages/dataset.d.ts +12 -1
  60. package/storages/dataset.js +121 -22
  61. package/storages/index.d.ts +1 -1
  62. package/storages/index.js +1 -1
  63. package/storages/key_value_store.d.ts +19 -4
  64. package/storages/key_value_store.js +174 -48
  65. package/storages/request_dedup_cache.d.ts +1 -2
  66. package/storages/request_dedup_cache.js +9 -9
  67. package/storages/request_list.d.ts +2 -22
  68. package/storages/request_list.js +74 -73
  69. package/storages/request_manager_tandem.d.ts +1 -10
  70. package/storages/request_manager_tandem.js +27 -27
  71. package/storages/request_queue.d.ts +21 -18
  72. package/storages/request_queue.js +256 -53
  73. package/storages/sitemap_request_loader.d.ts +1 -44
  74. package/storages/sitemap_request_loader.js +87 -87
  75. package/storages/storage_instance_manager.d.ts +1 -2
  76. package/storages/storage_instance_manager.js +17 -17
  77. package/storages/storage_stats.d.ts +1 -1
  78. package/storages/storage_stats.js +4 -4
  79. package/storages/transaction.d.ts +252 -0
  80. package/storages/transaction.js +251 -0
  81. package/system-info/runtime.js +7 -7
  82. package/storages/access_checking.d.ts +0 -12
  83. package/storages/access_checking.js +0 -17
@@ -22,26 +22,27 @@ export class ConcurrencySystem {
22
22
  desiredConcurrencyRatio;
23
23
  scaleUpStepRatio;
24
24
  scaleDownStepRatio;
25
- loggingIntervalMillis;
26
- autoscaleIntervalMillis;
25
+ #loggingIntervalMillis;
26
+ #autoscaleIntervalMillis;
27
27
  maxTasksPerMinute;
28
- _minConcurrency;
29
- _maxConcurrency;
30
- _desiredConcurrency;
28
+ #minConcurrency;
29
+ #maxConcurrency;
30
+ #desiredConcurrency;
31
+ // kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directly
31
32
  _currentConcurrency = 0;
32
- lastLoggingTime;
33
- _tasksPerMinute = Array.from({ length: 60 }, () => 0);
33
+ #lastLoggingTime;
34
+ #tasksPerMinute = Array.from({ length: 60 }, () => 0);
34
35
  snapshotter;
35
- loadSignals;
36
+ #loadSignals;
36
37
  systemStatus;
37
- autoscaleInterval;
38
- tasksDonePerSecondInterval;
38
+ #autoscaleInterval;
39
+ #tasksDonePerSecondInterval;
39
40
  /** Whether the snapshotter and autoscaling intervals are currently running. */
40
- running = false;
41
+ #running = false;
41
42
  /** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
42
- startPromise;
43
+ #startPromise;
43
44
  /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
44
- warnedAboutQueryWhileStopped = false;
45
+ #warnedAboutQueryWhileStopped = false;
45
46
  constructor(options = {}) {
46
47
  ow(options, ow.object.exactShape({
47
48
  maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
@@ -63,19 +64,19 @@ export class ConcurrencySystem {
63
64
  this.desiredConcurrencyRatio = desiredConcurrencyRatio;
64
65
  this.scaleUpStepRatio = scaleUpStepRatio;
65
66
  this.scaleDownStepRatio = scaleDownStepRatio;
66
- this.loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
67
- this.autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
67
+ this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
68
+ this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
68
69
  this.maxTasksPerMinute = maxTasksPerMinute;
69
- this._minConcurrency = minConcurrency;
70
- this._maxConcurrency = maxConcurrency;
71
- this._desiredConcurrency = desiredConcurrency ?? minConcurrency;
70
+ this.#minConcurrency = minConcurrency;
71
+ this.#maxConcurrency = maxConcurrency;
72
+ this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;
72
73
  this.clampDesiredConcurrency();
73
- this._autoscale = this._autoscale.bind(this);
74
- this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
74
+ this.autoscale = this.autoscale.bind(this);
75
+ this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
75
76
  // The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
76
77
  const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
77
78
  this.snapshotter = new Snapshotter(builtinSignalOptions);
78
- this.loadSignals = customLoadSignals;
79
+ this.#loadSignals = customLoadSignals;
79
80
  this.systemStatus = new SystemStatus({
80
81
  snapshotter: this.snapshotter,
81
82
  loadSignals: customLoadSignals,
@@ -89,7 +90,7 @@ export class ConcurrencySystem {
89
90
  * Gets the minimum number of tasks running in parallel.
90
91
  */
91
92
  get minConcurrency() {
92
- return this._minConcurrency;
93
+ return this.#minConcurrency;
93
94
  }
94
95
  /**
95
96
  * Sets the minimum number of tasks running in parallel.
@@ -99,14 +100,14 @@ export class ConcurrencySystem {
99
100
  */
100
101
  set minConcurrency(value) {
101
102
  ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
102
- this._minConcurrency = value;
103
+ this.#minConcurrency = value;
103
104
  this.clampDesiredConcurrency();
104
105
  }
105
106
  /**
106
107
  * Gets the maximum number of tasks running in parallel.
107
108
  */
108
109
  get maxConcurrency() {
109
- return this._maxConcurrency;
110
+ return this.#maxConcurrency;
110
111
  }
111
112
  /**
112
113
  * Sets the maximum number of tasks running in parallel. Lowering it below the current
@@ -116,7 +117,7 @@ export class ConcurrencySystem {
116
117
  */
117
118
  set maxConcurrency(value) {
118
119
  ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
119
- this._maxConcurrency = value;
120
+ this.#maxConcurrency = value;
120
121
  this.clampDesiredConcurrency();
121
122
  }
122
123
  /**
@@ -124,7 +125,7 @@ export class ConcurrencySystem {
124
125
  * which is an estimated number of parallel tasks that the system can currently support.
125
126
  */
126
127
  get desiredConcurrency() {
127
- return this._desiredConcurrency;
128
+ return this.#desiredConcurrency;
128
129
  }
129
130
  /**
130
131
  * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
@@ -132,7 +133,7 @@ export class ConcurrencySystem {
132
133
  */
133
134
  set desiredConcurrency(value) {
134
135
  ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
135
- this._desiredConcurrency = value;
136
+ this.#desiredConcurrency = value;
136
137
  this.clampDesiredConcurrency();
137
138
  }
138
139
  /**
@@ -142,15 +143,15 @@ export class ConcurrencySystem {
142
143
  * that is the limit callers set in order to protect something.
143
144
  */
144
145
  clampDesiredConcurrency() {
145
- const atLeastMin = Math.max(this._desiredConcurrency, this._minConcurrency);
146
- this._desiredConcurrency = Math.min(atLeastMin, this._maxConcurrency);
146
+ const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);
147
+ this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
147
148
  }
148
149
  get currentConcurrency() {
149
150
  return this._currentConcurrency;
150
151
  }
151
152
  /** Whether the system is currently monitoring load and autoscaling the budget. */
152
153
  get isRunning() {
153
- return this.running;
154
+ return this.#running;
154
155
  }
155
156
  /**
156
157
  * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
@@ -160,52 +161,52 @@ export class ConcurrencySystem {
160
161
  async start() {
161
162
  // Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
162
163
  // a system that is down.
163
- this.startPromise ??= this.boot().catch(async (error) => {
164
- this.startPromise = undefined;
164
+ this.#startPromise ??= this.boot().catch(async (error) => {
165
+ this.#startPromise = undefined;
165
166
  await this.shutDown();
166
167
  throw error;
167
168
  });
168
- await this.startPromise;
169
+ await this.#startPromise;
169
170
  }
170
171
  async boot() {
171
172
  // Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
172
173
  // per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
173
174
  // arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
174
- this._tasksPerMinute = Array.from({ length: 60 }, () => 0);
175
- this.lastLoggingTime = undefined;
176
- this.warnedAboutQueryWhileStopped = false;
175
+ this.#tasksPerMinute = Array.from({ length: 60 }, () => 0);
176
+ this.#lastLoggingTime = undefined;
177
+ this.#warnedAboutQueryWhileStopped = false;
177
178
  // Signals are told how much history to keep when they start: exactly the longest window they will be sampled
178
179
  // over, so nobody has to guess a retention value that matches this system's configuration.
179
180
  const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };
180
181
  await this.snapshotter.start(startContext);
181
- await Promise.all(this.loadSignals.map(async (s) => s.start(startContext)));
182
- this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
182
+ await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));
183
+ this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);
183
184
  if (this.maxTasksPerMinute !== Infinity) {
184
- this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);
185
+ this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);
185
186
  }
186
187
  // Last, so `isRunning` never claims a system whose signals aren't collecting yet.
187
- this.running = true;
188
+ this.#running = true;
188
189
  }
189
190
  /**
190
191
  * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
191
192
  */
192
193
  async stop() {
193
- if (this.startPromise === undefined) {
194
+ if (this.#startPromise === undefined) {
194
195
  return;
195
196
  }
196
197
  // Waited out rather than interrupted, or the intervals a starting signal is about to register outlive us.
197
- await this.startPromise.catch(() => { });
198
- this.startPromise = undefined;
199
- this.running = false;
198
+ await this.#startPromise.catch(() => { });
199
+ this.#startPromise = undefined;
200
+ this.#running = false;
200
201
  await this.shutDown();
201
202
  }
202
203
  async shutDown() {
203
- if (this.autoscaleInterval)
204
- betterClearInterval(this.autoscaleInterval);
205
- if (this.tasksDonePerSecondInterval)
206
- betterClearInterval(this.tasksDonePerSecondInterval);
204
+ if (this.#autoscaleInterval)
205
+ betterClearInterval(this.#autoscaleInterval);
206
+ if (this.#tasksDonePerSecondInterval)
207
+ betterClearInterval(this.#tasksDonePerSecondInterval);
207
208
  await this.snapshotter.stop();
208
- await Promise.all(this.loadSignals.map(async (s) => s.stop()));
209
+ await Promise.all(this.#loadSignals.map(async (s) => s.stop()));
209
210
  }
210
211
  /**
211
212
  * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
@@ -214,10 +215,10 @@ export class ConcurrencySystem {
214
215
  * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
215
216
  */
216
217
  warnIfNotRunning() {
217
- if (this.running || this.warnedAboutQueryWhileStopped) {
218
+ if (this.#running || this.#warnedAboutQueryWhileStopped) {
218
219
  return;
219
220
  }
220
- this.warnedAboutQueryWhileStopped = true;
221
+ this.#warnedAboutQueryWhileStopped = true;
221
222
  this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
222
223
  'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
223
224
  'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
@@ -232,13 +233,13 @@ export class ConcurrencySystem {
232
233
  */
233
234
  hasCapacityForTask(_consumer) {
234
235
  this.warnIfNotRunning();
235
- if (this._currentConcurrency >= this._desiredConcurrency) {
236
+ if (this._currentConcurrency >= this.#desiredConcurrency) {
236
237
  this.log.perf('Task will not run. Desired concurrency achieved.');
237
238
  return false;
238
239
  }
239
240
  const currentStatus = this.systemStatus.getCurrentStatus();
240
241
  const { isSystemIdle } = currentStatus;
241
- if (!isSystemIdle && this._currentConcurrency >= this._minConcurrency) {
242
+ if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {
242
243
  this.log.perf('Task will not be run. System is overloaded.', currentStatus);
243
244
  return false;
244
245
  }
@@ -249,7 +250,7 @@ export class ConcurrencySystem {
249
250
  if (this.maxTasksPerMinute === Infinity) {
250
251
  return false;
251
252
  }
252
- return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
253
+ return this.#tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
253
254
  }
254
255
  /**
255
256
  * Atomically books a task against the shared budget: re-checks
@@ -269,7 +270,7 @@ export class ConcurrencySystem {
269
270
  return false;
270
271
  }
271
272
  this._currentConcurrency++;
272
- this._tasksPerMinute[0]++;
273
+ this.#tasksPerMinute[0]++;
273
274
  return true;
274
275
  }
275
276
  /** Returns a slot to the shared budget, whoever booked it. */
@@ -288,30 +289,30 @@ export class ConcurrencySystem {
288
289
  * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
289
290
  * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
290
291
  */
291
- _autoscale(intervalCallback) {
292
+ autoscale(intervalCallback) {
292
293
  if (this.isOverMaxRequestLimit)
293
294
  return intervalCallback();
294
295
  const systemStatus = this.systemStatus.getHistoricalStatus();
295
296
  const { isSystemIdle } = systemStatus;
296
- const weAreNotAtMax = this._desiredConcurrency < this._maxConcurrency;
297
- const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);
297
+ const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
298
+ const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
298
299
  const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
299
300
  if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
300
- this._scaleUp(systemStatus);
301
+ this.scaleUp(systemStatus);
301
302
  const isSystemOverloaded = !isSystemIdle;
302
- const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;
303
+ const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;
303
304
  if (isSystemOverloaded && weAreNotAtMin)
304
- this._scaleDown(systemStatus);
305
- if (this.loggingIntervalMillis > 0) {
305
+ this.scaleDown(systemStatus);
306
+ if (this.#loggingIntervalMillis > 0) {
306
307
  const now = Date.now();
307
- if (this.lastLoggingTime == null) {
308
- this.lastLoggingTime = now;
308
+ if (this.#lastLoggingTime == null) {
309
+ this.#lastLoggingTime = now;
309
310
  }
310
- else if (now > this.lastLoggingTime + this.loggingIntervalMillis) {
311
- this.lastLoggingTime = now;
311
+ else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
312
+ this.#lastLoggingTime = now;
312
313
  this.log.info('state', {
313
314
  currentConcurrency: this._currentConcurrency,
314
- desiredConcurrency: this._desiredConcurrency,
315
+ desiredConcurrency: this.#desiredConcurrency,
315
316
  systemStatus,
316
317
  });
317
318
  }
@@ -321,30 +322,30 @@ export class ConcurrencySystem {
321
322
  /**
322
323
  * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
323
324
  */
324
- _scaleUp(systemStatus) {
325
- const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
326
- this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);
325
+ scaleUp(systemStatus) {
326
+ const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);
327
+ this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);
327
328
  this.log.debug('scaling up', {
328
- oldConcurrency: this._desiredConcurrency - step,
329
- newConcurrency: this._desiredConcurrency,
329
+ oldConcurrency: this.#desiredConcurrency - step,
330
+ newConcurrency: this.#desiredConcurrency,
330
331
  systemStatus,
331
332
  });
332
333
  }
333
334
  /**
334
335
  * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
335
336
  */
336
- _scaleDown(systemStatus) {
337
- const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);
338
- this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);
337
+ scaleDown(systemStatus) {
338
+ const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);
339
+ this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);
339
340
  this.log.debug('scaling down', {
340
- oldConcurrency: this._desiredConcurrency + step,
341
- newConcurrency: this._desiredConcurrency,
341
+ oldConcurrency: this.#desiredConcurrency + step,
342
+ newConcurrency: this.#desiredConcurrency,
342
343
  systemStatus,
343
344
  });
344
345
  }
345
- _incrementTasksDonePerSecond(intervalCallback) {
346
- this._tasksPerMinute.unshift(0);
347
- this._tasksPerMinute.pop();
346
+ incrementTasksDonePerSecond(intervalCallback) {
347
+ this.#tasksPerMinute.unshift(0);
348
+ this.#tasksPerMinute.pop();
348
349
  return intervalCallback();
349
350
  }
350
351
  }
@@ -31,10 +31,9 @@ export interface CpuLoadSignalOptions {
31
31
  * @category Scaling
32
32
  */
33
33
  export declare class CpuLoadSignal implements LoadSignal {
34
+ #private;
34
35
  readonly name = "cpuInfo";
35
36
  readonly overloadedRatio: number;
36
- private readonly store;
37
- private events?;
38
37
  constructor(options?: CpuLoadSignalOptions);
39
38
  start(context: LoadSignalStartContext): Promise<void>;
40
39
  stop(): Promise<void>;
@@ -11,33 +11,33 @@ import { SnapshotStore } from './load_signal.js';
11
11
  export class CpuLoadSignal {
12
12
  name = 'cpuInfo';
13
13
  overloadedRatio;
14
- store = new SnapshotStore();
15
- events;
14
+ #store = new SnapshotStore();
15
+ #events;
16
16
  constructor(options = {}) {
17
17
  this.overloadedRatio = options.overloadedRatio ?? 0.4;
18
18
  this.handle = this.handle.bind(this);
19
19
  }
20
20
  async start(context) {
21
- this.store.useSampleWindow(context.maxSampleWindowMillis);
21
+ this.#store.useSampleWindow(context.maxSampleWindowMillis);
22
22
  // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
23
- this.store.clear();
23
+ this.#store.clear();
24
24
  // Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
25
25
  // between systems) cannot capture whichever event manager happened to be registered at that moment.
26
- this.events = serviceLocator.getEventManager();
27
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
26
+ this.#events = serviceLocator.getEventManager();
27
+ this.#events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
28
28
  }
29
29
  async stop() {
30
- this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
31
- this.events = undefined;
30
+ this.#events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
31
+ this.#events = undefined;
32
32
  }
33
33
  getSample(sampleDurationMillis) {
34
- return this.store.getSample(sampleDurationMillis);
34
+ return this.#store.getSample(sampleDurationMillis);
35
35
  }
36
36
  /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
37
37
  handle(systemInfo) {
38
38
  const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
39
39
  const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
40
- this.store.push({
40
+ this.#store.push({
41
41
  createdAt,
42
42
  isOverloaded: isCpuOverloaded,
43
43
  usedRatio: Math.ceil(cpuCurrentUsage / 100),
@@ -36,12 +36,9 @@ export interface EventLoopLoadSignalOptions {
36
36
  * @category Scaling
37
37
  */
38
38
  export declare class EventLoopLoadSignal implements LoadSignal {
39
+ #private;
39
40
  readonly name = "eventLoopInfo";
40
41
  readonly overloadedRatio: number;
41
- private readonly store;
42
- private readonly intervalMillis;
43
- private readonly maxBlockedMillis;
44
- private interval?;
45
42
  constructor(options?: EventLoopLoadSignalOptions);
46
43
  start(context: LoadSignalStartContext): Promise<void>;
47
44
  stop(): Promise<void>;
@@ -10,30 +10,30 @@ import { SnapshotStore } from './load_signal.js';
10
10
  export class EventLoopLoadSignal {
11
11
  name = 'eventLoopInfo';
12
12
  overloadedRatio;
13
- store = new SnapshotStore();
14
- intervalMillis;
15
- maxBlockedMillis;
16
- interval;
13
+ #store = new SnapshotStore();
14
+ #intervalMillis;
15
+ #maxBlockedMillis;
16
+ #interval;
17
17
  constructor(options = {}) {
18
18
  this.overloadedRatio = options.overloadedRatio ?? 0.6;
19
- this.intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
20
- this.maxBlockedMillis = options.maxBlockedMillis ?? 50;
19
+ this.#intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
20
+ this.#maxBlockedMillis = options.maxBlockedMillis ?? 50;
21
21
  this.handle = this.handle.bind(this);
22
22
  }
23
23
  async start(context) {
24
- this.store.useSampleWindow(context.maxSampleWindowMillis);
24
+ this.#store.useSampleWindow(context.maxSampleWindowMillis);
25
25
  // A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
26
26
  // the gap since the previous snapshot, which across a restart is however long the system was stopped.
27
- this.store.clear();
28
- this.interval = betterSetInterval(this.handle, this.intervalMillis);
27
+ this.#store.clear();
28
+ this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
29
29
  }
30
30
  async stop() {
31
- if (this.interval)
32
- betterClearInterval(this.interval);
33
- this.interval = undefined;
31
+ if (this.#interval)
32
+ betterClearInterval(this.#interval);
33
+ this.#interval = undefined;
34
34
  }
35
35
  getSample(sampleDurationMillis) {
36
- return this.store.getSample(sampleDurationMillis);
36
+ return this.#store.getSample(sampleDurationMillis);
37
37
  }
38
38
  /**
39
39
  * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
@@ -46,15 +46,15 @@ export class EventLoopLoadSignal {
46
46
  isOverloaded: false,
47
47
  exceededMillis: 0,
48
48
  };
49
- const all = this.store.getAll();
49
+ const all = this.#store.getAll();
50
50
  const previousSnapshot = all[all.length - 1];
51
51
  if (previousSnapshot) {
52
- const delta = now.getTime() - +previousSnapshot.createdAt - this.intervalMillis;
53
- if (delta > this.maxBlockedMillis)
52
+ const delta = now.getTime() - +previousSnapshot.createdAt - this.#intervalMillis;
53
+ if (delta > this.#maxBlockedMillis)
54
54
  snapshot.isOverloaded = true;
55
- snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
55
+ snapshot.exceededMillis = Math.max(delta - this.#maxBlockedMillis, 0);
56
56
  }
57
- this.store.push(snapshot, now);
57
+ this.#store.push(snapshot, now);
58
58
  intervalCallback();
59
59
  }
60
60
  }
@@ -60,7 +60,7 @@ export interface LoadSignal {
60
60
  * and so can yours — it is the only part of their machinery worth reusing.
61
61
  */
62
62
  export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
63
- private snapshots;
63
+ #private;
64
64
  /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
65
65
  private historyMillis;
66
66
  /**
@@ -4,8 +4,9 @@ import { weightedAvg } from './weighted_avg.js';
4
4
  * and so can yours — it is the only part of their machinery worth reusing.
5
5
  */
6
6
  export class SnapshotStore {
7
- snapshots = [];
7
+ #snapshots = [];
8
8
  /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
9
+ // kept as TS-private: concurrency_system tests read this retention window directly
9
10
  historyMillis = Infinity;
10
11
  /**
11
12
  * Sizes retention to the window the signal will be sampled over, as handed to it in
@@ -21,30 +22,30 @@ export class SnapshotStore {
21
22
  push(snapshot, now = snapshot.createdAt) {
22
23
  // Inline pruning to avoid private-method transpilation issues
23
24
  let oldCount = 0;
24
- for (let i = 0; i < this.snapshots.length; i++) {
25
- const { createdAt } = this.snapshots[i];
25
+ for (let i = 0; i < this.#snapshots.length; i++) {
26
+ const { createdAt } = this.#snapshots[i];
26
27
  if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
27
28
  oldCount++;
28
29
  else
29
30
  break;
30
31
  }
31
32
  if (oldCount)
32
- this.snapshots.splice(0, oldCount);
33
- this.snapshots.push(snapshot);
33
+ this.#snapshots.splice(0, oldCount);
34
+ this.#snapshots.push(snapshot);
34
35
  }
35
36
  /**
36
37
  * Return all snapshots, or only those within the given time window.
37
38
  */
38
39
  getSample(sampleDurationMillis) {
39
40
  if (!sampleDurationMillis)
40
- return this.snapshots;
41
+ return this.#snapshots;
41
42
  const sample = [];
42
- let idx = this.snapshots.length;
43
+ let idx = this.#snapshots.length;
43
44
  if (!idx)
44
45
  return sample;
45
- const latestTime = this.snapshots[idx - 1].createdAt;
46
+ const latestTime = this.#snapshots[idx - 1].createdAt;
46
47
  while (idx--) {
47
- const snapshot = this.snapshots[idx];
48
+ const snapshot = this.#snapshots[idx];
48
49
  if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
49
50
  sample.unshift(snapshot);
50
51
  }
@@ -59,7 +60,7 @@ export class SnapshotStore {
59
60
  * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
60
61
  */
61
62
  getAll() {
62
- return this.snapshots;
63
+ return this.#snapshots;
63
64
  }
64
65
  /**
65
66
  * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
@@ -68,7 +69,7 @@ export class SnapshotStore {
68
69
  * rather than on stop leaves a finished session readable.
69
70
  */
70
71
  clear() {
71
- this.snapshots = [];
72
+ this.#snapshots = [];
72
73
  }
73
74
  }
74
75
  /**
@@ -33,24 +33,15 @@ export interface MemoryLoadSignalOptions {
33
33
  * @category Scaling
34
34
  */
35
35
  export declare class MemoryLoadSignal implements LoadSignal {
36
+ #private;
36
37
  readonly name = "memInfo";
37
38
  readonly overloadedRatio: number;
38
- private readonly store;
39
- private readonly maxUsedRatio;
40
- /** All resolved in `start()`, before anything that reads them can fire. */
41
- private config;
42
- private log;
43
- private maxMemoryBytes;
44
- private events?;
45
- private maxMemoryRatio;
46
- private lastLoggedCriticalMemoryOverloadAt;
47
39
  constructor(options?: MemoryLoadSignalOptions);
48
40
  start(context: LoadSignalStartContext): Promise<void>;
49
41
  stop(): Promise<void>;
50
42
  getSample(sampleDurationMillis?: number): LoadSnapshot[];
51
43
  /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
52
44
  handle(systemInfo: SystemInfo): void;
53
- /** @internal */
54
- _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
55
- private _getTotalMemoryBytes;
45
+ private memoryOverloadWarning;
46
+ private getTotalMemoryBytes;
56
47
  }