@crawlee/basic 4.0.0-beta.210 → 4.0.0-beta.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -175,24 +175,4 @@ export declare class AutoscaledPool {
175
175
  * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
176
176
  */
177
177
  notify(): Promise<void>;
178
- /**
179
- * Starts a new task
180
- * if the number of running tasks (current concurrency) is lower than desired concurrency
181
- * and the system is not currently overloaded
182
- * and `isTaskReadyFunction()` returns true.
183
- *
184
- * It doesn't allow multiple concurrent runs of this method.
185
- */
186
- private maybeRunTask;
187
- /**
188
- * If there are no running tasks and `isFinishedFunction()` returns true then closes
189
- * the pool and resolves the pool's promise returned by the run() method.
190
- *
191
- * It doesn't allow multiple concurrent runs of this method.
192
- */
193
- private maybeFinish;
194
- /**
195
- * Cleans up resources.
196
- */
197
- private destroy;
198
178
  }
@@ -111,7 +111,6 @@ export class AutoscaledPool {
111
111
  this.#isStopped = false;
112
112
  this.#resolve = null;
113
113
  this.#reject = null;
114
- this.maybeRunTask = this.maybeRunTask.bind(this);
115
114
  }
116
115
  /**
117
116
  * The governor backing this pool, as supplied to the constructor — exposed as the read-only
@@ -157,14 +156,14 @@ export class AutoscaledPool {
157
156
  // This is here because if we scale down to let's say 1, then after each promise is finished
158
157
  // this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
159
158
  // in the crawler getting stuck and even after scaling up it never triggers another promise.
160
- this.#maybeRunInterval = betterSetInterval(this.maybeRunTask, this.#maybeRunIntervalMillis);
159
+ this.#maybeRunInterval = betterSetInterval((cb) => this.#maybeRunTask(cb), this.#maybeRunIntervalMillis);
161
160
  try {
162
161
  await poolPromise;
163
162
  }
164
163
  finally {
165
164
  // If resolve is null, the pool is already destroyed.
166
165
  if (this.#resolve)
167
- await this.destroy();
166
+ await this.#destroy();
168
167
  }
169
168
  }
170
169
  /**
@@ -182,7 +181,7 @@ export class AutoscaledPool {
182
181
  this.#isStopped = true;
183
182
  if (this.#resolve) {
184
183
  this.#resolve();
185
- await this.destroy();
184
+ await this.#destroy();
186
185
  }
187
186
  }
188
187
  /**
@@ -241,7 +240,7 @@ export class AutoscaledPool {
241
240
  * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
242
241
  */
243
242
  async notify() {
244
- setImmediate(this.maybeRunTask);
243
+ setImmediate(() => this.#maybeRunTask());
245
244
  }
246
245
  /**
247
246
  * Starts a new task
@@ -251,7 +250,7 @@ export class AutoscaledPool {
251
250
  *
252
251
  * It doesn't allow multiple concurrent runs of this method.
253
252
  */
254
- async maybeRunTask(intervalCallback) {
253
+ async #maybeRunTask(intervalCallback) {
255
254
  this.#log.perf('Attempting to run a task.');
256
255
  // Check if the function was invoked by the maybeRunInterval and use an empty function if not.
257
256
  const done = intervalCallback || (() => { });
@@ -273,7 +272,7 @@ export class AutoscaledPool {
273
272
  // to notice that *this* pool has run out of work — `maybeFinish()` is the only thing that ever resolves
274
273
  // `run()`. It no-ops while this pool has tasks of its own in flight, which is every case in which an
275
274
  // unshared governor reports no capacity.
276
- return this.maybeFinish();
275
+ return this.#maybeFinish();
277
276
  }
278
277
  // - a task is ready.
279
278
  this.#queryingIsTaskReady = true;
@@ -299,7 +298,7 @@ export class AutoscaledPool {
299
298
  this.#log.perf('Task will not run. No tasks are ready.');
300
299
  done();
301
300
  // No tasks could mean that we're finished with all tasks.
302
- return this.maybeFinish();
301
+ return this.#maybeFinish();
303
302
  }
304
303
  const taskId = String(this.#nextTaskId++);
305
304
  // - the budget still has room. Re-checked atomically, because another pool sharing the governor may have taken
@@ -312,7 +311,7 @@ export class AutoscaledPool {
312
311
  // Everything's fine. Run task.
313
312
  // Try to run next task to build up concurrency,
314
313
  // but defer it so it doesn't create a cycle.
315
- setImmediate(this.maybeRunTask);
314
+ setImmediate(() => this.#maybeRunTask());
316
315
  // We need to restart interval here, so that it doesn't get blocked by a stalled task.
317
316
  done();
318
317
  // Execute the current task.
@@ -326,7 +325,7 @@ export class AutoscaledPool {
326
325
  this.#log.perf('Task finished.');
327
326
  // Run task after the previous one finished. Only on success: a failed task rejects the pool, and
328
327
  // nudging the loop afterwards could start work on an already destroyed pool.
329
- setImmediate(this.maybeRunTask);
328
+ setImmediate(() => this.#maybeRunTask());
330
329
  }
331
330
  catch (e) {
332
331
  const err = e;
@@ -354,7 +353,7 @@ export class AutoscaledPool {
354
353
  *
355
354
  * It doesn't allow multiple concurrent runs of this method.
356
355
  */
357
- async maybeFinish() {
356
+ async #maybeFinish() {
358
357
  if (this.#queryingIsFinished)
359
358
  return;
360
359
  if (this.#ownConcurrency > 0)
@@ -380,7 +379,7 @@ export class AutoscaledPool {
380
379
  /**
381
380
  * Cleans up resources.
382
381
  */
383
- async destroy() {
382
+ async #destroy() {
384
383
  this.#resolve = null;
385
384
  this.#reject = null;
386
385
  betterClearInterval(this.#maybeRunInterval);
@@ -159,14 +159,8 @@ export interface IConcurrencySystem {
159
159
  */
160
160
  export declare class ConcurrencySystem implements IConcurrencySystem {
161
161
  #private;
162
- private readonly log;
163
- private readonly desiredConcurrencyRatio;
164
- private readonly scaleUpStepRatio;
165
- private readonly scaleDownStepRatio;
166
162
  /** The cap on tasks started per minute, or `Infinity` when uncapped. */
167
163
  readonly maxTasksPerMinute: number;
168
- private readonly snapshotter;
169
- private readonly systemStatus;
170
164
  constructor(options?: ConcurrencySystemOptions);
171
165
  /**
172
166
  * Gets the minimum number of tasks running in parallel.
@@ -198,13 +192,6 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
198
192
  * Tune the budget through `minConcurrency`/`maxConcurrency` instead.
199
193
  */
200
194
  get desiredConcurrency(): number;
201
- /**
202
- * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
203
- * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
204
- * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
205
- * that is the limit callers set in order to protect something.
206
- */
207
- private clampDesiredConcurrency;
208
195
  get currentConcurrency(): number;
209
196
  /** Whether the system is currently monitoring load and autoscaling the budget. */
210
197
  get isRunning(): boolean;
@@ -214,20 +201,11 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
214
201
  * signal fails to start.
215
202
  */
216
203
  start(): Promise<void>;
217
- private boot;
218
204
  [Symbol.asyncDispose](): Promise<void>;
219
205
  /**
220
206
  * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
221
207
  */
222
208
  stop(): Promise<void>;
223
- private shutDown;
224
- /**
225
- * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
226
- * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
227
- * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
228
- * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
229
- */
230
- private warnIfNotRunning;
231
209
  /**
232
210
  * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
233
211
  * reached) or when the machine is overloaded past `minConcurrency`.
@@ -236,8 +214,6 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
236
214
  * interface, letting the answer be queried directly.
237
215
  */
238
216
  hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
239
- /** Whether the per-minute task cap has been reached. */
240
- private get isOverMaxRequestLimit();
241
217
  /**
242
218
  * Atomically books a task against the shared budget: re-checks
243
219
  * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
@@ -256,18 +232,4 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
256
232
  * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
257
233
  */
258
234
  getCurrentStatus(): SystemInfo;
259
- /**
260
- * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
261
- * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
262
- */
263
- private autoscale;
264
- /**
265
- * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
266
- */
267
- private scaleUp;
268
- /**
269
- * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
270
- */
271
- private scaleDown;
272
- private incrementTasksDonePerSecond;
273
235
  }
@@ -44,10 +44,10 @@ const concurrencySystemOptionsSchema = z.strictObject({
44
44
  * @category Scaling
45
45
  */
46
46
  export class ConcurrencySystem {
47
- log;
48
- desiredConcurrencyRatio;
49
- scaleUpStepRatio;
50
- scaleDownStepRatio;
47
+ #log;
48
+ #desiredConcurrencyRatio;
49
+ #scaleUpStepRatio;
50
+ #scaleDownStepRatio;
51
51
  #loggingIntervalMillis;
52
52
  #autoscaleIntervalMillis;
53
53
  /** The cap on tasks started per minute, or `Infinity` when uncapped. */
@@ -58,9 +58,9 @@ export class ConcurrencySystem {
58
58
  #currentConcurrency = 0;
59
59
  #lastLoggingTime;
60
60
  #tasksPerMinute = Array.from({ length: 60 }, () => 0);
61
- snapshotter;
61
+ #snapshotter;
62
62
  #loadSignals;
63
- systemStatus;
63
+ #systemStatus;
64
64
  #autoscaleInterval;
65
65
  #tasksDonePerSecondInterval;
66
66
  /** Whether the snapshotter and autoscaling intervals are currently running. */
@@ -71,25 +71,23 @@ export class ConcurrencySystem {
71
71
  #warnedAboutQueryWhileStopped = false;
72
72
  constructor(options = {}) {
73
73
  const { maxConcurrency, minConcurrency, desiredConcurrency, desiredConcurrencyRatio, scaleUpStepRatio, scaleDownStepRatio, loggingIntervalSecs, autoscaleIntervalSecs, loadSignals, snapshotHistorySecs, currentHistorySecs, log, maxTasksPerMinute, } = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');
74
- this.log = log.child({ prefix: 'ConcurrencySystem' });
75
- this.desiredConcurrencyRatio = desiredConcurrencyRatio;
76
- this.scaleUpStepRatio = scaleUpStepRatio;
77
- this.scaleDownStepRatio = scaleDownStepRatio;
74
+ this.#log = log.child({ prefix: 'ConcurrencySystem' });
75
+ this.#desiredConcurrencyRatio = desiredConcurrencyRatio;
76
+ this.#scaleUpStepRatio = scaleUpStepRatio;
77
+ this.#scaleDownStepRatio = scaleDownStepRatio;
78
78
  this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
79
79
  this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
80
80
  this.maxTasksPerMinute = maxTasksPerMinute;
81
81
  this.#minConcurrency = minConcurrency;
82
82
  this.#maxConcurrency = maxConcurrency;
83
83
  this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;
84
- this.clampDesiredConcurrency();
85
- this.autoscale = this.autoscale.bind(this);
86
- this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
84
+ this.#clampDesiredConcurrency();
87
85
  // The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
88
86
  const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
89
- this.snapshotter = new Snapshotter(builtinSignalOptions);
87
+ this.#snapshotter = new Snapshotter(builtinSignalOptions);
90
88
  this.#loadSignals = customLoadSignals;
91
- this.systemStatus = new SystemStatus({
92
- snapshotter: this.snapshotter,
89
+ this.#systemStatus = new SystemStatus({
90
+ snapshotter: this.#snapshotter,
93
91
  loadSignals: customLoadSignals,
94
92
  currentHistorySecs,
95
93
  // Both windows are requested from the signals explicitly, so a signal's own retention can neither widen
@@ -112,7 +110,7 @@ export class ConcurrencySystem {
112
110
  set minConcurrency(value) {
113
111
  parseArgument(value, concurrencySchema);
114
112
  this.#minConcurrency = value;
115
- this.clampDesiredConcurrency();
113
+ this.#clampDesiredConcurrency();
116
114
  }
117
115
  /**
118
116
  * Gets the maximum number of tasks running in parallel.
@@ -129,7 +127,7 @@ export class ConcurrencySystem {
129
127
  set maxConcurrency(value) {
130
128
  parseArgument(value, concurrencySchema);
131
129
  this.#maxConcurrency = value;
132
- this.clampDesiredConcurrency();
130
+ this.#clampDesiredConcurrency();
133
131
  }
134
132
  /**
135
133
  * Gets the desired concurrency for the system,
@@ -147,7 +145,7 @@ export class ConcurrencySystem {
147
145
  * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
148
146
  * that is the limit callers set in order to protect something.
149
147
  */
150
- clampDesiredConcurrency() {
148
+ #clampDesiredConcurrency() {
151
149
  const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);
152
150
  this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
153
151
  }
@@ -166,14 +164,14 @@ export class ConcurrencySystem {
166
164
  async start() {
167
165
  // Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
168
166
  // a system that is down.
169
- this.#startPromise ??= this.boot().catch(async (error) => {
167
+ this.#startPromise ??= this.#boot().catch(async (error) => {
170
168
  this.#startPromise = undefined;
171
- await this.shutDown();
169
+ await this.#shutDown();
172
170
  throw error;
173
171
  });
174
172
  await this.#startPromise;
175
173
  }
176
- async boot() {
174
+ async #boot() {
177
175
  // Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
178
176
  // per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
179
177
  // arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
@@ -182,12 +180,12 @@ export class ConcurrencySystem {
182
180
  this.#warnedAboutQueryWhileStopped = false;
183
181
  // Signals are told how much history to keep when they start: exactly the longest window they will be sampled
184
182
  // over, so nobody has to guess a retention value that matches this system's configuration.
185
- const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };
186
- await this.snapshotter.start(startContext);
183
+ const startContext = { maxSampleWindowMillis: this.#systemStatus.maxSampleWindowMillis };
184
+ await this.#snapshotter.start(startContext);
187
185
  await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));
188
- this.#autoscaleInterval = betterSetInterval(this.autoscale, this.#autoscaleIntervalMillis);
186
+ this.#autoscaleInterval = betterSetInterval((cb) => this.#autoscale(cb), this.#autoscaleIntervalMillis);
189
187
  if (this.maxTasksPerMinute !== Infinity) {
190
- this.#tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);
188
+ this.#tasksDonePerSecondInterval = betterSetInterval((cb) => this.#incrementTasksDonePerSecond(cb), 1000);
191
189
  }
192
190
  // Last, so `isRunning` never claims a system whose signals aren't collecting yet.
193
191
  this.#running = true;
@@ -206,14 +204,14 @@ export class ConcurrencySystem {
206
204
  await this.#startPromise.catch(() => { });
207
205
  this.#startPromise = undefined;
208
206
  this.#running = false;
209
- await this.shutDown();
207
+ await this.#shutDown();
210
208
  }
211
- async shutDown() {
209
+ async #shutDown() {
212
210
  if (this.#autoscaleInterval)
213
211
  betterClearInterval(this.#autoscaleInterval);
214
212
  if (this.#tasksDonePerSecondInterval)
215
213
  betterClearInterval(this.#tasksDonePerSecondInterval);
216
- await this.snapshotter.stop();
214
+ await this.#snapshotter.stop();
217
215
  await Promise.all(this.#loadSignals.map(async (s) => s.stop()));
218
216
  }
219
217
  /**
@@ -222,12 +220,12 @@ export class ConcurrencySystem {
222
220
  * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
223
221
  * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
224
222
  */
225
- warnIfNotRunning() {
223
+ #warnIfNotRunning() {
226
224
  if (this.#running || this.#warnedAboutQueryWhileStopped) {
227
225
  return;
228
226
  }
229
227
  this.#warnedAboutQueryWhileStopped = true;
230
- this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
228
+ this.#log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
231
229
  'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
232
230
  'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
233
231
  'has finished.');
@@ -240,21 +238,21 @@ export class ConcurrencySystem {
240
238
  * interface, letting the answer be queried directly.
241
239
  */
242
240
  hasCapacityForTask(_consumer) {
243
- this.warnIfNotRunning();
241
+ this.#warnIfNotRunning();
244
242
  if (this.#currentConcurrency >= this.#desiredConcurrency) {
245
- this.log.perf('Task will not run. Desired concurrency achieved.');
243
+ this.#log.perf('Task will not run. Desired concurrency achieved.');
246
244
  return false;
247
245
  }
248
- const currentStatus = this.systemStatus.getCurrentStatus();
246
+ const currentStatus = this.#systemStatus.getCurrentStatus();
249
247
  const { isSystemIdle } = currentStatus;
250
248
  if (!isSystemIdle && this.#currentConcurrency >= this.#minConcurrency) {
251
- this.log.perf('Task will not be run. System is overloaded.', currentStatus);
249
+ this.#log.perf('Task will not be run. System is overloaded.', currentStatus);
252
250
  return false;
253
251
  }
254
252
  return true;
255
253
  }
256
254
  /** Whether the per-minute task cap has been reached. */
257
- get isOverMaxRequestLimit() {
255
+ get #isOverMaxRequestLimit() {
258
256
  if (this.maxTasksPerMinute === Infinity) {
259
257
  return false;
260
258
  }
@@ -273,8 +271,8 @@ export class ConcurrencySystem {
273
271
  if (!this.hasCapacityForTask(consumer)) {
274
272
  return false;
275
273
  }
276
- if (this.isOverMaxRequestLimit) {
277
- this.log.perf('Task will not run. Maximum tasks per minute reached.');
274
+ if (this.#isOverMaxRequestLimit) {
275
+ this.#log.perf('Task will not run. Maximum tasks per minute reached.');
278
276
  return false;
279
277
  }
280
278
  this.#currentConcurrency++;
@@ -291,26 +289,26 @@ export class ConcurrencySystem {
291
289
  * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
292
290
  */
293
291
  getCurrentStatus() {
294
- return this.systemStatus.getCurrentStatus();
292
+ return this.#systemStatus.getCurrentStatus();
295
293
  }
296
294
  /**
297
295
  * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
298
296
  * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
299
297
  */
300
- autoscale(intervalCallback) {
301
- if (this.isOverMaxRequestLimit)
298
+ #autoscale(intervalCallback) {
299
+ if (this.#isOverMaxRequestLimit)
302
300
  return intervalCallback();
303
- const systemStatus = this.systemStatus.getHistoricalStatus();
301
+ const systemStatus = this.#systemStatus.getHistoricalStatus();
304
302
  const { isSystemIdle } = systemStatus;
305
303
  const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
306
- const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
304
+ const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.#desiredConcurrencyRatio);
307
305
  const weAreReachingDesiredConcurrency = this.#currentConcurrency >= minCurrentConcurrency;
308
306
  if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
309
- this.scaleUp(systemStatus);
307
+ this.#scaleUp(systemStatus);
310
308
  const isSystemOverloaded = !isSystemIdle;
311
309
  const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;
312
310
  if (isSystemOverloaded && weAreNotAtMin)
313
- this.scaleDown(systemStatus);
311
+ this.#scaleDown(systemStatus);
314
312
  if (this.#loggingIntervalMillis > 0) {
315
313
  const now = Date.now();
316
314
  if (this.#lastLoggingTime == null) {
@@ -318,7 +316,7 @@ export class ConcurrencySystem {
318
316
  }
319
317
  else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
320
318
  this.#lastLoggingTime = now;
321
- this.log.info('state', {
319
+ this.#log.info('state', {
322
320
  currentConcurrency: this.#currentConcurrency,
323
321
  desiredConcurrency: this.#desiredConcurrency,
324
322
  systemStatus,
@@ -330,10 +328,10 @@ export class ConcurrencySystem {
330
328
  /**
331
329
  * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
332
330
  */
333
- scaleUp(systemStatus) {
334
- const step = Math.ceil(this.#desiredConcurrency * this.scaleUpStepRatio);
331
+ #scaleUp(systemStatus) {
332
+ const step = Math.ceil(this.#desiredConcurrency * this.#scaleUpStepRatio);
335
333
  this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);
336
- this.log.debug('scaling up', {
334
+ this.#log.debug('scaling up', {
337
335
  oldConcurrency: this.#desiredConcurrency - step,
338
336
  newConcurrency: this.#desiredConcurrency,
339
337
  systemStatus,
@@ -342,16 +340,16 @@ export class ConcurrencySystem {
342
340
  /**
343
341
  * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
344
342
  */
345
- scaleDown(systemStatus) {
346
- const step = Math.ceil(this.#desiredConcurrency * this.scaleDownStepRatio);
343
+ #scaleDown(systemStatus) {
344
+ const step = Math.ceil(this.#desiredConcurrency * this.#scaleDownStepRatio);
347
345
  this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);
348
- this.log.debug('scaling down', {
346
+ this.#log.debug('scaling down', {
349
347
  oldConcurrency: this.#desiredConcurrency + step,
350
348
  newConcurrency: this.#desiredConcurrency,
351
349
  systemStatus,
352
350
  });
353
351
  }
354
- incrementTasksDonePerSecond(intervalCallback) {
352
+ #incrementTasksDonePerSecond(intervalCallback) {
355
353
  this.#tasksPerMinute.unshift(0);
356
354
  this.#tasksPerMinute.pop();
357
355
  return intervalCallback();
@@ -42,6 +42,4 @@ export declare class MemoryLoadSignal implements LoadSignal {
42
42
  getSample(sampleDurationMillis?: number): LoadSnapshot[];
43
43
  /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
44
44
  handle(systemInfo: SystemInfo): void;
45
- private memoryOverloadWarning;
46
- private getTotalMemoryBytes;
47
45
  }
@@ -51,7 +51,7 @@ export class MemoryLoadSignal {
51
51
  'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
52
52
  }
53
53
  // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
54
- this.#maxMemoryBytes = await this.getTotalMemoryBytes();
54
+ this.#maxMemoryBytes = await this.#getTotalMemoryBytes();
55
55
  }
56
56
  this.#events.on(EventType.SYSTEM_INFO, this.handle);
57
57
  }
@@ -76,9 +76,9 @@ export class MemoryLoadSignal {
76
76
  usedBytes: memCurrentBytes,
77
77
  };
78
78
  this.#store.push(snapshot, createdAt);
79
- this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
79
+ this.#memoryOverloadWarning(systemInfo, maxMemoryBytes);
80
80
  }
81
- memoryOverloadWarning(systemInfo, maxMemoryBytes) {
81
+ #memoryOverloadWarning(systemInfo, maxMemoryBytes) {
82
82
  const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
83
83
  const { memCurrentBytes } = systemInfo;
84
84
  const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
@@ -97,7 +97,7 @@ export class MemoryLoadSignal {
97
97
  this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
98
98
  }
99
99
  }
100
- async getTotalMemoryBytes() {
100
+ async #getTotalMemoryBytes() {
101
101
  const containerized = this.#config.containerized ?? (await isContainerized());
102
102
  return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
103
103
  }
@@ -76,12 +76,6 @@ export declare class SystemStatus {
76
76
  * defaults.
77
77
  */
78
78
  get maxSampleWindowMillis(): number;
79
- /**
80
- * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
81
- * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
82
- * only the last is reported.
83
- */
84
- private assertUniqueSignalNames;
85
79
  /**
86
80
  * Returns an {@link SystemInfo} object with the following structure:
87
81
  *
@@ -115,8 +109,4 @@ export declare class SystemStatus {
115
109
  * `historySecs` seconds and `true` otherwise.
116
110
  */
117
111
  getHistoricalStatus(): SystemInfo;
118
- /**
119
- * Returns a system status object.
120
- */
121
- private isSystemIdle;
122
112
  }
@@ -40,7 +40,7 @@ export class SystemStatus {
40
40
  this.#currentHistoryMillis = currentHistorySecs * 1000;
41
41
  this.#historyMillis = historySecs * 1000;
42
42
  this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
43
- this.assertUniqueSignalNames();
43
+ this.#assertUniqueSignalNames();
44
44
  }
45
45
  /**
46
46
  * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
@@ -55,7 +55,7 @@ export class SystemStatus {
55
55
  * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
56
56
  * only the last is reported.
57
57
  */
58
- assertUniqueSignalNames() {
58
+ #assertUniqueSignalNames() {
59
59
  const seen = new Set();
60
60
  for (const { name } of this.#signals) {
61
61
  if (!seen.has(name)) {
@@ -85,7 +85,7 @@ export class SystemStatus {
85
85
  * and `true` otherwise.
86
86
  */
87
87
  getCurrentStatus() {
88
- return this.isSystemIdle(this.#currentHistoryMillis);
88
+ return this.#isSystemIdle(this.#currentHistoryMillis);
89
89
  }
90
90
  /**
91
91
  * Returns an {@link SystemInfo} object with the following structure:
@@ -103,12 +103,12 @@ export class SystemStatus {
103
103
  * `historySecs` seconds and `true` otherwise.
104
104
  */
105
105
  getHistoricalStatus() {
106
- return this.isSystemIdle(this.#historyMillis);
106
+ return this.#isSystemIdle(this.#historyMillis);
107
107
  }
108
108
  /**
109
109
  * Returns a system status object.
110
110
  */
111
- isSystemIdle(sampleDurationMillis) {
111
+ #isSystemIdle(sampleDurationMillis) {
112
112
  const result = {
113
113
  isSystemIdle: true,
114
114
  memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },