@crawlee/basic 4.0.0-beta.210 → 4.0.0-beta.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/autoscaling/autoscaled_pool.d.ts +0 -20
- package/internals/autoscaling/autoscaled_pool.js +11 -12
- package/internals/autoscaling/concurrency_system.d.ts +0 -38
- package/internals/autoscaling/concurrency_system.js +51 -53
- package/internals/autoscaling/memory_load_signal.d.ts +0 -2
- package/internals/autoscaling/memory_load_signal.js +4 -4
- package/internals/autoscaling/system_status.d.ts +0 -10
- package/internals/autoscaling/system_status.js +5 -5
- package/internals/basic-crawler.d.ts +0 -88
- package/internals/basic-crawler.js +70 -67
- package/internals/crawlers/context_pipeline.js +3 -3
- package/internals/crawlers/error_tracker.d.ts +0 -1
- package/internals/crawlers/error_tracker.js +3 -3
- package/internals/crawlers/statistics.d.ts +0 -2
- package/internals/crawlers/statistics.js +11 -11
- package/internals/crawling_request.d.ts +1 -1
- package/internals/crawling_request.js +12 -12
- package/internals/router.d.ts +0 -9
- package/internals/router.js +5 -5
- package/internals/session_pool/session.d.ts +0 -5
- package/internals/session_pool/session.js +3 -3
- package/internals/session_pool/session_pool.d.ts +0 -51
- package/internals/session_pool/session_pool.js +36 -36
- package/internals/sitemap_request_loader.d.ts +0 -39
- package/internals/sitemap_request_loader.js +18 -18
- package/internals/throttling_request_manager.d.ts +0 -1
- package/internals/throttling_request_manager.js +5 -5
- package/package.json +7 -7
|
@@ -175,24 +175,4 @@ export declare class AutoscaledPool {
|
|
|
175
175
|
* every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
|
|
176
176
|
*/
|
|
177
177
|
notify(): Promise<void>;
|
|
178
|
-
/**
|
|
179
|
-
* Starts a new task
|
|
180
|
-
* if the number of running tasks (current concurrency) is lower than desired concurrency
|
|
181
|
-
* and the system is not currently overloaded
|
|
182
|
-
* and `isTaskReadyFunction()` returns true.
|
|
183
|
-
*
|
|
184
|
-
* It doesn't allow multiple concurrent runs of this method.
|
|
185
|
-
*/
|
|
186
|
-
private maybeRunTask;
|
|
187
|
-
/**
|
|
188
|
-
* If there are no running tasks and `isFinishedFunction()` returns true then closes
|
|
189
|
-
* the pool and resolves the pool's promise returned by the run() method.
|
|
190
|
-
*
|
|
191
|
-
* It doesn't allow multiple concurrent runs of this method.
|
|
192
|
-
*/
|
|
193
|
-
private maybeFinish;
|
|
194
|
-
/**
|
|
195
|
-
* Cleans up resources.
|
|
196
|
-
*/
|
|
197
|
-
private destroy;
|
|
198
178
|
}
|
|
@@ -111,7 +111,6 @@ export class AutoscaledPool {
|
|
|
111
111
|
this.#isStopped = false;
|
|
112
112
|
this.#resolve = null;
|
|
113
113
|
this.#reject = null;
|
|
114
|
-
this.maybeRunTask = this.maybeRunTask.bind(this);
|
|
115
114
|
}
|
|
116
115
|
/**
|
|
117
116
|
* The governor backing this pool, as supplied to the constructor — exposed as the read-only
|
|
@@ -157,14 +156,14 @@ export class AutoscaledPool {
|
|
|
157
156
|
// This is here because if we scale down to let's say 1, then after each promise is finished
|
|
158
157
|
// this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
|
|
159
158
|
// in the crawler getting stuck and even after scaling up it never triggers another promise.
|
|
160
|
-
this.#maybeRunInterval = betterSetInterval(this
|
|
159
|
+
this.#maybeRunInterval = betterSetInterval((cb) => this.#maybeRunTask(cb), this.#maybeRunIntervalMillis);
|
|
161
160
|
try {
|
|
162
161
|
await poolPromise;
|
|
163
162
|
}
|
|
164
163
|
finally {
|
|
165
164
|
// If resolve is null, the pool is already destroyed.
|
|
166
165
|
if (this.#resolve)
|
|
167
|
-
await this
|
|
166
|
+
await this.#destroy();
|
|
168
167
|
}
|
|
169
168
|
}
|
|
170
169
|
/**
|
|
@@ -182,7 +181,7 @@ export class AutoscaledPool {
|
|
|
182
181
|
this.#isStopped = true;
|
|
183
182
|
if (this.#resolve) {
|
|
184
183
|
this.#resolve();
|
|
185
|
-
await this
|
|
184
|
+
await this.#destroy();
|
|
186
185
|
}
|
|
187
186
|
}
|
|
188
187
|
/**
|
|
@@ -241,7 +240,7 @@ export class AutoscaledPool {
|
|
|
241
240
|
* every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
|
|
242
241
|
*/
|
|
243
242
|
async notify() {
|
|
244
|
-
setImmediate(this
|
|
243
|
+
setImmediate(() => this.#maybeRunTask());
|
|
245
244
|
}
|
|
246
245
|
/**
|
|
247
246
|
* Starts a new task
|
|
@@ -251,7 +250,7 @@ export class AutoscaledPool {
|
|
|
251
250
|
*
|
|
252
251
|
* It doesn't allow multiple concurrent runs of this method.
|
|
253
252
|
*/
|
|
254
|
-
async maybeRunTask(intervalCallback) {
|
|
253
|
+
async #maybeRunTask(intervalCallback) {
|
|
255
254
|
this.#log.perf('Attempting to run a task.');
|
|
256
255
|
// Check if the function was invoked by the maybeRunInterval and use an empty function if not.
|
|
257
256
|
const done = intervalCallback || (() => { });
|
|
@@ -273,7 +272,7 @@ export class AutoscaledPool {
|
|
|
273
272
|
// to notice that *this* pool has run out of work — `maybeFinish()` is the only thing that ever resolves
|
|
274
273
|
// `run()`. It no-ops while this pool has tasks of its own in flight, which is every case in which an
|
|
275
274
|
// unshared governor reports no capacity.
|
|
276
|
-
return this
|
|
275
|
+
return this.#maybeFinish();
|
|
277
276
|
}
|
|
278
277
|
// - a task is ready.
|
|
279
278
|
this.#queryingIsTaskReady = true;
|
|
@@ -299,7 +298,7 @@ export class AutoscaledPool {
|
|
|
299
298
|
this.#log.perf('Task will not run. No tasks are ready.');
|
|
300
299
|
done();
|
|
301
300
|
// No tasks could mean that we're finished with all tasks.
|
|
302
|
-
return this
|
|
301
|
+
return this.#maybeFinish();
|
|
303
302
|
}
|
|
304
303
|
const taskId = String(this.#nextTaskId++);
|
|
305
304
|
// - the budget still has room. Re-checked atomically, because another pool sharing the governor may have taken
|
|
@@ -312,7 +311,7 @@ export class AutoscaledPool {
|
|
|
312
311
|
// Everything's fine. Run task.
|
|
313
312
|
// Try to run next task to build up concurrency,
|
|
314
313
|
// but defer it so it doesn't create a cycle.
|
|
315
|
-
setImmediate(this
|
|
314
|
+
setImmediate(() => this.#maybeRunTask());
|
|
316
315
|
// We need to restart interval here, so that it doesn't get blocked by a stalled task.
|
|
317
316
|
done();
|
|
318
317
|
// Execute the current task.
|
|
@@ -326,7 +325,7 @@ export class AutoscaledPool {
|
|
|
326
325
|
this.#log.perf('Task finished.');
|
|
327
326
|
// Run task after the previous one finished. Only on success: a failed task rejects the pool, and
|
|
328
327
|
// nudging the loop afterwards could start work on an already destroyed pool.
|
|
329
|
-
setImmediate(this
|
|
328
|
+
setImmediate(() => this.#maybeRunTask());
|
|
330
329
|
}
|
|
331
330
|
catch (e) {
|
|
332
331
|
const err = e;
|
|
@@ -354,7 +353,7 @@ export class AutoscaledPool {
|
|
|
354
353
|
*
|
|
355
354
|
* It doesn't allow multiple concurrent runs of this method.
|
|
356
355
|
*/
|
|
357
|
-
async maybeFinish() {
|
|
356
|
+
async #maybeFinish() {
|
|
358
357
|
if (this.#queryingIsFinished)
|
|
359
358
|
return;
|
|
360
359
|
if (this.#ownConcurrency > 0)
|
|
@@ -380,7 +379,7 @@ export class AutoscaledPool {
|
|
|
380
379
|
/**
|
|
381
380
|
* Cleans up resources.
|
|
382
381
|
*/
|
|
383
|
-
async destroy() {
|
|
382
|
+
async #destroy() {
|
|
384
383
|
this.#resolve = null;
|
|
385
384
|
this.#reject = null;
|
|
386
385
|
betterClearInterval(this.#maybeRunInterval);
|
|
@@ -159,14 +159,8 @@ export interface IConcurrencySystem {
|
|
|
159
159
|
*/
|
|
160
160
|
export declare class ConcurrencySystem implements IConcurrencySystem {
|
|
161
161
|
#private;
|
|
162
|
-
private readonly log;
|
|
163
|
-
private readonly desiredConcurrencyRatio;
|
|
164
|
-
private readonly scaleUpStepRatio;
|
|
165
|
-
private readonly scaleDownStepRatio;
|
|
166
162
|
/** The cap on tasks started per minute, or `Infinity` when uncapped. */
|
|
167
163
|
readonly maxTasksPerMinute: number;
|
|
168
|
-
private readonly snapshotter;
|
|
169
|
-
private readonly systemStatus;
|
|
170
164
|
constructor(options?: ConcurrencySystemOptions);
|
|
171
165
|
/**
|
|
172
166
|
* Gets the minimum number of tasks running in parallel.
|
|
@@ -198,13 +192,6 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
|
|
|
198
192
|
* Tune the budget through `minConcurrency`/`maxConcurrency` instead.
|
|
199
193
|
*/
|
|
200
194
|
get desiredConcurrency(): number;
|
|
201
|
-
/**
|
|
202
|
-
* Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
|
|
203
|
-
* Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
|
|
204
|
-
* meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
|
|
205
|
-
* that is the limit callers set in order to protect something.
|
|
206
|
-
*/
|
|
207
|
-
private clampDesiredConcurrency;
|
|
208
195
|
get currentConcurrency(): number;
|
|
209
196
|
/** Whether the system is currently monitoring load and autoscaling the budget. */
|
|
210
197
|
get isRunning(): boolean;
|
|
@@ -214,20 +201,11 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
|
|
|
214
201
|
* signal fails to start.
|
|
215
202
|
*/
|
|
216
203
|
start(): Promise<void>;
|
|
217
|
-
private boot;
|
|
218
204
|
[Symbol.asyncDispose](): Promise<void>;
|
|
219
205
|
/**
|
|
220
206
|
* Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
|
|
221
207
|
*/
|
|
222
208
|
stop(): Promise<void>;
|
|
223
|
-
private shutDown;
|
|
224
|
-
/**
|
|
225
|
-
* Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
|
|
226
|
-
* else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
|
|
227
|
-
* {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
|
|
228
|
-
* `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
|
|
229
|
-
*/
|
|
230
|
-
private warnIfNotRunning;
|
|
231
209
|
/**
|
|
232
210
|
* May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
|
|
233
211
|
* reached) or when the machine is overloaded past `minConcurrency`.
|
|
@@ -236,8 +214,6 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
|
|
|
236
214
|
* interface, letting the answer be queried directly.
|
|
237
215
|
*/
|
|
238
216
|
hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
|
|
239
|
-
/** Whether the per-minute task cap has been reached. */
|
|
240
|
-
private get isOverMaxRequestLimit();
|
|
241
217
|
/**
|
|
242
218
|
* Atomically books a task against the shared budget: re-checks
|
|
243
219
|
* {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
|
|
@@ -256,18 +232,4 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
|
|
|
256
232
|
* The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
|
|
257
233
|
*/
|
|
258
234
|
getCurrentStatus(): SystemInfo;
|
|
259
|
-
/**
|
|
260
|
-
* Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
|
|
261
|
-
* by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
|
|
262
|
-
*/
|
|
263
|
-
private autoscale;
|
|
264
|
-
/**
|
|
265
|
-
* Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
|
|
266
|
-
*/
|
|
267
|
-
private scaleUp;
|
|
268
|
-
/**
|
|
269
|
-
* Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
|
|
270
|
-
*/
|
|
271
|
-
private scaleDown;
|
|
272
|
-
private incrementTasksDonePerSecond;
|
|
273
235
|
}
|
|
@@ -44,10 +44,10 @@ const concurrencySystemOptionsSchema = z.strictObject({
|
|
|
44
44
|
* @category Scaling
|
|
45
45
|
*/
|
|
46
46
|
export class ConcurrencySystem {
|
|
47
|
-
log;
|
|
48
|
-
desiredConcurrencyRatio;
|
|
49
|
-
scaleUpStepRatio;
|
|
50
|
-
scaleDownStepRatio;
|
|
47
|
+
#log;
|
|
48
|
+
#desiredConcurrencyRatio;
|
|
49
|
+
#scaleUpStepRatio;
|
|
50
|
+
#scaleDownStepRatio;
|
|
51
51
|
#loggingIntervalMillis;
|
|
52
52
|
#autoscaleIntervalMillis;
|
|
53
53
|
/** The cap on tasks started per minute, or `Infinity` when uncapped. */
|
|
@@ -58,9 +58,9 @@ export class ConcurrencySystem {
|
|
|
58
58
|
#currentConcurrency = 0;
|
|
59
59
|
#lastLoggingTime;
|
|
60
60
|
#tasksPerMinute = Array.from({ length: 60 }, () => 0);
|
|
61
|
-
snapshotter;
|
|
61
|
+
#snapshotter;
|
|
62
62
|
#loadSignals;
|
|
63
|
-
systemStatus;
|
|
63
|
+
#systemStatus;
|
|
64
64
|
#autoscaleInterval;
|
|
65
65
|
#tasksDonePerSecondInterval;
|
|
66
66
|
/** Whether the snapshotter and autoscaling intervals are currently running. */
|
|
@@ -71,25 +71,23 @@ export class ConcurrencySystem {
|
|
|
71
71
|
#warnedAboutQueryWhileStopped = false;
|
|
72
72
|
constructor(options = {}) {
|
|
73
73
|
const { maxConcurrency, minConcurrency, desiredConcurrency, desiredConcurrencyRatio, scaleUpStepRatio, scaleDownStepRatio, loggingIntervalSecs, autoscaleIntervalSecs, loadSignals, snapshotHistorySecs, currentHistorySecs, log, maxTasksPerMinute, } = parseArgument(options, concurrencySystemOptionsSchema, 'ConcurrencySystemOptions');
|
|
74
|
-
this
|
|
75
|
-
this
|
|
76
|
-
this
|
|
77
|
-
this
|
|
74
|
+
this.#log = log.child({ prefix: 'ConcurrencySystem' });
|
|
75
|
+
this.#desiredConcurrencyRatio = desiredConcurrencyRatio;
|
|
76
|
+
this.#scaleUpStepRatio = scaleUpStepRatio;
|
|
77
|
+
this.#scaleDownStepRatio = scaleDownStepRatio;
|
|
78
78
|
this.#loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
|
|
79
79
|
this.#autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
|
|
80
80
|
this.maxTasksPerMinute = maxTasksPerMinute;
|
|
81
81
|
this.#minConcurrency = minConcurrency;
|
|
82
82
|
this.#maxConcurrency = maxConcurrency;
|
|
83
83
|
this.#desiredConcurrency = desiredConcurrency ?? minConcurrency;
|
|
84
|
-
this
|
|
85
|
-
this.autoscale = this.autoscale.bind(this);
|
|
86
|
-
this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
|
|
84
|
+
this.#clampDesiredConcurrency();
|
|
87
85
|
// The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
|
|
88
86
|
const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
|
|
89
|
-
this
|
|
87
|
+
this.#snapshotter = new Snapshotter(builtinSignalOptions);
|
|
90
88
|
this.#loadSignals = customLoadSignals;
|
|
91
|
-
this
|
|
92
|
-
snapshotter: this
|
|
89
|
+
this.#systemStatus = new SystemStatus({
|
|
90
|
+
snapshotter: this.#snapshotter,
|
|
93
91
|
loadSignals: customLoadSignals,
|
|
94
92
|
currentHistorySecs,
|
|
95
93
|
// Both windows are requested from the signals explicitly, so a signal's own retention can neither widen
|
|
@@ -112,7 +110,7 @@ export class ConcurrencySystem {
|
|
|
112
110
|
set minConcurrency(value) {
|
|
113
111
|
parseArgument(value, concurrencySchema);
|
|
114
112
|
this.#minConcurrency = value;
|
|
115
|
-
this
|
|
113
|
+
this.#clampDesiredConcurrency();
|
|
116
114
|
}
|
|
117
115
|
/**
|
|
118
116
|
* Gets the maximum number of tasks running in parallel.
|
|
@@ -129,7 +127,7 @@ export class ConcurrencySystem {
|
|
|
129
127
|
set maxConcurrency(value) {
|
|
130
128
|
parseArgument(value, concurrencySchema);
|
|
131
129
|
this.#maxConcurrency = value;
|
|
132
|
-
this
|
|
130
|
+
this.#clampDesiredConcurrency();
|
|
133
131
|
}
|
|
134
132
|
/**
|
|
135
133
|
* Gets the desired concurrency for the system,
|
|
@@ -147,7 +145,7 @@ export class ConcurrencySystem {
|
|
|
147
145
|
* meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
|
|
148
146
|
* that is the limit callers set in order to protect something.
|
|
149
147
|
*/
|
|
150
|
-
clampDesiredConcurrency() {
|
|
148
|
+
#clampDesiredConcurrency() {
|
|
151
149
|
const atLeastMin = Math.max(this.#desiredConcurrency, this.#minConcurrency);
|
|
152
150
|
this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
|
|
153
151
|
}
|
|
@@ -166,14 +164,14 @@ export class ConcurrencySystem {
|
|
|
166
164
|
async start() {
|
|
167
165
|
// Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
|
|
168
166
|
// a system that is down.
|
|
169
|
-
this.#startPromise ??= this
|
|
167
|
+
this.#startPromise ??= this.#boot().catch(async (error) => {
|
|
170
168
|
this.#startPromise = undefined;
|
|
171
|
-
await this
|
|
169
|
+
await this.#shutDown();
|
|
172
170
|
throw error;
|
|
173
171
|
});
|
|
174
172
|
await this.#startPromise;
|
|
175
173
|
}
|
|
176
|
-
async boot() {
|
|
174
|
+
async #boot() {
|
|
177
175
|
// Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
|
|
178
176
|
// per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
|
|
179
177
|
// arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
|
|
@@ -182,12 +180,12 @@ export class ConcurrencySystem {
|
|
|
182
180
|
this.#warnedAboutQueryWhileStopped = false;
|
|
183
181
|
// Signals are told how much history to keep when they start: exactly the longest window they will be sampled
|
|
184
182
|
// over, so nobody has to guess a retention value that matches this system's configuration.
|
|
185
|
-
const startContext = { maxSampleWindowMillis: this
|
|
186
|
-
await this
|
|
183
|
+
const startContext = { maxSampleWindowMillis: this.#systemStatus.maxSampleWindowMillis };
|
|
184
|
+
await this.#snapshotter.start(startContext);
|
|
187
185
|
await Promise.all(this.#loadSignals.map(async (s) => s.start(startContext)));
|
|
188
|
-
this.#autoscaleInterval = betterSetInterval(this
|
|
186
|
+
this.#autoscaleInterval = betterSetInterval((cb) => this.#autoscale(cb), this.#autoscaleIntervalMillis);
|
|
189
187
|
if (this.maxTasksPerMinute !== Infinity) {
|
|
190
|
-
this.#tasksDonePerSecondInterval = betterSetInterval(this
|
|
188
|
+
this.#tasksDonePerSecondInterval = betterSetInterval((cb) => this.#incrementTasksDonePerSecond(cb), 1000);
|
|
191
189
|
}
|
|
192
190
|
// Last, so `isRunning` never claims a system whose signals aren't collecting yet.
|
|
193
191
|
this.#running = true;
|
|
@@ -206,14 +204,14 @@ export class ConcurrencySystem {
|
|
|
206
204
|
await this.#startPromise.catch(() => { });
|
|
207
205
|
this.#startPromise = undefined;
|
|
208
206
|
this.#running = false;
|
|
209
|
-
await this
|
|
207
|
+
await this.#shutDown();
|
|
210
208
|
}
|
|
211
|
-
async shutDown() {
|
|
209
|
+
async #shutDown() {
|
|
212
210
|
if (this.#autoscaleInterval)
|
|
213
211
|
betterClearInterval(this.#autoscaleInterval);
|
|
214
212
|
if (this.#tasksDonePerSecondInterval)
|
|
215
213
|
betterClearInterval(this.#tasksDonePerSecondInterval);
|
|
216
|
-
await this
|
|
214
|
+
await this.#snapshotter.stop();
|
|
217
215
|
await Promise.all(this.#loadSignals.map(async (s) => s.stop()));
|
|
218
216
|
}
|
|
219
217
|
/**
|
|
@@ -222,12 +220,12 @@ export class ConcurrencySystem {
|
|
|
222
220
|
* {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
|
|
223
221
|
* `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
|
|
224
222
|
*/
|
|
225
|
-
warnIfNotRunning() {
|
|
223
|
+
#warnIfNotRunning() {
|
|
226
224
|
if (this.#running || this.#warnedAboutQueryWhileStopped) {
|
|
227
225
|
return;
|
|
228
226
|
}
|
|
229
227
|
this.#warnedAboutQueryWhileStopped = true;
|
|
230
|
-
this
|
|
228
|
+
this.#log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
|
|
231
229
|
'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
|
|
232
230
|
'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
|
|
233
231
|
'has finished.');
|
|
@@ -240,21 +238,21 @@ export class ConcurrencySystem {
|
|
|
240
238
|
* interface, letting the answer be queried directly.
|
|
241
239
|
*/
|
|
242
240
|
hasCapacityForTask(_consumer) {
|
|
243
|
-
this
|
|
241
|
+
this.#warnIfNotRunning();
|
|
244
242
|
if (this.#currentConcurrency >= this.#desiredConcurrency) {
|
|
245
|
-
this
|
|
243
|
+
this.#log.perf('Task will not run. Desired concurrency achieved.');
|
|
246
244
|
return false;
|
|
247
245
|
}
|
|
248
|
-
const currentStatus = this
|
|
246
|
+
const currentStatus = this.#systemStatus.getCurrentStatus();
|
|
249
247
|
const { isSystemIdle } = currentStatus;
|
|
250
248
|
if (!isSystemIdle && this.#currentConcurrency >= this.#minConcurrency) {
|
|
251
|
-
this
|
|
249
|
+
this.#log.perf('Task will not be run. System is overloaded.', currentStatus);
|
|
252
250
|
return false;
|
|
253
251
|
}
|
|
254
252
|
return true;
|
|
255
253
|
}
|
|
256
254
|
/** Whether the per-minute task cap has been reached. */
|
|
257
|
-
get isOverMaxRequestLimit() {
|
|
255
|
+
get #isOverMaxRequestLimit() {
|
|
258
256
|
if (this.maxTasksPerMinute === Infinity) {
|
|
259
257
|
return false;
|
|
260
258
|
}
|
|
@@ -273,8 +271,8 @@ export class ConcurrencySystem {
|
|
|
273
271
|
if (!this.hasCapacityForTask(consumer)) {
|
|
274
272
|
return false;
|
|
275
273
|
}
|
|
276
|
-
if (this
|
|
277
|
-
this
|
|
274
|
+
if (this.#isOverMaxRequestLimit) {
|
|
275
|
+
this.#log.perf('Task will not run. Maximum tasks per minute reached.');
|
|
278
276
|
return false;
|
|
279
277
|
}
|
|
280
278
|
this.#currentConcurrency++;
|
|
@@ -291,26 +289,26 @@ export class ConcurrencySystem {
|
|
|
291
289
|
* The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
|
|
292
290
|
*/
|
|
293
291
|
getCurrentStatus() {
|
|
294
|
-
return this
|
|
292
|
+
return this.#systemStatus.getCurrentStatus();
|
|
295
293
|
}
|
|
296
294
|
/**
|
|
297
295
|
* Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
|
|
298
296
|
* by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
|
|
299
297
|
*/
|
|
300
|
-
autoscale(intervalCallback) {
|
|
301
|
-
if (this
|
|
298
|
+
#autoscale(intervalCallback) {
|
|
299
|
+
if (this.#isOverMaxRequestLimit)
|
|
302
300
|
return intervalCallback();
|
|
303
|
-
const systemStatus = this
|
|
301
|
+
const systemStatus = this.#systemStatus.getHistoricalStatus();
|
|
304
302
|
const { isSystemIdle } = systemStatus;
|
|
305
303
|
const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
|
|
306
|
-
const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this
|
|
304
|
+
const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.#desiredConcurrencyRatio);
|
|
307
305
|
const weAreReachingDesiredConcurrency = this.#currentConcurrency >= minCurrentConcurrency;
|
|
308
306
|
if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
|
|
309
|
-
this
|
|
307
|
+
this.#scaleUp(systemStatus);
|
|
310
308
|
const isSystemOverloaded = !isSystemIdle;
|
|
311
309
|
const weAreNotAtMin = this.#desiredConcurrency > this.#minConcurrency;
|
|
312
310
|
if (isSystemOverloaded && weAreNotAtMin)
|
|
313
|
-
this
|
|
311
|
+
this.#scaleDown(systemStatus);
|
|
314
312
|
if (this.#loggingIntervalMillis > 0) {
|
|
315
313
|
const now = Date.now();
|
|
316
314
|
if (this.#lastLoggingTime == null) {
|
|
@@ -318,7 +316,7 @@ export class ConcurrencySystem {
|
|
|
318
316
|
}
|
|
319
317
|
else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
|
|
320
318
|
this.#lastLoggingTime = now;
|
|
321
|
-
this
|
|
319
|
+
this.#log.info('state', {
|
|
322
320
|
currentConcurrency: this.#currentConcurrency,
|
|
323
321
|
desiredConcurrency: this.#desiredConcurrency,
|
|
324
322
|
systemStatus,
|
|
@@ -330,10 +328,10 @@ export class ConcurrencySystem {
|
|
|
330
328
|
/**
|
|
331
329
|
* Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
|
|
332
330
|
*/
|
|
333
|
-
scaleUp(systemStatus) {
|
|
334
|
-
const step = Math.ceil(this.#desiredConcurrency * this
|
|
331
|
+
#scaleUp(systemStatus) {
|
|
332
|
+
const step = Math.ceil(this.#desiredConcurrency * this.#scaleUpStepRatio);
|
|
335
333
|
this.#desiredConcurrency = Math.min(this.#maxConcurrency, this.#desiredConcurrency + step);
|
|
336
|
-
this
|
|
334
|
+
this.#log.debug('scaling up', {
|
|
337
335
|
oldConcurrency: this.#desiredConcurrency - step,
|
|
338
336
|
newConcurrency: this.#desiredConcurrency,
|
|
339
337
|
systemStatus,
|
|
@@ -342,16 +340,16 @@ export class ConcurrencySystem {
|
|
|
342
340
|
/**
|
|
343
341
|
* Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
|
|
344
342
|
*/
|
|
345
|
-
scaleDown(systemStatus) {
|
|
346
|
-
const step = Math.ceil(this.#desiredConcurrency * this
|
|
343
|
+
#scaleDown(systemStatus) {
|
|
344
|
+
const step = Math.ceil(this.#desiredConcurrency * this.#scaleDownStepRatio);
|
|
347
345
|
this.#desiredConcurrency = Math.max(this.#minConcurrency, this.#desiredConcurrency - step);
|
|
348
|
-
this
|
|
346
|
+
this.#log.debug('scaling down', {
|
|
349
347
|
oldConcurrency: this.#desiredConcurrency + step,
|
|
350
348
|
newConcurrency: this.#desiredConcurrency,
|
|
351
349
|
systemStatus,
|
|
352
350
|
});
|
|
353
351
|
}
|
|
354
|
-
incrementTasksDonePerSecond(intervalCallback) {
|
|
352
|
+
#incrementTasksDonePerSecond(intervalCallback) {
|
|
355
353
|
this.#tasksPerMinute.unshift(0);
|
|
356
354
|
this.#tasksPerMinute.pop();
|
|
357
355
|
return intervalCallback();
|
|
@@ -42,6 +42,4 @@ export declare class MemoryLoadSignal implements LoadSignal {
|
|
|
42
42
|
getSample(sampleDurationMillis?: number): LoadSnapshot[];
|
|
43
43
|
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
44
44
|
handle(systemInfo: SystemInfo): void;
|
|
45
|
-
private memoryOverloadWarning;
|
|
46
|
-
private getTotalMemoryBytes;
|
|
47
45
|
}
|
|
@@ -51,7 +51,7 @@ export class MemoryLoadSignal {
|
|
|
51
51
|
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
52
52
|
}
|
|
53
53
|
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
54
|
-
this.#maxMemoryBytes = await this
|
|
54
|
+
this.#maxMemoryBytes = await this.#getTotalMemoryBytes();
|
|
55
55
|
}
|
|
56
56
|
this.#events.on(EventType.SYSTEM_INFO, this.handle);
|
|
57
57
|
}
|
|
@@ -76,9 +76,9 @@ export class MemoryLoadSignal {
|
|
|
76
76
|
usedBytes: memCurrentBytes,
|
|
77
77
|
};
|
|
78
78
|
this.#store.push(snapshot, createdAt);
|
|
79
|
-
this
|
|
79
|
+
this.#memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
80
80
|
}
|
|
81
|
-
memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
81
|
+
#memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
82
82
|
const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
|
|
83
83
|
const { memCurrentBytes } = systemInfo;
|
|
84
84
|
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
@@ -97,7 +97,7 @@ export class MemoryLoadSignal {
|
|
|
97
97
|
this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
|
-
async getTotalMemoryBytes() {
|
|
100
|
+
async #getTotalMemoryBytes() {
|
|
101
101
|
const containerized = this.#config.containerized ?? (await isContainerized());
|
|
102
102
|
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
103
103
|
}
|
|
@@ -76,12 +76,6 @@ export declare class SystemStatus {
|
|
|
76
76
|
* defaults.
|
|
77
77
|
*/
|
|
78
78
|
get maxSampleWindowMillis(): number;
|
|
79
|
-
/**
|
|
80
|
-
* Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
|
|
81
|
-
* contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
|
|
82
|
-
* only the last is reported.
|
|
83
|
-
*/
|
|
84
|
-
private assertUniqueSignalNames;
|
|
85
79
|
/**
|
|
86
80
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
87
81
|
*
|
|
@@ -115,8 +109,4 @@ export declare class SystemStatus {
|
|
|
115
109
|
* `historySecs` seconds and `true` otherwise.
|
|
116
110
|
*/
|
|
117
111
|
getHistoricalStatus(): SystemInfo;
|
|
118
|
-
/**
|
|
119
|
-
* Returns a system status object.
|
|
120
|
-
*/
|
|
121
|
-
private isSystemIdle;
|
|
122
112
|
}
|
|
@@ -40,7 +40,7 @@ export class SystemStatus {
|
|
|
40
40
|
this.#currentHistoryMillis = currentHistorySecs * 1000;
|
|
41
41
|
this.#historyMillis = historySecs * 1000;
|
|
42
42
|
this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
|
|
43
|
-
this
|
|
43
|
+
this.#assertUniqueSignalNames();
|
|
44
44
|
}
|
|
45
45
|
/**
|
|
46
46
|
* The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
|
|
@@ -55,7 +55,7 @@ export class SystemStatus {
|
|
|
55
55
|
* contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
|
|
56
56
|
* only the last is reported.
|
|
57
57
|
*/
|
|
58
|
-
assertUniqueSignalNames() {
|
|
58
|
+
#assertUniqueSignalNames() {
|
|
59
59
|
const seen = new Set();
|
|
60
60
|
for (const { name } of this.#signals) {
|
|
61
61
|
if (!seen.has(name)) {
|
|
@@ -85,7 +85,7 @@ export class SystemStatus {
|
|
|
85
85
|
* and `true` otherwise.
|
|
86
86
|
*/
|
|
87
87
|
getCurrentStatus() {
|
|
88
|
-
return this
|
|
88
|
+
return this.#isSystemIdle(this.#currentHistoryMillis);
|
|
89
89
|
}
|
|
90
90
|
/**
|
|
91
91
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -103,12 +103,12 @@ export class SystemStatus {
|
|
|
103
103
|
* `historySecs` seconds and `true` otherwise.
|
|
104
104
|
*/
|
|
105
105
|
getHistoricalStatus() {
|
|
106
|
-
return this
|
|
106
|
+
return this.#isSystemIdle(this.#historyMillis);
|
|
107
107
|
}
|
|
108
108
|
/**
|
|
109
109
|
* Returns a system status object.
|
|
110
110
|
*/
|
|
111
|
-
isSystemIdle(sampleDurationMillis) {
|
|
111
|
+
#isSystemIdle(sampleDurationMillis) {
|
|
112
112
|
const result = {
|
|
113
113
|
isSystemIdle: true,
|
|
114
114
|
memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|