@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -21
- package/autoscaling/autoscaled_pool.js +85 -85
- package/autoscaling/client_load_signal.d.ts +1 -5
- package/autoscaling/client_load_signal.js +20 -20
- package/autoscaling/concurrency_system.d.ts +5 -20
- package/autoscaling/concurrency_system.js +81 -80
- package/autoscaling/cpu_load_signal.d.ts +1 -2
- package/autoscaling/cpu_load_signal.js +10 -10
- package/autoscaling/event_loop_load_signal.d.ts +1 -4
- package/autoscaling/event_loop_load_signal.js +18 -18
- package/autoscaling/load_signal.d.ts +1 -1
- package/autoscaling/load_signal.js +12 -11
- package/autoscaling/memory_load_signal.d.ts +3 -12
- package/autoscaling/memory_load_signal.js +40 -41
- package/autoscaling/snapshotter.d.ts +1 -4
- package/autoscaling/snapshotter.js +12 -12
- package/autoscaling/system_status.d.ts +1 -3
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +1 -1
- package/configuration.js +3 -3
- package/crawlers/context_pipeline.js +6 -6
- package/crawlers/statistics.d.ts +1 -8
- package/crawlers/statistics.js +45 -44
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.js +3 -3
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +3 -3
- package/log.js +5 -1
- package/memory-storage/memory-storage.d.ts +1 -5
- package/memory-storage/memory-storage.js +2 -2
- package/memory-storage/resource-clients/dataset.d.ts +1 -1
- package/memory-storage/resource-clients/dataset.js +6 -5
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
- package/memory-storage/resource-clients/key-value-store.js +13 -12
- package/memory-storage/resource-clients/request-queue.d.ts +4 -23
- package/memory-storage/resource-clients/request-queue.js +59 -58
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +5 -5
- package/proxy_configuration.d.ts +1 -3
- package/proxy_configuration.js +8 -8
- package/recoverable_state.d.ts +1 -10
- package/recoverable_state.js +41 -41
- package/request.d.ts +1 -2
- package/request.js +10 -13
- package/router.d.ts +1 -4
- package/router.js +23 -23
- package/serialization.js +8 -9
- package/service_locator.d.ts +1 -10
- package/service_locator.js +48 -48
- package/session_pool/session.d.ts +1 -12
- package/session_pool/session.js +50 -50
- package/session_pool/session_pool.d.ts +2 -11
- package/session_pool/session_pool.js +59 -58
- package/storages/dataset.d.ts +1 -1
- package/storages/dataset.js +5 -5
- package/storages/key_value_store.d.ts +1 -4
- package/storages/key_value_store.js +21 -20
- package/storages/request_dedup_cache.d.ts +1 -2
- package/storages/request_dedup_cache.js +9 -9
- package/storages/request_list.d.ts +2 -22
- package/storages/request_list.js +74 -73
- package/storages/request_manager_tandem.d.ts +1 -10
- package/storages/request_manager_tandem.js +27 -27
- package/storages/request_queue.d.ts +2 -18
- package/storages/request_queue.js +37 -35
- package/storages/sitemap_request_loader.d.ts +1 -44
- package/storages/sitemap_request_loader.js +87 -87
- package/storages/storage_instance_manager.d.ts +1 -2
- package/storages/storage_instance_manager.js +17 -17
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/transaction.d.ts +1 -3
- package/storages/transaction.js +17 -17
- package/system-info/runtime.js +7 -7
|
@@ -15,92 +15,91 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
|
|
|
15
15
|
export class MemoryLoadSignal {
|
|
16
16
|
name = 'memInfo';
|
|
17
17
|
overloadedRatio;
|
|
18
|
-
store = new SnapshotStore();
|
|
19
|
-
maxUsedRatio;
|
|
18
|
+
#store = new SnapshotStore();
|
|
19
|
+
#maxUsedRatio;
|
|
20
20
|
/** All resolved in `start()`, before anything that reads them can fire. */
|
|
21
|
-
config;
|
|
22
|
-
log;
|
|
23
|
-
maxMemoryBytes;
|
|
24
|
-
events;
|
|
25
|
-
maxMemoryRatio;
|
|
26
|
-
lastLoggedCriticalMemoryOverloadAt = null;
|
|
21
|
+
#config;
|
|
22
|
+
#log;
|
|
23
|
+
#maxMemoryBytes;
|
|
24
|
+
#events;
|
|
25
|
+
#maxMemoryRatio;
|
|
26
|
+
#lastLoggedCriticalMemoryOverloadAt = null;
|
|
27
27
|
constructor(options = {}) {
|
|
28
|
-
this
|
|
28
|
+
this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;
|
|
29
29
|
this.overloadedRatio = options.overloadedRatio ?? 0.2;
|
|
30
30
|
this.handle = this.handle.bind(this);
|
|
31
31
|
}
|
|
32
32
|
async start(context) {
|
|
33
|
-
this
|
|
33
|
+
this.#store.useSampleWindow(context.maxSampleWindowMillis);
|
|
34
34
|
// A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
|
|
35
|
-
this
|
|
35
|
+
this.#store.clear();
|
|
36
36
|
// Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
|
|
37
37
|
// between systems) must not capture whichever services happened to be registered at that moment.
|
|
38
|
-
this
|
|
39
|
-
this
|
|
40
|
-
this
|
|
41
|
-
const memoryMbytes = this
|
|
38
|
+
this.#config = serviceLocator.getConfiguration();
|
|
39
|
+
this.#events = serviceLocator.getEventManager();
|
|
40
|
+
this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
41
|
+
const memoryMbytes = this.#config.memoryMbytes ?? 0;
|
|
42
42
|
if (memoryMbytes > 0) {
|
|
43
|
-
this
|
|
43
|
+
this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
44
44
|
}
|
|
45
45
|
else {
|
|
46
|
-
this
|
|
47
|
-
if (!this
|
|
46
|
+
this.#maxMemoryRatio = this.#config.availableMemoryRatio;
|
|
47
|
+
if (!this.#maxMemoryRatio) {
|
|
48
48
|
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
49
49
|
}
|
|
50
50
|
else {
|
|
51
|
-
this
|
|
51
|
+
this.#log.debug(`Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +
|
|
52
52
|
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
53
53
|
}
|
|
54
54
|
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
|
|
55
|
-
this
|
|
55
|
+
this.#maxMemoryBytes = await this.getTotalMemoryBytes();
|
|
56
56
|
}
|
|
57
|
-
this
|
|
57
|
+
this.#events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
58
58
|
}
|
|
59
59
|
async stop() {
|
|
60
|
-
this
|
|
61
|
-
this
|
|
60
|
+
this.#events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
|
|
61
|
+
this.#events = undefined;
|
|
62
62
|
}
|
|
63
63
|
getSample(sampleDurationMillis) {
|
|
64
|
-
return this
|
|
64
|
+
return this.#store.getSample(sampleDurationMillis);
|
|
65
65
|
}
|
|
66
66
|
/** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
|
|
67
67
|
handle(systemInfo) {
|
|
68
68
|
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
69
69
|
const { memCurrentBytes, memTotalBytes } = systemInfo;
|
|
70
|
-
let maxMemoryBytes = this
|
|
71
|
-
if (this
|
|
72
|
-
maxMemoryBytes = this
|
|
70
|
+
let maxMemoryBytes = this.#maxMemoryBytes;
|
|
71
|
+
if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {
|
|
72
|
+
maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);
|
|
73
73
|
}
|
|
74
74
|
const snapshot = {
|
|
75
75
|
createdAt,
|
|
76
|
-
isOverloaded: memCurrentBytes / maxMemoryBytes > this
|
|
76
|
+
isOverloaded: memCurrentBytes / maxMemoryBytes > this.#maxUsedRatio,
|
|
77
77
|
usedBytes: memCurrentBytes,
|
|
78
78
|
};
|
|
79
|
-
this
|
|
80
|
-
this.
|
|
79
|
+
this.#store.push(snapshot, createdAt);
|
|
80
|
+
this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
|
|
81
81
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
|
|
82
|
+
memoryOverloadWarning(systemInfo, maxMemoryBytes) {
|
|
83
|
+
const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
|
|
85
84
|
const { memCurrentBytes } = systemInfo;
|
|
86
85
|
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
|
|
87
|
-
if (this
|
|
88
|
-
+createdAt < +this
|
|
86
|
+
if (this.#lastLoggedCriticalMemoryOverloadAt &&
|
|
87
|
+
+createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
|
|
89
88
|
return;
|
|
90
|
-
const maxDesiredMemoryBytes = this
|
|
91
|
-
const reserveMemory = effectiveMax * (1 - this
|
|
89
|
+
const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;
|
|
90
|
+
const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;
|
|
92
91
|
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
|
|
93
92
|
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
|
|
94
93
|
if (isCriticalOverload) {
|
|
95
94
|
const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
|
|
96
95
|
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
|
|
97
|
-
this
|
|
96
|
+
this.#log.warning('Memory is critically overloaded. ' +
|
|
98
97
|
`Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
|
|
99
|
-
this
|
|
98
|
+
this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
|
|
100
99
|
}
|
|
101
100
|
}
|
|
102
|
-
async
|
|
103
|
-
const containerized = this
|
|
101
|
+
async getTotalMemoryBytes() {
|
|
102
|
+
const containerized = this.#config.containerized ?? (await isContainerized());
|
|
104
103
|
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
105
104
|
}
|
|
106
105
|
}
|
|
@@ -61,10 +61,7 @@ export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
|
|
|
61
61
|
* @internal
|
|
62
62
|
*/
|
|
63
63
|
export declare class Snapshotter {
|
|
64
|
-
private
|
|
65
|
-
private readonly eventLoopSignal?;
|
|
66
|
-
private readonly cpuSignal?;
|
|
67
|
-
private readonly clientSignal?;
|
|
64
|
+
#private;
|
|
68
65
|
/**
|
|
69
66
|
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
70
67
|
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
@@ -13,10 +13,10 @@ import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
|
13
13
|
*/
|
|
14
14
|
export class Snapshotter {
|
|
15
15
|
// Absent when switched off through the corresponding option (e.g. `client: false`).
|
|
16
|
-
memorySignal;
|
|
17
|
-
eventLoopSignal;
|
|
18
|
-
cpuSignal;
|
|
19
|
-
clientSignal;
|
|
16
|
+
#memorySignal;
|
|
17
|
+
#eventLoopSignal;
|
|
18
|
+
#cpuSignal;
|
|
19
|
+
#clientSignal;
|
|
20
20
|
/**
|
|
21
21
|
* Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
|
|
22
22
|
* instances. Signals switched off through the options are simply absent — the system status reports them as
|
|
@@ -24,10 +24,10 @@ export class Snapshotter {
|
|
|
24
24
|
*/
|
|
25
25
|
getLoadSignals() {
|
|
26
26
|
const builtin = [
|
|
27
|
-
this
|
|
28
|
-
this
|
|
29
|
-
this
|
|
30
|
-
this
|
|
27
|
+
this.#memorySignal,
|
|
28
|
+
this.#eventLoopSignal,
|
|
29
|
+
this.#cpuSignal,
|
|
30
|
+
this.#clientSignal,
|
|
31
31
|
];
|
|
32
32
|
return builtin.filter((signal) => signal !== undefined);
|
|
33
33
|
}
|
|
@@ -39,13 +39,13 @@ export class Snapshotter {
|
|
|
39
39
|
// Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
|
|
40
40
|
// over then too - so there is nothing to thread in here beyond the caller's tuning.
|
|
41
41
|
if (memory !== false)
|
|
42
|
-
this
|
|
42
|
+
this.#memorySignal = new MemoryLoadSignal(memory);
|
|
43
43
|
if (eventLoop !== false)
|
|
44
|
-
this
|
|
44
|
+
this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
|
|
45
45
|
if (cpu !== false)
|
|
46
|
-
this
|
|
46
|
+
this.#cpuSignal = new CpuLoadSignal(cpu);
|
|
47
47
|
if (client !== false)
|
|
48
|
-
this
|
|
48
|
+
this.#clientSignal = new ClientLoadSignal(client);
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
51
|
* Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
|
|
@@ -105,9 +105,7 @@ export interface FinalStatistics {
|
|
|
105
105
|
* @internal
|
|
106
106
|
*/
|
|
107
107
|
export declare class SystemStatus {
|
|
108
|
-
private
|
|
109
|
-
private readonly historyMillis;
|
|
110
|
-
private readonly signals;
|
|
108
|
+
#private;
|
|
111
109
|
constructor(options: SystemStatusOptions);
|
|
112
110
|
/**
|
|
113
111
|
* The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
|
|
@@ -32,14 +32,14 @@ const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
|
|
|
32
32
|
* @internal
|
|
33
33
|
*/
|
|
34
34
|
export class SystemStatus {
|
|
35
|
-
currentHistoryMillis;
|
|
36
|
-
historyMillis;
|
|
37
|
-
signals;
|
|
35
|
+
#currentHistoryMillis;
|
|
36
|
+
#historyMillis;
|
|
37
|
+
#signals;
|
|
38
38
|
constructor(options) {
|
|
39
39
|
const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
|
|
40
|
-
this
|
|
41
|
-
this
|
|
42
|
-
this
|
|
40
|
+
this.#currentHistoryMillis = currentHistorySecs * 1000;
|
|
41
|
+
this.#historyMillis = historySecs * 1000;
|
|
42
|
+
this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
|
|
43
43
|
this.assertUniqueSignalNames();
|
|
44
44
|
}
|
|
45
45
|
/**
|
|
@@ -48,7 +48,7 @@ export class SystemStatus {
|
|
|
48
48
|
* defaults.
|
|
49
49
|
*/
|
|
50
50
|
get maxSampleWindowMillis() {
|
|
51
|
-
return Math.max(this
|
|
51
|
+
return Math.max(this.#currentHistoryMillis, this.#historyMillis);
|
|
52
52
|
}
|
|
53
53
|
/**
|
|
54
54
|
* Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
|
|
@@ -57,7 +57,7 @@ export class SystemStatus {
|
|
|
57
57
|
*/
|
|
58
58
|
assertUniqueSignalNames() {
|
|
59
59
|
const seen = new Set();
|
|
60
|
-
for (const { name } of this
|
|
60
|
+
for (const { name } of this.#signals) {
|
|
61
61
|
if (!seen.has(name)) {
|
|
62
62
|
seen.add(name);
|
|
63
63
|
continue;
|
|
@@ -85,7 +85,7 @@ export class SystemStatus {
|
|
|
85
85
|
* and `true` otherwise.
|
|
86
86
|
*/
|
|
87
87
|
getCurrentStatus() {
|
|
88
|
-
return this.isSystemIdle(this
|
|
88
|
+
return this.isSystemIdle(this.#currentHistoryMillis);
|
|
89
89
|
}
|
|
90
90
|
/**
|
|
91
91
|
* Returns an {@link SystemInfo} object with the following structure:
|
|
@@ -103,7 +103,7 @@ export class SystemStatus {
|
|
|
103
103
|
* `historySecs` seconds and `true` otherwise.
|
|
104
104
|
*/
|
|
105
105
|
getHistoricalStatus() {
|
|
106
|
-
return this.isSystemIdle(this
|
|
106
|
+
return this.isSystemIdle(this.#historyMillis);
|
|
107
107
|
}
|
|
108
108
|
/**
|
|
109
109
|
* Returns a system status object.
|
|
@@ -117,7 +117,7 @@ export class SystemStatus {
|
|
|
117
117
|
clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
|
|
118
118
|
};
|
|
119
119
|
let loadSignalInfo;
|
|
120
|
-
for (const signal of this
|
|
120
|
+
for (const signal of this.#signals) {
|
|
121
121
|
const sample = signal.getSample(sampleDurationMillis);
|
|
122
122
|
const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
|
|
123
123
|
if (info.isOverloaded) {
|
package/configuration.d.ts
CHANGED
|
@@ -123,12 +123,12 @@ export interface Configuration extends ResolvedConfigValues {
|
|
|
123
123
|
* `containerized` | `CRAWLEE_CONTAINERIZED` | -
|
|
124
124
|
*/
|
|
125
125
|
export declare class Configuration {
|
|
126
|
+
#private;
|
|
126
127
|
/**
|
|
127
128
|
* Field definitions for this configuration class.
|
|
128
129
|
* Subclasses override this to register additional fields.
|
|
129
130
|
*/
|
|
130
131
|
protected static fields: Record<string, ConfigField>;
|
|
131
|
-
private resolvedValues;
|
|
132
132
|
/**
|
|
133
133
|
* Creates new `Configuration` instance with provided options.
|
|
134
134
|
* Constructor options take precedence over environment variables, which take precedence
|
package/configuration.js
CHANGED
|
@@ -147,7 +147,7 @@ export class Configuration {
|
|
|
147
147
|
* Subclasses override this to register additional fields.
|
|
148
148
|
*/
|
|
149
149
|
static fields = crawleeConfigFields;
|
|
150
|
-
resolvedValues;
|
|
150
|
+
#resolvedValues;
|
|
151
151
|
/**
|
|
152
152
|
* Creates new `Configuration` instance with provided options.
|
|
153
153
|
* Constructor options take precedence over environment variables, which take precedence
|
|
@@ -156,7 +156,7 @@ export class Configuration {
|
|
|
156
156
|
constructor(options = {}) {
|
|
157
157
|
const fields = this.constructor.fields;
|
|
158
158
|
const fileOptions = Configuration.loadFileOptions();
|
|
159
|
-
this
|
|
159
|
+
this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
|
|
160
160
|
this.registerAccessors();
|
|
161
161
|
// Set the log level
|
|
162
162
|
const logLevel = this.logLevel;
|
|
@@ -209,7 +209,7 @@ export class Configuration {
|
|
|
209
209
|
const descriptors = {};
|
|
210
210
|
for (const key of Object.keys(fields)) {
|
|
211
211
|
descriptors[key] = {
|
|
212
|
-
get: () => this
|
|
212
|
+
get: () => this.#resolvedValues[key],
|
|
213
213
|
set() {
|
|
214
214
|
throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
|
|
215
215
|
},
|
|
@@ -26,12 +26,12 @@ export class ContextPipeline {
|
|
|
26
26
|
* properties from the `ContextPipeline` interface, making type checking more reliable.
|
|
27
27
|
*/
|
|
28
28
|
class ContextPipelineImpl extends ContextPipeline {
|
|
29
|
-
middleware;
|
|
30
|
-
parent;
|
|
29
|
+
#middleware;
|
|
30
|
+
#parent;
|
|
31
31
|
constructor(middleware, parent) {
|
|
32
32
|
super();
|
|
33
|
-
this
|
|
34
|
-
this
|
|
33
|
+
this.#middleware = middleware;
|
|
34
|
+
this.#parent = parent;
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
37
37
|
* @inheritdoc
|
|
@@ -50,8 +50,8 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
50
50
|
*middlewareChain() {
|
|
51
51
|
let step = this;
|
|
52
52
|
while (step !== undefined) {
|
|
53
|
-
yield step
|
|
54
|
-
step = step
|
|
53
|
+
yield step.#middleware;
|
|
54
|
+
step = step.#parent;
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
/**
|
package/crawlers/statistics.d.ts
CHANGED
|
@@ -81,6 +81,7 @@ export interface CalculatedStatistics {
|
|
|
81
81
|
* @category Crawlers
|
|
82
82
|
*/
|
|
83
83
|
export declare class Statistics implements IStatistics {
|
|
84
|
+
#private;
|
|
84
85
|
private static id;
|
|
85
86
|
/**
|
|
86
87
|
* An error tracker for final retry errors.
|
|
@@ -104,15 +105,7 @@ export declare class Statistics implements IStatistics {
|
|
|
104
105
|
readonly requestRetryHistogram: number[];
|
|
105
106
|
protected keyValueStore?: KeyValueStore;
|
|
106
107
|
protected readonly persistStateKey: string;
|
|
107
|
-
private logIntervalMillis;
|
|
108
|
-
private logMessage;
|
|
109
|
-
private listener;
|
|
110
|
-
private requestsInProgress;
|
|
111
108
|
private readonly log;
|
|
112
|
-
private instanceStart;
|
|
113
|
-
private logInterval;
|
|
114
|
-
private _events?;
|
|
115
|
-
private persistenceOptions;
|
|
116
109
|
private get events();
|
|
117
110
|
/**
|
|
118
111
|
* Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
|
package/crawlers/statistics.js
CHANGED
|
@@ -6,14 +6,14 @@ import { ErrorTracker } from './error_tracker.js';
|
|
|
6
6
|
* @ignore
|
|
7
7
|
*/
|
|
8
8
|
class Job {
|
|
9
|
-
lastRunAt = null;
|
|
10
|
-
durationMillis;
|
|
9
|
+
#lastRunAt = null;
|
|
10
|
+
#durationMillis;
|
|
11
11
|
run() {
|
|
12
|
-
this
|
|
12
|
+
this.#lastRunAt = Date.now();
|
|
13
13
|
}
|
|
14
14
|
finish() {
|
|
15
|
-
this
|
|
16
|
-
return this
|
|
15
|
+
this.#durationMillis = Date.now() - this.#lastRunAt;
|
|
16
|
+
return this.#durationMillis;
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
const errorTrackerConfig = {
|
|
@@ -35,6 +35,7 @@ const errorTrackerConfig = {
|
|
|
35
35
|
* @category Crawlers
|
|
36
36
|
*/
|
|
37
37
|
export class Statistics {
|
|
38
|
+
// kept as TS-private: statistics tests read the static counter directly
|
|
38
39
|
static id = 0;
|
|
39
40
|
/**
|
|
40
41
|
* An error tracker for final retry errors.
|
|
@@ -58,20 +59,20 @@ export class Statistics {
|
|
|
58
59
|
requestRetryHistogram = [];
|
|
59
60
|
keyValueStore = undefined;
|
|
60
61
|
persistStateKey;
|
|
61
|
-
logIntervalMillis;
|
|
62
|
-
logMessage;
|
|
63
|
-
listener;
|
|
64
|
-
requestsInProgress = new Map();
|
|
62
|
+
#logIntervalMillis;
|
|
63
|
+
#logMessage;
|
|
64
|
+
#listener;
|
|
65
|
+
#requestsInProgress = new Map();
|
|
65
66
|
log;
|
|
66
|
-
instanceStart;
|
|
67
|
-
logInterval;
|
|
68
|
-
|
|
69
|
-
persistenceOptions;
|
|
67
|
+
#instanceStart;
|
|
68
|
+
#logInterval;
|
|
69
|
+
#events;
|
|
70
|
+
#persistenceOptions;
|
|
70
71
|
get events() {
|
|
71
|
-
if (!this
|
|
72
|
-
this
|
|
72
|
+
if (!this.#events) {
|
|
73
|
+
this.#events = serviceLocator.getEventManager();
|
|
73
74
|
}
|
|
74
|
-
return this
|
|
75
|
+
return this.#events;
|
|
75
76
|
}
|
|
76
77
|
/**
|
|
77
78
|
* Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
|
|
@@ -95,11 +96,11 @@ export class Statistics {
|
|
|
95
96
|
this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
|
|
96
97
|
this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
97
98
|
this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
98
|
-
this
|
|
99
|
-
this
|
|
99
|
+
this.#logIntervalMillis = logIntervalSecs * 1000;
|
|
100
|
+
this.#logMessage = logMessage;
|
|
100
101
|
this.keyValueStore = keyValueStore;
|
|
101
|
-
this
|
|
102
|
-
this
|
|
102
|
+
this.#listener = this.persistState.bind(this);
|
|
103
|
+
this.#persistenceOptions = persistenceOptions;
|
|
103
104
|
// initialize by "resetting"
|
|
104
105
|
this.reset();
|
|
105
106
|
}
|
|
@@ -128,15 +129,15 @@ export class Statistics {
|
|
|
128
129
|
retryErrors: this.errorTrackerRetry.result,
|
|
129
130
|
};
|
|
130
131
|
this.requestRetryHistogram.length = 0;
|
|
131
|
-
this
|
|
132
|
-
this
|
|
132
|
+
this.#requestsInProgress.clear();
|
|
133
|
+
this.#instanceStart = Date.now();
|
|
133
134
|
this.teardown();
|
|
134
135
|
}
|
|
135
136
|
/**
|
|
136
137
|
* @param options - Override the persistence options provided in the constructor
|
|
137
138
|
*/
|
|
138
139
|
async resetStore(options) {
|
|
139
|
-
if (!this
|
|
140
|
+
if (!this.#persistenceOptions.enable && !options?.enable) {
|
|
140
141
|
return;
|
|
141
142
|
}
|
|
142
143
|
if (!this.keyValueStore) {
|
|
@@ -159,18 +160,18 @@ export class Statistics {
|
|
|
159
160
|
* @ignore
|
|
160
161
|
*/
|
|
161
162
|
startJob(id) {
|
|
162
|
-
let job = this
|
|
163
|
+
let job = this.#requestsInProgress.get(id);
|
|
163
164
|
if (!job)
|
|
164
165
|
job = new Job();
|
|
165
166
|
job.run();
|
|
166
|
-
this
|
|
167
|
+
this.#requestsInProgress.set(id, job);
|
|
167
168
|
}
|
|
168
169
|
/**
|
|
169
170
|
* Mark job as finished and sets the state
|
|
170
171
|
* @ignore
|
|
171
172
|
*/
|
|
172
173
|
finishJob(id, retryCount) {
|
|
173
|
-
const job = this
|
|
174
|
+
const job = this.#requestsInProgress.get(id);
|
|
174
175
|
if (!job)
|
|
175
176
|
return;
|
|
176
177
|
const jobDurationMillis = job.finish();
|
|
@@ -181,20 +182,20 @@ export class Statistics {
|
|
|
181
182
|
this.state.requestMinDurationMillis = jobDurationMillis;
|
|
182
183
|
if (jobDurationMillis > this.state.requestMaxDurationMillis)
|
|
183
184
|
this.state.requestMaxDurationMillis = jobDurationMillis;
|
|
184
|
-
this
|
|
185
|
+
this.#requestsInProgress.delete(id);
|
|
185
186
|
}
|
|
186
187
|
/**
|
|
187
188
|
* Mark job as failed and sets the state
|
|
188
189
|
* @ignore
|
|
189
190
|
*/
|
|
190
191
|
failJob(id, retryCount) {
|
|
191
|
-
const job = this
|
|
192
|
+
const job = this.#requestsInProgress.get(id);
|
|
192
193
|
if (!job)
|
|
193
194
|
return;
|
|
194
195
|
this.state.requestTotalFailedDurationMillis += job.finish();
|
|
195
196
|
this.state.requestsFailed++;
|
|
196
197
|
this.saveRetryCountForJob(retryCount);
|
|
197
|
-
this
|
|
198
|
+
this.#requestsInProgress.delete(id);
|
|
198
199
|
}
|
|
199
200
|
/**
|
|
200
201
|
* Discards a started job without affecting the finished/failed counters, e.g. when a request
|
|
@@ -202,14 +203,14 @@ export class Statistics {
|
|
|
202
203
|
* @ignore
|
|
203
204
|
*/
|
|
204
205
|
discardJob(id) {
|
|
205
|
-
this
|
|
206
|
+
this.#requestsInProgress.delete(id);
|
|
206
207
|
}
|
|
207
208
|
/**
|
|
208
209
|
* Calculate the current statistics
|
|
209
210
|
*/
|
|
210
211
|
calculate() {
|
|
211
212
|
const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
|
|
212
|
-
const totalMillis = Date.now() - this
|
|
213
|
+
const totalMillis = Date.now() - this.#instanceStart;
|
|
213
214
|
const totalMinutes = totalMillis / 1000 / 60;
|
|
214
215
|
return {
|
|
215
216
|
requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
|
|
@@ -228,23 +229,23 @@ export class Statistics {
|
|
|
228
229
|
async startCapturing() {
|
|
229
230
|
// A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
|
|
230
231
|
// capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
|
|
231
|
-
if (this
|
|
232
|
+
if (this.#logInterval) {
|
|
232
233
|
throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
|
|
233
234
|
}
|
|
234
235
|
this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
235
236
|
if (this.state.crawlerStartedAt === null) {
|
|
236
237
|
this.state.crawlerStartedAt = new Date();
|
|
237
238
|
}
|
|
238
|
-
if (this
|
|
239
|
+
if (this.#persistenceOptions.enable) {
|
|
239
240
|
await this.maybeLoadStatistics();
|
|
240
|
-
this.events.on("persistState" /* EventType.PERSIST_STATE */, this
|
|
241
|
+
this.events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
|
|
241
242
|
}
|
|
242
|
-
this
|
|
243
|
-
this.log.info(this
|
|
243
|
+
this.#logInterval = setInterval(() => {
|
|
244
|
+
this.log.info(this.#logMessage, {
|
|
244
245
|
...this.calculate(),
|
|
245
246
|
retryHistogram: this.requestRetryHistogram,
|
|
246
247
|
});
|
|
247
|
-
}, this
|
|
248
|
+
}, this.#logIntervalMillis);
|
|
248
249
|
}
|
|
249
250
|
/**
|
|
250
251
|
* Stops logging and remove event listeners, then persist
|
|
@@ -265,7 +266,7 @@ export class Statistics {
|
|
|
265
266
|
* @param options - Override the persistence options provided in the constructor
|
|
266
267
|
*/
|
|
267
268
|
async persistState(options) {
|
|
268
|
-
if (!this
|
|
269
|
+
if (!this.#persistenceOptions.enable && !options?.enable) {
|
|
269
270
|
return;
|
|
270
271
|
}
|
|
271
272
|
// this might be called before startCapturing was called without using await, should not crash
|
|
@@ -312,17 +313,17 @@ export class Statistics {
|
|
|
312
313
|
this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
|
|
313
314
|
this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
|
|
314
315
|
this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
|
|
315
|
-
this
|
|
316
|
+
this.#instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
|
|
316
317
|
this.log.debug('Loaded from KeyValueStore');
|
|
317
318
|
}
|
|
318
319
|
teardown() {
|
|
319
320
|
// this can be called before a call to startCapturing happens (or in a 'finally' block)
|
|
320
321
|
// Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
|
|
321
322
|
// (e.g. during the constructor's reset() call, which would capture the wrong context)
|
|
322
|
-
this
|
|
323
|
-
if (this
|
|
324
|
-
clearInterval(this
|
|
325
|
-
this
|
|
323
|
+
this.#events?.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
|
|
324
|
+
if (this.#logInterval) {
|
|
325
|
+
clearInterval(this.#logInterval);
|
|
326
|
+
this.#logInterval = null;
|
|
326
327
|
}
|
|
327
328
|
}
|
|
328
329
|
/**
|
|
@@ -335,7 +336,7 @@ export class Statistics {
|
|
|
335
336
|
// omit duplicated information
|
|
336
337
|
const result = {
|
|
337
338
|
...this.state,
|
|
338
|
-
crawlerLastStartTimestamp: this
|
|
339
|
+
crawlerLastStartTimestamp: this.#instanceStart,
|
|
339
340
|
crawlerFinishedAt: this.state.crawlerFinishedAt
|
|
340
341
|
? new Date(this.state.crawlerFinishedAt).toISOString()
|
|
341
342
|
: null,
|
|
@@ -41,12 +41,12 @@ interface Intervals {
|
|
|
41
41
|
systemInfo?: BetterIntervalID;
|
|
42
42
|
}
|
|
43
43
|
export declare abstract class EventManager {
|
|
44
|
+
#private;
|
|
44
45
|
protected events: AsyncEventEmitter<{}>;
|
|
45
46
|
protected initialized: boolean;
|
|
46
47
|
protected intervals: Intervals;
|
|
47
48
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
48
49
|
protected log: import("@crawlee/types").CrawleeLogger;
|
|
49
|
-
private persistStateIntervalMillis;
|
|
50
50
|
constructor(options: EventManagerOptions);
|
|
51
51
|
/**
|
|
52
52
|
* Initializes the event manager by starting the `persistState` event interval.
|
package/events/event_manager.js
CHANGED
|
@@ -15,9 +15,9 @@ export class EventManager {
|
|
|
15
15
|
initialized = false;
|
|
16
16
|
intervals = {};
|
|
17
17
|
log = serviceLocator.getLogger().child({ prefix: 'Events' });
|
|
18
|
-
persistStateIntervalMillis;
|
|
18
|
+
#persistStateIntervalMillis;
|
|
19
19
|
constructor(options) {
|
|
20
|
-
this
|
|
20
|
+
this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
|
|
21
21
|
this.events.setMaxListeners(50);
|
|
22
22
|
}
|
|
23
23
|
/**
|
|
@@ -31,7 +31,7 @@ export class EventManager {
|
|
|
31
31
|
this.intervals.persistState = betterSetInterval((intervalCallback) => {
|
|
32
32
|
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
33
33
|
intervalCallback();
|
|
34
|
-
}, this
|
|
34
|
+
}, this.#persistStateIntervalMillis);
|
|
35
35
|
this.initialized = true;
|
|
36
36
|
}
|
|
37
37
|
/**
|
|
@@ -5,7 +5,7 @@ export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
|
5
5
|
systemInfoIntervalMillis: number;
|
|
6
6
|
}
|
|
7
7
|
export declare class LocalEventManager extends EventManager {
|
|
8
|
-
private
|
|
8
|
+
#private;
|
|
9
9
|
constructor(options: LocalEventManagerOptions);
|
|
10
10
|
/**
|
|
11
11
|
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|