@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.106

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/autoscaling/autoscaled_pool.d.ts +3 -21
  2. package/autoscaling/autoscaled_pool.js +85 -85
  3. package/autoscaling/client_load_signal.d.ts +1 -5
  4. package/autoscaling/client_load_signal.js +20 -20
  5. package/autoscaling/concurrency_system.d.ts +5 -20
  6. package/autoscaling/concurrency_system.js +81 -80
  7. package/autoscaling/cpu_load_signal.d.ts +1 -2
  8. package/autoscaling/cpu_load_signal.js +10 -10
  9. package/autoscaling/event_loop_load_signal.d.ts +1 -4
  10. package/autoscaling/event_loop_load_signal.js +18 -18
  11. package/autoscaling/load_signal.d.ts +1 -1
  12. package/autoscaling/load_signal.js +12 -11
  13. package/autoscaling/memory_load_signal.d.ts +3 -12
  14. package/autoscaling/memory_load_signal.js +40 -41
  15. package/autoscaling/snapshotter.d.ts +1 -4
  16. package/autoscaling/snapshotter.js +12 -12
  17. package/autoscaling/system_status.d.ts +1 -3
  18. package/autoscaling/system_status.js +11 -11
  19. package/configuration.d.ts +1 -1
  20. package/configuration.js +3 -3
  21. package/crawlers/context_pipeline.js +6 -6
  22. package/crawlers/statistics.d.ts +1 -8
  23. package/crawlers/statistics.js +45 -44
  24. package/events/event_manager.d.ts +1 -1
  25. package/events/event_manager.js +3 -3
  26. package/events/local_event_manager.d.ts +1 -1
  27. package/events/local_event_manager.js +3 -3
  28. package/log.js +5 -1
  29. package/memory-storage/memory-storage.d.ts +1 -5
  30. package/memory-storage/memory-storage.js +2 -2
  31. package/memory-storage/resource-clients/dataset.d.ts +1 -1
  32. package/memory-storage/resource-clients/dataset.js +6 -5
  33. package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
  34. package/memory-storage/resource-clients/key-value-store.js +13 -12
  35. package/memory-storage/resource-clients/request-queue.d.ts +4 -23
  36. package/memory-storage/resource-clients/request-queue.js +59 -58
  37. package/owned_or_injected.d.ts +1 -3
  38. package/owned_or_injected.js +17 -17
  39. package/package.json +5 -5
  40. package/proxy_configuration.d.ts +1 -3
  41. package/proxy_configuration.js +8 -8
  42. package/recoverable_state.d.ts +1 -10
  43. package/recoverable_state.js +41 -41
  44. package/request.d.ts +1 -2
  45. package/request.js +10 -13
  46. package/router.d.ts +1 -4
  47. package/router.js +23 -23
  48. package/serialization.js +8 -9
  49. package/service_locator.d.ts +1 -10
  50. package/service_locator.js +48 -48
  51. package/session_pool/session.d.ts +1 -12
  52. package/session_pool/session.js +50 -50
  53. package/session_pool/session_pool.d.ts +2 -11
  54. package/session_pool/session_pool.js +59 -58
  55. package/storages/dataset.d.ts +1 -1
  56. package/storages/dataset.js +5 -5
  57. package/storages/key_value_store.d.ts +1 -4
  58. package/storages/key_value_store.js +21 -20
  59. package/storages/request_dedup_cache.d.ts +1 -2
  60. package/storages/request_dedup_cache.js +9 -9
  61. package/storages/request_list.d.ts +2 -22
  62. package/storages/request_list.js +74 -73
  63. package/storages/request_manager_tandem.d.ts +1 -10
  64. package/storages/request_manager_tandem.js +27 -27
  65. package/storages/request_queue.d.ts +2 -18
  66. package/storages/request_queue.js +37 -35
  67. package/storages/sitemap_request_loader.d.ts +1 -44
  68. package/storages/sitemap_request_loader.js +87 -87
  69. package/storages/storage_instance_manager.d.ts +1 -2
  70. package/storages/storage_instance_manager.js +17 -17
  71. package/storages/storage_stats.d.ts +1 -1
  72. package/storages/storage_stats.js +4 -4
  73. package/storages/transaction.d.ts +1 -3
  74. package/storages/transaction.js +17 -17
  75. package/system-info/runtime.js +7 -7
@@ -15,92 +15,91 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
15
15
  export class MemoryLoadSignal {
16
16
  name = 'memInfo';
17
17
  overloadedRatio;
18
- store = new SnapshotStore();
19
- maxUsedRatio;
18
+ #store = new SnapshotStore();
19
+ #maxUsedRatio;
20
20
  /** All resolved in `start()`, before anything that reads them can fire. */
21
- config;
22
- log;
23
- maxMemoryBytes;
24
- events;
25
- maxMemoryRatio;
26
- lastLoggedCriticalMemoryOverloadAt = null;
21
+ #config;
22
+ #log;
23
+ #maxMemoryBytes;
24
+ #events;
25
+ #maxMemoryRatio;
26
+ #lastLoggedCriticalMemoryOverloadAt = null;
27
27
  constructor(options = {}) {
28
- this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
28
+ this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;
29
29
  this.overloadedRatio = options.overloadedRatio ?? 0.2;
30
30
  this.handle = this.handle.bind(this);
31
31
  }
32
32
  async start(context) {
33
- this.store.useSampleWindow(context.maxSampleWindowMillis);
33
+ this.#store.useSampleWindow(context.maxSampleWindowMillis);
34
34
  // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
35
- this.store.clear();
35
+ this.#store.clear();
36
36
  // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
37
37
  // between systems) must not capture whichever services happened to be registered at that moment.
38
- this.config = serviceLocator.getConfiguration();
39
- this.events = serviceLocator.getEventManager();
40
- this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
41
- const memoryMbytes = this.config.memoryMbytes ?? 0;
38
+ this.#config = serviceLocator.getConfiguration();
39
+ this.#events = serviceLocator.getEventManager();
40
+ this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
41
+ const memoryMbytes = this.#config.memoryMbytes ?? 0;
42
42
  if (memoryMbytes > 0) {
43
- this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
43
+ this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;
44
44
  }
45
45
  else {
46
- this.maxMemoryRatio = this.config.availableMemoryRatio;
47
- if (!this.maxMemoryRatio) {
46
+ this.#maxMemoryRatio = this.#config.availableMemoryRatio;
47
+ if (!this.#maxMemoryRatio) {
48
48
  throw new Error('availableMemoryRatio is not set in configuration.');
49
49
  }
50
50
  else {
51
- this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
51
+ this.#log.debug(`Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +
52
52
  'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
53
53
  }
54
54
  // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
55
- this.maxMemoryBytes = await this._getTotalMemoryBytes();
55
+ this.#maxMemoryBytes = await this.getTotalMemoryBytes();
56
56
  }
57
- this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
57
+ this.#events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
58
58
  }
59
59
  async stop() {
60
- this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
61
- this.events = undefined;
60
+ this.#events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
61
+ this.#events = undefined;
62
62
  }
63
63
  getSample(sampleDurationMillis) {
64
- return this.store.getSample(sampleDurationMillis);
64
+ return this.#store.getSample(sampleDurationMillis);
65
65
  }
66
66
  /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
67
67
  handle(systemInfo) {
68
68
  const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
69
69
  const { memCurrentBytes, memTotalBytes } = systemInfo;
70
- let maxMemoryBytes = this.maxMemoryBytes;
71
- if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
72
- maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
70
+ let maxMemoryBytes = this.#maxMemoryBytes;
71
+ if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {
72
+ maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);
73
73
  }
74
74
  const snapshot = {
75
75
  createdAt,
76
- isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
76
+ isOverloaded: memCurrentBytes / maxMemoryBytes > this.#maxUsedRatio,
77
77
  usedBytes: memCurrentBytes,
78
78
  };
79
- this.store.push(snapshot, createdAt);
80
- this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
79
+ this.#store.push(snapshot, createdAt);
80
+ this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
81
81
  }
82
- /** @internal */
83
- _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
84
- const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
82
+ memoryOverloadWarning(systemInfo, maxMemoryBytes) {
83
+ const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
85
84
  const { memCurrentBytes } = systemInfo;
86
85
  const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
87
- if (this.lastLoggedCriticalMemoryOverloadAt &&
88
- +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
86
+ if (this.#lastLoggedCriticalMemoryOverloadAt &&
87
+ +createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
89
88
  return;
90
- const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
91
- const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
89
+ const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;
90
+ const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;
92
91
  const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
93
92
  const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
94
93
  if (isCriticalOverload) {
95
94
  const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
96
95
  const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
97
- this.log.warning('Memory is critically overloaded. ' +
96
+ this.#log.warning('Memory is critically overloaded. ' +
98
97
  `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
99
- this.lastLoggedCriticalMemoryOverloadAt = createdAt;
98
+ this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
100
99
  }
101
100
  }
102
- async _getTotalMemoryBytes() {
103
- const containerized = this.config.containerized ?? (await isContainerized());
101
+ async getTotalMemoryBytes() {
102
+ const containerized = this.#config.containerized ?? (await isContainerized());
104
103
  return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
105
104
  }
106
105
  }
@@ -61,10 +61,7 @@ export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
61
61
  * @internal
62
62
  */
63
63
  export declare class Snapshotter {
64
- private readonly memorySignal?;
65
- private readonly eventLoopSignal?;
66
- private readonly cpuSignal?;
67
- private readonly clientSignal?;
64
+ #private;
68
65
  /**
69
66
  * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
70
67
  * instances. Signals switched off through the options are simply absent — the system status reports them as
@@ -13,10 +13,10 @@ import { MemoryLoadSignal } from './memory_load_signal.js';
13
13
  */
14
14
  export class Snapshotter {
15
15
  // Absent when switched off through the corresponding option (e.g. `client: false`).
16
- memorySignal;
17
- eventLoopSignal;
18
- cpuSignal;
19
- clientSignal;
16
+ #memorySignal;
17
+ #eventLoopSignal;
18
+ #cpuSignal;
19
+ #clientSignal;
20
20
  /**
21
21
  * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
22
22
  * instances. Signals switched off through the options are simply absent — the system status reports them as
@@ -24,10 +24,10 @@ export class Snapshotter {
24
24
  */
25
25
  getLoadSignals() {
26
26
  const builtin = [
27
- this.memorySignal,
28
- this.eventLoopSignal,
29
- this.cpuSignal,
30
- this.clientSignal,
27
+ this.#memorySignal,
28
+ this.#eventLoopSignal,
29
+ this.#cpuSignal,
30
+ this.#clientSignal,
31
31
  ];
32
32
  return builtin.filter((signal) => signal !== undefined);
33
33
  }
@@ -39,13 +39,13 @@ export class Snapshotter {
39
39
  // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
40
40
  // over then too - so there is nothing to thread in here beyond the caller's tuning.
41
41
  if (memory !== false)
42
- this.memorySignal = new MemoryLoadSignal(memory);
42
+ this.#memorySignal = new MemoryLoadSignal(memory);
43
43
  if (eventLoop !== false)
44
- this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
44
+ this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
45
45
  if (cpu !== false)
46
- this.cpuSignal = new CpuLoadSignal(cpu);
46
+ this.#cpuSignal = new CpuLoadSignal(cpu);
47
47
  if (client !== false)
48
- this.clientSignal = new ClientLoadSignal(client);
48
+ this.#clientSignal = new ClientLoadSignal(client);
49
49
  }
50
50
  /**
51
51
  * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
@@ -105,9 +105,7 @@ export interface FinalStatistics {
105
105
  * @internal
106
106
  */
107
107
  export declare class SystemStatus {
108
- private readonly currentHistoryMillis;
109
- private readonly historyMillis;
110
- private readonly signals;
108
+ #private;
111
109
  constructor(options: SystemStatusOptions);
112
110
  /**
113
111
  * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
@@ -32,14 +32,14 @@ const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
32
32
  * @internal
33
33
  */
34
34
  export class SystemStatus {
35
- currentHistoryMillis;
36
- historyMillis;
37
- signals;
35
+ #currentHistoryMillis;
36
+ #historyMillis;
37
+ #signals;
38
38
  constructor(options) {
39
39
  const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
40
- this.currentHistoryMillis = currentHistorySecs * 1000;
41
- this.historyMillis = historySecs * 1000;
42
- this.signals = [...snapshotter.getLoadSignals(), ...loadSignals];
40
+ this.#currentHistoryMillis = currentHistorySecs * 1000;
41
+ this.#historyMillis = historySecs * 1000;
42
+ this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
43
43
  this.assertUniqueSignalNames();
44
44
  }
45
45
  /**
@@ -48,7 +48,7 @@ export class SystemStatus {
48
48
  * defaults.
49
49
  */
50
50
  get maxSampleWindowMillis() {
51
- return Math.max(this.currentHistoryMillis, this.historyMillis);
51
+ return Math.max(this.#currentHistoryMillis, this.#historyMillis);
52
52
  }
53
53
  /**
54
54
  * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
@@ -57,7 +57,7 @@ export class SystemStatus {
57
57
  */
58
58
  assertUniqueSignalNames() {
59
59
  const seen = new Set();
60
- for (const { name } of this.signals) {
60
+ for (const { name } of this.#signals) {
61
61
  if (!seen.has(name)) {
62
62
  seen.add(name);
63
63
  continue;
@@ -85,7 +85,7 @@ export class SystemStatus {
85
85
  * and `true` otherwise.
86
86
  */
87
87
  getCurrentStatus() {
88
- return this.isSystemIdle(this.currentHistoryMillis);
88
+ return this.isSystemIdle(this.#currentHistoryMillis);
89
89
  }
90
90
  /**
91
91
  * Returns an {@link SystemInfo} object with the following structure:
@@ -103,7 +103,7 @@ export class SystemStatus {
103
103
  * `historySecs` seconds and `true` otherwise.
104
104
  */
105
105
  getHistoricalStatus() {
106
- return this.isSystemIdle(this.historyMillis);
106
+ return this.isSystemIdle(this.#historyMillis);
107
107
  }
108
108
  /**
109
109
  * Returns a system status object.
@@ -117,7 +117,7 @@ export class SystemStatus {
117
117
  clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
118
118
  };
119
119
  let loadSignalInfo;
120
- for (const signal of this.signals) {
120
+ for (const signal of this.#signals) {
121
121
  const sample = signal.getSample(sampleDurationMillis);
122
122
  const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
123
123
  if (info.isOverloaded) {
@@ -123,12 +123,12 @@ export interface Configuration extends ResolvedConfigValues {
123
123
  * `containerized` | `CRAWLEE_CONTAINERIZED` | -
124
124
  */
125
125
  export declare class Configuration {
126
+ #private;
126
127
  /**
127
128
  * Field definitions for this configuration class.
128
129
  * Subclasses override this to register additional fields.
129
130
  */
130
131
  protected static fields: Record<string, ConfigField>;
131
- private resolvedValues;
132
132
  /**
133
133
  * Creates new `Configuration` instance with provided options.
134
134
  * Constructor options take precedence over environment variables, which take precedence
package/configuration.js CHANGED
@@ -147,7 +147,7 @@ export class Configuration {
147
147
  * Subclasses override this to register additional fields.
148
148
  */
149
149
  static fields = crawleeConfigFields;
150
- resolvedValues;
150
+ #resolvedValues;
151
151
  /**
152
152
  * Creates new `Configuration` instance with provided options.
153
153
  * Constructor options take precedence over environment variables, which take precedence
@@ -156,7 +156,7 @@ export class Configuration {
156
156
  constructor(options = {}) {
157
157
  const fields = this.constructor.fields;
158
158
  const fileOptions = Configuration.loadFileOptions();
159
- this.resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
159
+ this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
160
160
  this.registerAccessors();
161
161
  // Set the log level
162
162
  const logLevel = this.logLevel;
@@ -209,7 +209,7 @@ export class Configuration {
209
209
  const descriptors = {};
210
210
  for (const key of Object.keys(fields)) {
211
211
  descriptors[key] = {
212
- get: () => this.resolvedValues[key],
212
+ get: () => this.#resolvedValues[key],
213
213
  set() {
214
214
  throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
215
215
  },
@@ -26,12 +26,12 @@ export class ContextPipeline {
26
26
  * properties from the `ContextPipeline` interface, making type checking more reliable.
27
27
  */
28
28
  class ContextPipelineImpl extends ContextPipeline {
29
- middleware;
30
- parent;
29
+ #middleware;
30
+ #parent;
31
31
  constructor(middleware, parent) {
32
32
  super();
33
- this.middleware = middleware;
34
- this.parent = parent;
33
+ this.#middleware = middleware;
34
+ this.#parent = parent;
35
35
  }
36
36
  /**
37
37
  * @inheritdoc
@@ -50,8 +50,8 @@ class ContextPipelineImpl extends ContextPipeline {
50
50
  *middlewareChain() {
51
51
  let step = this;
52
52
  while (step !== undefined) {
53
- yield step.middleware;
54
- step = step.parent;
53
+ yield step.#middleware;
54
+ step = step.#parent;
55
55
  }
56
56
  }
57
57
  /**
@@ -81,6 +81,7 @@ export interface CalculatedStatistics {
81
81
  * @category Crawlers
82
82
  */
83
83
  export declare class Statistics implements IStatistics {
84
+ #private;
84
85
  private static id;
85
86
  /**
86
87
  * An error tracker for final retry errors.
@@ -104,15 +105,7 @@ export declare class Statistics implements IStatistics {
104
105
  readonly requestRetryHistogram: number[];
105
106
  protected keyValueStore?: KeyValueStore;
106
107
  protected readonly persistStateKey: string;
107
- private logIntervalMillis;
108
- private logMessage;
109
- private listener;
110
- private requestsInProgress;
111
108
  private readonly log;
112
- private instanceStart;
113
- private logInterval;
114
- private _events?;
115
- private persistenceOptions;
116
109
  private get events();
117
110
  /**
118
111
  * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
@@ -6,14 +6,14 @@ import { ErrorTracker } from './error_tracker.js';
6
6
  * @ignore
7
7
  */
8
8
  class Job {
9
- lastRunAt = null;
10
- durationMillis;
9
+ #lastRunAt = null;
10
+ #durationMillis;
11
11
  run() {
12
- this.lastRunAt = Date.now();
12
+ this.#lastRunAt = Date.now();
13
13
  }
14
14
  finish() {
15
- this.durationMillis = Date.now() - this.lastRunAt;
16
- return this.durationMillis;
15
+ this.#durationMillis = Date.now() - this.#lastRunAt;
16
+ return this.#durationMillis;
17
17
  }
18
18
  }
19
19
  const errorTrackerConfig = {
@@ -35,6 +35,7 @@ const errorTrackerConfig = {
35
35
  * @category Crawlers
36
36
  */
37
37
  export class Statistics {
38
+ // kept as TS-private: statistics tests read the static counter directly
38
39
  static id = 0;
39
40
  /**
40
41
  * An error tracker for final retry errors.
@@ -58,20 +59,20 @@ export class Statistics {
58
59
  requestRetryHistogram = [];
59
60
  keyValueStore = undefined;
60
61
  persistStateKey;
61
- logIntervalMillis;
62
- logMessage;
63
- listener;
64
- requestsInProgress = new Map();
62
+ #logIntervalMillis;
63
+ #logMessage;
64
+ #listener;
65
+ #requestsInProgress = new Map();
65
66
  log;
66
- instanceStart;
67
- logInterval;
68
- _events;
69
- persistenceOptions;
67
+ #instanceStart;
68
+ #logInterval;
69
+ #events;
70
+ #persistenceOptions;
70
71
  get events() {
71
- if (!this._events) {
72
- this._events = serviceLocator.getEventManager();
72
+ if (!this.#events) {
73
+ this.#events = serviceLocator.getEventManager();
73
74
  }
74
- return this._events;
75
+ return this.#events;
75
76
  }
76
77
  /**
77
78
  * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
@@ -95,11 +96,11 @@ export class Statistics {
95
96
  this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
96
97
  this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
97
98
  this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
98
- this.logIntervalMillis = logIntervalSecs * 1000;
99
- this.logMessage = logMessage;
99
+ this.#logIntervalMillis = logIntervalSecs * 1000;
100
+ this.#logMessage = logMessage;
100
101
  this.keyValueStore = keyValueStore;
101
- this.listener = this.persistState.bind(this);
102
- this.persistenceOptions = persistenceOptions;
102
+ this.#listener = this.persistState.bind(this);
103
+ this.#persistenceOptions = persistenceOptions;
103
104
  // initialize by "resetting"
104
105
  this.reset();
105
106
  }
@@ -128,15 +129,15 @@ export class Statistics {
128
129
  retryErrors: this.errorTrackerRetry.result,
129
130
  };
130
131
  this.requestRetryHistogram.length = 0;
131
- this.requestsInProgress.clear();
132
- this.instanceStart = Date.now();
132
+ this.#requestsInProgress.clear();
133
+ this.#instanceStart = Date.now();
133
134
  this.teardown();
134
135
  }
135
136
  /**
136
137
  * @param options - Override the persistence options provided in the constructor
137
138
  */
138
139
  async resetStore(options) {
139
- if (!this.persistenceOptions.enable && !options?.enable) {
140
+ if (!this.#persistenceOptions.enable && !options?.enable) {
140
141
  return;
141
142
  }
142
143
  if (!this.keyValueStore) {
@@ -159,18 +160,18 @@ export class Statistics {
159
160
  * @ignore
160
161
  */
161
162
  startJob(id) {
162
- let job = this.requestsInProgress.get(id);
163
+ let job = this.#requestsInProgress.get(id);
163
164
  if (!job)
164
165
  job = new Job();
165
166
  job.run();
166
- this.requestsInProgress.set(id, job);
167
+ this.#requestsInProgress.set(id, job);
167
168
  }
168
169
  /**
169
170
  * Mark job as finished and sets the state
170
171
  * @ignore
171
172
  */
172
173
  finishJob(id, retryCount) {
173
- const job = this.requestsInProgress.get(id);
174
+ const job = this.#requestsInProgress.get(id);
174
175
  if (!job)
175
176
  return;
176
177
  const jobDurationMillis = job.finish();
@@ -181,20 +182,20 @@ export class Statistics {
181
182
  this.state.requestMinDurationMillis = jobDurationMillis;
182
183
  if (jobDurationMillis > this.state.requestMaxDurationMillis)
183
184
  this.state.requestMaxDurationMillis = jobDurationMillis;
184
- this.requestsInProgress.delete(id);
185
+ this.#requestsInProgress.delete(id);
185
186
  }
186
187
  /**
187
188
  * Mark job as failed and sets the state
188
189
  * @ignore
189
190
  */
190
191
  failJob(id, retryCount) {
191
- const job = this.requestsInProgress.get(id);
192
+ const job = this.#requestsInProgress.get(id);
192
193
  if (!job)
193
194
  return;
194
195
  this.state.requestTotalFailedDurationMillis += job.finish();
195
196
  this.state.requestsFailed++;
196
197
  this.saveRetryCountForJob(retryCount);
197
- this.requestsInProgress.delete(id);
198
+ this.#requestsInProgress.delete(id);
198
199
  }
199
200
  /**
200
201
  * Discards a started job without affecting the finished/failed counters, e.g. when a request
@@ -202,14 +203,14 @@ export class Statistics {
202
203
  * @ignore
203
204
  */
204
205
  discardJob(id) {
205
- this.requestsInProgress.delete(id);
206
+ this.#requestsInProgress.delete(id);
206
207
  }
207
208
  /**
208
209
  * Calculate the current statistics
209
210
  */
210
211
  calculate() {
211
212
  const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
212
- const totalMillis = Date.now() - this.instanceStart;
213
+ const totalMillis = Date.now() - this.#instanceStart;
213
214
  const totalMinutes = totalMillis / 1000 / 60;
214
215
  return {
215
216
  requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
@@ -228,23 +229,23 @@ export class Statistics {
228
229
  async startCapturing() {
229
230
  // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
230
231
  // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
231
- if (this.logInterval) {
232
+ if (this.#logInterval) {
232
233
  throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
233
234
  }
234
235
  this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
235
236
  if (this.state.crawlerStartedAt === null) {
236
237
  this.state.crawlerStartedAt = new Date();
237
238
  }
238
- if (this.persistenceOptions.enable) {
239
+ if (this.#persistenceOptions.enable) {
239
240
  await this.maybeLoadStatistics();
240
- this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
241
+ this.events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
241
242
  }
242
- this.logInterval = setInterval(() => {
243
- this.log.info(this.logMessage, {
243
+ this.#logInterval = setInterval(() => {
244
+ this.log.info(this.#logMessage, {
244
245
  ...this.calculate(),
245
246
  retryHistogram: this.requestRetryHistogram,
246
247
  });
247
- }, this.logIntervalMillis);
248
+ }, this.#logIntervalMillis);
248
249
  }
249
250
  /**
250
251
  * Stops logging and remove event listeners, then persist
@@ -265,7 +266,7 @@ export class Statistics {
265
266
  * @param options - Override the persistence options provided in the constructor
266
267
  */
267
268
  async persistState(options) {
268
- if (!this.persistenceOptions.enable && !options?.enable) {
269
+ if (!this.#persistenceOptions.enable && !options?.enable) {
269
270
  return;
270
271
  }
271
272
  // this might be called before startCapturing was called without using await, should not crash
@@ -312,17 +313,17 @@ export class Statistics {
312
313
  this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
313
314
  this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
314
315
  this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
315
- this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
316
+ this.#instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
316
317
  this.log.debug('Loaded from KeyValueStore');
317
318
  }
318
319
  teardown() {
319
320
  // this can be called before a call to startCapturing happens (or in a 'finally' block)
320
321
  // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
321
322
  // (e.g. during the constructor's reset() call, which would capture the wrong context)
322
- this._events?.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
323
- if (this.logInterval) {
324
- clearInterval(this.logInterval);
325
- this.logInterval = null;
323
+ this.#events?.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
324
+ if (this.#logInterval) {
325
+ clearInterval(this.#logInterval);
326
+ this.#logInterval = null;
326
327
  }
327
328
  }
328
329
  /**
@@ -335,7 +336,7 @@ export class Statistics {
335
336
  // omit duplicated information
336
337
  const result = {
337
338
  ...this.state,
338
- crawlerLastStartTimestamp: this.instanceStart,
339
+ crawlerLastStartTimestamp: this.#instanceStart,
339
340
  crawlerFinishedAt: this.state.crawlerFinishedAt
340
341
  ? new Date(this.state.crawlerFinishedAt).toISOString()
341
342
  : null,
@@ -41,12 +41,12 @@ interface Intervals {
41
41
  systemInfo?: BetterIntervalID;
42
42
  }
43
43
  export declare abstract class EventManager {
44
+ #private;
44
45
  protected events: AsyncEventEmitter<{}>;
45
46
  protected initialized: boolean;
46
47
  protected intervals: Intervals;
47
48
  // @ts-ignore optional peer dependency or compatibility with es2022
48
49
  protected log: import("@crawlee/types").CrawleeLogger;
49
- private persistStateIntervalMillis;
50
50
  constructor(options: EventManagerOptions);
51
51
  /**
52
52
  * Initializes the event manager by starting the `persistState` event interval.
@@ -15,9 +15,9 @@ export class EventManager {
15
15
  initialized = false;
16
16
  intervals = {};
17
17
  log = serviceLocator.getLogger().child({ prefix: 'Events' });
18
- persistStateIntervalMillis;
18
+ #persistStateIntervalMillis;
19
19
  constructor(options) {
20
- this.persistStateIntervalMillis = options.persistStateIntervalMillis;
20
+ this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
21
21
  this.events.setMaxListeners(50);
22
22
  }
23
23
  /**
@@ -31,7 +31,7 @@ export class EventManager {
31
31
  this.intervals.persistState = betterSetInterval((intervalCallback) => {
32
32
  this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
33
33
  intervalCallback();
34
- }, this.persistStateIntervalMillis);
34
+ }, this.#persistStateIntervalMillis);
35
35
  this.initialized = true;
36
36
  }
37
37
  /**
@@ -5,7 +5,7 @@ export interface LocalEventManagerOptions extends EventManagerOptions {
5
5
  systemInfoIntervalMillis: number;
6
6
  }
7
7
  export declare class LocalEventManager extends EventManager {
8
- private systemInfoIntervalMillis;
8
+ #private;
9
9
  constructor(options: LocalEventManagerOptions);
10
10
  /**
11
11
  * Creates a new `LocalEventManager` based on the provided `Configuration`.