@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (271) hide show
  1. package/README.md +17 -13
  2. package/autoscaling/autoscaled_pool.d.ts +67 -172
  3. package/autoscaling/autoscaled_pool.js +165 -320
  4. package/autoscaling/client_load_signal.d.ts +55 -0
  5. package/autoscaling/client_load_signal.js +73 -0
  6. package/autoscaling/concurrency_system.d.ts +268 -0
  7. package/autoscaling/concurrency_system.js +351 -0
  8. package/autoscaling/cpu_load_signal.d.ts +43 -0
  9. package/autoscaling/cpu_load_signal.js +46 -0
  10. package/autoscaling/event_loop_load_signal.d.ts +51 -0
  11. package/autoscaling/event_loop_load_signal.js +60 -0
  12. package/autoscaling/index.d.ts +6 -1
  13. package/autoscaling/index.js +6 -1
  14. package/autoscaling/load_signal.d.ts +99 -0
  15. package/autoscaling/load_signal.js +104 -0
  16. package/autoscaling/memory_load_signal.d.ts +47 -0
  17. package/autoscaling/memory_load_signal.js +105 -0
  18. package/autoscaling/snapshotter.d.ts +58 -163
  19. package/autoscaling/snapshotter.js +45 -263
  20. package/autoscaling/system_status.d.ts +62 -84
  21. package/autoscaling/system_status.js +92 -122
  22. package/autoscaling/weighted_avg.d.ts +5 -0
  23. package/autoscaling/weighted_avg.js +14 -0
  24. package/byte_utils.d.ts +17 -0
  25. package/byte_utils.js +42 -0
  26. package/configuration.d.ts +96 -223
  27. package/configuration.js +170 -222
  28. package/cookie_utils.d.ts +4 -3
  29. package/cookie_utils.js +22 -13
  30. package/crawlers/context_pipeline.d.ts +10 -1
  31. package/crawlers/context_pipeline.js +31 -8
  32. package/crawlers/crawler_commons.d.ts +90 -83
  33. package/crawlers/crawler_commons.js +1 -116
  34. package/crawlers/error_snapshotter.d.ts +1 -4
  35. package/crawlers/error_snapshotter.js +3 -6
  36. package/crawlers/error_tracker.d.ts +0 -1
  37. package/crawlers/error_tracker.js +0 -1
  38. package/crawlers/index.d.ts +1 -3
  39. package/crawlers/index.js +0 -3
  40. package/crawlers/internals/types.d.ts +0 -1
  41. package/crawlers/internals/types.js +0 -1
  42. package/crawlers/statistics.d.ts +91 -41
  43. package/crawlers/statistics.js +83 -72
  44. package/debug.d.ts +36 -0
  45. package/debug.js +70 -0
  46. package/enqueue_links/enqueue_links.d.ts +59 -68
  47. package/enqueue_links/enqueue_links.js +57 -62
  48. package/enqueue_links/index.d.ts +0 -1
  49. package/enqueue_links/index.js +0 -1
  50. package/enqueue_links/shared.d.ts +40 -27
  51. package/enqueue_links/shared.js +90 -68
  52. package/errors.d.ts +41 -4
  53. package/errors.js +50 -5
  54. package/events/event_manager.d.ts +34 -8
  55. package/events/event_manager.js +8 -10
  56. package/events/index.d.ts +0 -1
  57. package/events/index.js +0 -1
  58. package/events/local_event_manager.d.ts +15 -3
  59. package/events/local_event_manager.js +37 -11
  60. package/index.d.ts +6 -4
  61. package/index.js +5 -3
  62. package/iterables.d.ts +79 -0
  63. package/iterables.js +134 -0
  64. package/log.d.ts +82 -3
  65. package/log.js +106 -1
  66. package/memory-storage/consts.d.ts +4 -0
  67. package/memory-storage/consts.js +4 -0
  68. package/memory-storage/index.d.ts +1 -0
  69. package/memory-storage/index.js +1 -0
  70. package/memory-storage/memory-storage.d.ts +42 -0
  71. package/memory-storage/memory-storage.js +136 -0
  72. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  73. package/memory-storage/resource-clients/common/base-client.js +6 -0
  74. package/memory-storage/resource-clients/dataset.d.ts +40 -0
  75. package/memory-storage/resource-clients/dataset.js +114 -0
  76. package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
  77. package/memory-storage/resource-clients/key-value-store.js +204 -0
  78. package/memory-storage/resource-clients/request-queue.d.ts +77 -0
  79. package/memory-storage/resource-clients/request-queue.js +422 -0
  80. package/memory-storage/utils.d.ts +16 -0
  81. package/memory-storage/utils.js +41 -0
  82. package/owned_or_injected.d.ts +58 -0
  83. package/owned_or_injected.js +98 -0
  84. package/package.json +13 -12
  85. package/proxy_configuration.d.ts +27 -152
  86. package/proxy_configuration.js +27 -179
  87. package/recoverable_state.d.ts +111 -0
  88. package/recoverable_state.js +143 -0
  89. package/request.d.ts +86 -17
  90. package/request.js +117 -41
  91. package/router.d.ts +193 -21
  92. package/router.js +188 -43
  93. package/serialization.d.ts +0 -1
  94. package/serialization.js +9 -11
  95. package/service_locator.d.ts +147 -0
  96. package/service_locator.js +244 -0
  97. package/session_pool/consts.d.ts +1 -2
  98. package/session_pool/consts.js +1 -2
  99. package/session_pool/errors.d.ts +0 -1
  100. package/session_pool/errors.js +0 -1
  101. package/session_pool/fingerprint.d.ts +9 -0
  102. package/session_pool/fingerprint.js +30 -0
  103. package/session_pool/index.d.ts +0 -2
  104. package/session_pool/index.js +0 -2
  105. package/session_pool/session.d.ts +37 -86
  106. package/session_pool/session.js +85 -138
  107. package/session_pool/session_pool.d.ts +77 -91
  108. package/session_pool/session_pool.js +159 -147
  109. package/storages/dataset.d.ts +114 -54
  110. package/storages/dataset.js +285 -144
  111. package/storages/index.d.ts +9 -8
  112. package/storages/index.js +7 -8
  113. package/storages/key_value_store.d.ts +185 -42
  114. package/storages/key_value_store.js +424 -151
  115. package/storages/key_value_store_codec.d.ts +32 -0
  116. package/storages/key_value_store_codec.js +113 -0
  117. package/storages/request_dedup_cache.d.ts +22 -0
  118. package/storages/request_dedup_cache.js +48 -0
  119. package/storages/request_list.d.ts +52 -116
  120. package/storages/request_list.js +158 -133
  121. package/storages/request_loader.d.ts +96 -0
  122. package/storages/request_loader.js +1 -0
  123. package/storages/request_manager.d.ts +33 -0
  124. package/storages/request_manager.js +1 -0
  125. package/storages/request_manager_tandem.d.ts +97 -0
  126. package/storages/request_manager_tandem.js +197 -0
  127. package/storages/request_queue.d.ts +290 -47
  128. package/storages/request_queue.js +825 -216
  129. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +37 -88
  130. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +136 -143
  131. package/storages/storage_instance_manager.d.ts +87 -0
  132. package/storages/storage_instance_manager.js +256 -0
  133. package/storages/storage_stats.d.ts +48 -0
  134. package/storages/storage_stats.js +29 -0
  135. package/storages/transaction.d.ts +252 -0
  136. package/storages/transaction.js +251 -0
  137. package/storages/utils.d.ts +54 -9
  138. package/storages/utils.js +64 -13
  139. package/system-info/cpu-info.d.ts +67 -0
  140. package/system-info/cpu-info.js +216 -0
  141. package/system-info/memory-info.d.ts +31 -0
  142. package/system-info/memory-info.js +115 -0
  143. package/system-info/ps-tree.d.ts +17 -0
  144. package/system-info/ps-tree.js +144 -0
  145. package/system-info/runtime.d.ts +14 -0
  146. package/system-info/runtime.js +80 -0
  147. package/typedefs.d.ts +0 -6
  148. package/typedefs.js +0 -1
  149. package/validators.d.ts +8 -1
  150. package/validators.js +10 -3
  151. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  152. package/autoscaling/autoscaled_pool.js.map +0 -1
  153. package/autoscaling/index.d.ts.map +0 -1
  154. package/autoscaling/index.js.map +0 -1
  155. package/autoscaling/snapshotter.d.ts.map +0 -1
  156. package/autoscaling/snapshotter.js.map +0 -1
  157. package/autoscaling/system_status.d.ts.map +0 -1
  158. package/autoscaling/system_status.js.map +0 -1
  159. package/configuration.d.ts.map +0 -1
  160. package/configuration.js.map +0 -1
  161. package/cookie_utils.d.ts.map +0 -1
  162. package/cookie_utils.js.map +0 -1
  163. package/crawlers/context_pipeline.d.ts.map +0 -1
  164. package/crawlers/context_pipeline.js.map +0 -1
  165. package/crawlers/crawler_commons.d.ts.map +0 -1
  166. package/crawlers/crawler_commons.js.map +0 -1
  167. package/crawlers/crawler_utils.d.ts +0 -10
  168. package/crawlers/crawler_utils.d.ts.map +0 -1
  169. package/crawlers/crawler_utils.js +0 -12
  170. package/crawlers/crawler_utils.js.map +0 -1
  171. package/crawlers/error_snapshotter.d.ts.map +0 -1
  172. package/crawlers/error_snapshotter.js.map +0 -1
  173. package/crawlers/error_tracker.d.ts.map +0 -1
  174. package/crawlers/error_tracker.js.map +0 -1
  175. package/crawlers/index.d.ts.map +0 -1
  176. package/crawlers/index.js.map +0 -1
  177. package/crawlers/internals/types.d.ts.map +0 -1
  178. package/crawlers/internals/types.js.map +0 -1
  179. package/crawlers/statistics.d.ts.map +0 -1
  180. package/crawlers/statistics.js.map +0 -1
  181. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  182. package/enqueue_links/enqueue_links.js.map +0 -1
  183. package/enqueue_links/index.d.ts.map +0 -1
  184. package/enqueue_links/index.js.map +0 -1
  185. package/enqueue_links/shared.d.ts.map +0 -1
  186. package/enqueue_links/shared.js.map +0 -1
  187. package/errors.d.ts.map +0 -1
  188. package/errors.js.map +0 -1
  189. package/events/event_manager.d.ts.map +0 -1
  190. package/events/event_manager.js.map +0 -1
  191. package/events/index.d.ts.map +0 -1
  192. package/events/index.js.map +0 -1
  193. package/events/local_event_manager.d.ts.map +0 -1
  194. package/events/local_event_manager.js.map +0 -1
  195. package/http_clients/base-http-client.d.ts +0 -134
  196. package/http_clients/base-http-client.d.ts.map +0 -1
  197. package/http_clients/base-http-client.js +0 -33
  198. package/http_clients/base-http-client.js.map +0 -1
  199. package/http_clients/form-data-like.d.ts +0 -67
  200. package/http_clients/form-data-like.d.ts.map +0 -1
  201. package/http_clients/form-data-like.js +0 -5
  202. package/http_clients/form-data-like.js.map +0 -1
  203. package/http_clients/got-scraping-http-client.d.ts +0 -15
  204. package/http_clients/got-scraping-http-client.d.ts.map +0 -1
  205. package/http_clients/got-scraping-http-client.js +0 -69
  206. package/http_clients/got-scraping-http-client.js.map +0 -1
  207. package/http_clients/index.d.ts +0 -3
  208. package/http_clients/index.d.ts.map +0 -1
  209. package/http_clients/index.js +0 -3
  210. package/http_clients/index.js.map +0 -1
  211. package/index.d.ts.map +0 -1
  212. package/index.js.map +0 -1
  213. package/log.d.ts.map +0 -1
  214. package/log.js.map +0 -1
  215. package/proxy_configuration.d.ts.map +0 -1
  216. package/proxy_configuration.js.map +0 -1
  217. package/request.d.ts.map +0 -1
  218. package/request.js.map +0 -1
  219. package/router.d.ts.map +0 -1
  220. package/router.js.map +0 -1
  221. package/serialization.d.ts.map +0 -1
  222. package/serialization.js.map +0 -1
  223. package/session_pool/consts.d.ts.map +0 -1
  224. package/session_pool/consts.js.map +0 -1
  225. package/session_pool/errors.d.ts.map +0 -1
  226. package/session_pool/errors.js.map +0 -1
  227. package/session_pool/events.d.ts +0 -3
  228. package/session_pool/events.d.ts.map +0 -1
  229. package/session_pool/events.js +0 -3
  230. package/session_pool/events.js.map +0 -1
  231. package/session_pool/index.d.ts.map +0 -1
  232. package/session_pool/index.js.map +0 -1
  233. package/session_pool/session.d.ts.map +0 -1
  234. package/session_pool/session.js.map +0 -1
  235. package/session_pool/session_pool.d.ts.map +0 -1
  236. package/session_pool/session_pool.js.map +0 -1
  237. package/storages/access_checking.d.ts +0 -13
  238. package/storages/access_checking.d.ts.map +0 -1
  239. package/storages/access_checking.js +0 -14
  240. package/storages/access_checking.js.map +0 -1
  241. package/storages/dataset.d.ts.map +0 -1
  242. package/storages/dataset.js.map +0 -1
  243. package/storages/index.d.ts.map +0 -1
  244. package/storages/index.js.map +0 -1
  245. package/storages/key_value_store.d.ts.map +0 -1
  246. package/storages/key_value_store.js.map +0 -1
  247. package/storages/request_list.d.ts.map +0 -1
  248. package/storages/request_list.js.map +0 -1
  249. package/storages/request_provider.d.ts +0 -308
  250. package/storages/request_provider.d.ts.map +0 -1
  251. package/storages/request_provider.js +0 -555
  252. package/storages/request_provider.js.map +0 -1
  253. package/storages/request_queue.d.ts.map +0 -1
  254. package/storages/request_queue.js.map +0 -1
  255. package/storages/request_queue_v2.d.ts +0 -87
  256. package/storages/request_queue_v2.d.ts.map +0 -1
  257. package/storages/request_queue_v2.js +0 -438
  258. package/storages/request_queue_v2.js.map +0 -1
  259. package/storages/sitemap_request_list.d.ts.map +0 -1
  260. package/storages/sitemap_request_list.js.map +0 -1
  261. package/storages/storage_manager.d.ts +0 -58
  262. package/storages/storage_manager.d.ts.map +0 -1
  263. package/storages/storage_manager.js +0 -105
  264. package/storages/storage_manager.js.map +0 -1
  265. package/storages/utils.d.ts.map +0 -1
  266. package/storages/utils.js.map +0 -1
  267. package/tsconfig.build.tsbuildinfo +0 -1
  268. package/typedefs.d.ts.map +0 -1
  269. package/typedefs.js.map +0 -1
  270. package/validators.d.ts.map +0 -1
  271. package/validators.js.map +0 -1
@@ -1,5 +1,4 @@
1
- import type { Log } from '@apify/log';
2
- import { Configuration } from '../configuration.js';
1
+ import type { CrawleeLogger } from '../log.js';
3
2
  import { KeyValueStore } from '../storages/key_value_store.js';
4
3
  import { ErrorTracker } from './error_tracker.js';
5
4
  /**
@@ -12,30 +11,90 @@ export interface PersistenceOptions {
12
11
  */
13
12
  enable?: boolean;
14
13
  }
14
+ /**
15
+ * The statistics surface a crawler depends on: recording per-request outcomes, tracking errors, and driving the
16
+ * capture lifecycle for a run. Injected via the crawler's `statistics` option, so a custom implementation (or a
17
+ * {@link Statistics} subclass tracking extra fields) can be plugged in without subclassing the crawler.
18
+ *
19
+ * The owned-only mutators the crawler uses to *own* a default it built - `reset()`/`resetStore()` - are deliberately
20
+ * absent: an injected instance is borrowed, and the crawler never wipes it. Those live on the concrete
21
+ * {@link Statistics} only.
22
+ *
23
+ * @category Crawlers
24
+ */
25
+ export interface IStatistics {
26
+ /** Tracker for errors on the final retry of a request. */
27
+ readonly errorTracker: ErrorTracker;
28
+ /** Tracker for errors on retries prior to the final one. */
29
+ readonly errorTrackerRetry: ErrorTracker;
30
+ /** The live statistics state the crawler reads for status messages and the final summary. */
31
+ readonly state: StatisticState;
32
+ /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
33
+ readonly requestRetryHistogram: number[];
34
+ /** Marks a request as started, so its duration can be measured on finish/fail. */
35
+ startJob(id: number | string): void;
36
+ /** Marks a started request as finished, updating the finished counters and durations. */
37
+ finishJob(id: number | string, retryCount: number): void;
38
+ /** Marks a started request as failed, updating the failed counters and durations. */
39
+ failJob(id: number | string, retryCount: number): void;
40
+ /** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
41
+ discardJob(id: number | string): void;
42
+ /** Increments the counter for the given HTTP status code. */
43
+ registerStatusCode(code: number): void;
44
+ /** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
45
+ calculate(): CalculatedStatistics;
46
+ /** Begins a capture window: loads any persisted state, subscribes to persistence events, starts periodic logging. */
47
+ startCapturing(): Promise<void>;
48
+ /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
49
+ stopCapturing(): Promise<void>;
50
+ /**
51
+ * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
52
+ * with no persistence of its own can omit it.
53
+ */
54
+ persistState?(options?: PersistenceOptions): Promise<void>;
55
+ }
56
+ /** The derived aggregates computed by {@link IStatistics.calculate} from the current {@link StatisticState}. */
57
+ export interface CalculatedStatistics {
58
+ /** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
59
+ requestAvgFailedDurationMillis: number;
60
+ /** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
61
+ requestAvgFinishedDurationMillis: number;
62
+ /** Requests finished per minute over the run so far. */
63
+ requestsFinishedPerMinute: number;
64
+ /** Requests failed per minute over the run so far. */
65
+ requestsFailedPerMinute: number;
66
+ /** Combined duration of all finished and failed requests, in milliseconds. */
67
+ requestTotalDurationMillis: number;
68
+ /** Total number of settled requests (finished plus failed). */
69
+ requestsTotal: number;
70
+ /** Wall-clock runtime since capturing started, in milliseconds. */
71
+ crawlerRuntimeMillis: number;
72
+ }
15
73
  /**
16
74
  * The statistics class provides an interface to collecting and logging run
17
75
  * statistics for requests.
18
76
  *
19
77
  * All statistic information is saved on key value store
20
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
78
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
21
79
  * migrations and abort/resurrect
22
80
  *
23
81
  * @category Crawlers
24
82
  */
25
- export declare class Statistics {
83
+ export declare class Statistics implements IStatistics {
84
+ #private;
26
85
  private static id;
27
86
  /**
28
87
  * An error tracker for final retry errors.
29
88
  */
30
- errorTracker: ErrorTracker;
89
+ readonly errorTracker: ErrorTracker;
31
90
  /**
32
91
  * An error tracker for retry errors prior to the final retry.
33
92
  */
34
- errorTrackerRetry: ErrorTracker;
93
+ readonly errorTrackerRetry: ErrorTracker;
35
94
  /**
36
95
  * Statistic instance id.
37
96
  */
38
- readonly id: number;
97
+ readonly id: string;
39
98
  /**
40
99
  * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
41
100
  */
@@ -44,23 +103,13 @@ export declare class Statistics {
44
103
  * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
45
104
  */
46
105
  readonly requestRetryHistogram: number[];
47
- /**
48
- * Contains the associated Configuration instance
49
- */
50
- private readonly config;
51
106
  protected keyValueStore?: KeyValueStore;
52
- protected persistStateKey: string;
53
- private logIntervalMillis;
54
- private logMessage;
55
- private listener;
56
- private requestsInProgress;
107
+ protected readonly persistStateKey: string;
57
108
  private readonly log;
58
- private instanceStart;
59
- private logInterval;
60
- private events;
61
- private persistenceOptions;
109
+ private get events();
62
110
  /**
63
- * @internal
111
+ * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
112
+ * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
64
113
  */
65
114
  constructor(options?: StatisticsOptions);
66
115
  /**
@@ -90,18 +139,16 @@ export declare class Statistics {
90
139
  * @ignore
91
140
  */
92
141
  failJob(id: number | string, retryCount: number): void;
142
+ /**
143
+ * Discards a started job without affecting the finished/failed counters, e.g. when a request
144
+ * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
145
+ * @ignore
146
+ */
147
+ discardJob(id: number | string): void;
93
148
  /**
94
149
  * Calculate the current statistics
95
150
  */
96
- calculate(): {
97
- requestAvgFailedDurationMillis: number;
98
- requestAvgFinishedDurationMillis: number;
99
- requestsFinishedPerMinute: number;
100
- requestsFailedPerMinute: number;
101
- requestTotalDurationMillis: number;
102
- requestsTotal: number;
103
- crawlerRuntimeMillis: number;
104
- };
151
+ calculate(): CalculatedStatistics;
105
152
  /**
106
153
  * Initializes the key value store for persisting the statistics,
107
154
  * displaying the current state in predefined intervals
@@ -111,7 +158,7 @@ export declare class Statistics {
111
158
  * Stops logging and remove event listeners, then persist
112
159
  */
113
160
  stopCapturing(): Promise<void>;
114
- protected _saveRetryCountForJob(retryCount: number): void;
161
+ private saveRetryCountForJob;
115
162
  /**
116
163
  * Persist internal state to the key value store
117
164
  * @param options - Override the persistence options provided in the constructor
@@ -120,8 +167,8 @@ export declare class Statistics {
120
167
  /**
121
168
  * Loads the current statistic from the key value store if any
122
169
  */
123
- protected _maybeLoadStatistics(): Promise<void>;
124
- protected _teardown(): void;
170
+ protected maybeLoadStatistics(): Promise<void>;
171
+ private teardown;
125
172
  /**
126
173
  * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
127
174
  * or through `keyValueStore.setValue('KEY', statsInstance)`
@@ -146,17 +193,12 @@ export interface StatisticsOptions {
146
193
  * Parent logger instance, the statistics will create a child logger from this.
147
194
  * @default crawler.log
148
195
  */
149
- log?: Log;
196
+ log?: CrawleeLogger;
150
197
  /**
151
198
  * Key value store instance to persist the statistics.
152
199
  * If not provided, the default one will be used when capturing starts
153
200
  */
154
201
  keyValueStore?: KeyValueStore;
155
- /**
156
- * Configuration instance to use
157
- * @default Configuration.getGlobalConfig()
158
- */
159
- config?: Configuration;
160
202
  /**
161
203
  * Control how and when to persist the statistics.
162
204
  */
@@ -166,13 +208,22 @@ export interface StatisticsOptions {
166
208
  * @default false
167
209
  */
168
210
  saveErrorSnapshots?: boolean;
211
+ /**
212
+ * A unique identifier for this statistics instance. This ID is used for persistence
213
+ * to the key value store, ensuring the same statistics can be loaded after script restarts.
214
+ *
215
+ * If not provided, an auto-incremented ID will be used for backward compatibility.
216
+ * This means statistics may not persist correctly across script restarts
217
+ * if crawler creation order changes.
218
+ */
219
+ id?: string;
169
220
  }
170
221
  /**
171
222
  * Format of the persisted stats
172
223
  */
173
224
  export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
174
225
  requestRetryHistogram: number[];
175
- statsId: number;
226
+ statsId: string;
176
227
  requestAvgFailedDurationMillis: number;
177
228
  requestAvgFinishedDurationMillis: number;
178
229
  requestTotalDurationMillis: number;
@@ -201,4 +252,3 @@ export interface StatisticState {
201
252
  retryErrors: Record<string, unknown>;
202
253
  requestsWithStatusCode: Record<string, number>;
203
254
  }
204
- //# sourceMappingURL=statistics.d.ts.map
@@ -1,20 +1,19 @@
1
1
  import ow from 'ow';
2
- import { Configuration } from '../configuration.js';
3
- import { log as defaultLog } from '../log.js';
2
+ import { serviceLocator } from '../service_locator.js';
4
3
  import { KeyValueStore } from '../storages/key_value_store.js';
5
4
  import { ErrorTracker } from './error_tracker.js';
6
5
  /**
7
6
  * @ignore
8
7
  */
9
8
  class Job {
10
- lastRunAt = null;
11
- durationMillis;
9
+ #lastRunAt = null;
10
+ #durationMillis;
12
11
  run() {
13
- this.lastRunAt = Date.now();
12
+ this.#lastRunAt = Date.now();
14
13
  }
15
14
  finish() {
16
- this.durationMillis = Date.now() - this.lastRunAt;
17
- return this.durationMillis;
15
+ this.#durationMillis = Date.now() - this.#lastRunAt;
16
+ return this.#durationMillis;
18
17
  }
19
18
  }
20
19
  const errorTrackerConfig = {
@@ -30,12 +29,13 @@ const errorTrackerConfig = {
30
29
  * statistics for requests.
31
30
  *
32
31
  * All statistic information is saved on key value store
33
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
32
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
34
33
  * migrations and abort/resurrect
35
34
  *
36
35
  * @category Crawlers
37
36
  */
38
37
  export class Statistics {
38
+ // kept as TS-private: statistics tests read the static counter directly
39
39
  static id = 0;
40
40
  /**
41
41
  * An error tracker for final retry errors.
@@ -48,7 +48,7 @@ export class Statistics {
48
48
  /**
49
49
  * Statistic instance id.
50
50
  */
51
- id = Statistics.id++; // assign an id while incrementing so it can be saved/restored from KV
51
+ id;
52
52
  /**
53
53
  * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
54
54
  */
@@ -57,23 +57,26 @@ export class Statistics {
57
57
  * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
58
58
  */
59
59
  requestRetryHistogram = [];
60
- /**
61
- * Contains the associated Configuration instance
62
- */
63
- config;
64
60
  keyValueStore = undefined;
65
- persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
66
- logIntervalMillis;
67
- logMessage;
68
- listener;
69
- requestsInProgress = new Map();
61
+ persistStateKey;
62
+ #logIntervalMillis;
63
+ #logMessage;
64
+ #listener;
65
+ #requestsInProgress = new Map();
70
66
  log;
71
- instanceStart;
72
- logInterval;
73
- events;
74
- persistenceOptions;
67
+ #instanceStart;
68
+ #logInterval;
69
+ #events;
70
+ #persistenceOptions;
71
+ get events() {
72
+ if (!this.#events) {
73
+ this.#events = serviceLocator.getEventManager();
74
+ }
75
+ return this.#events;
76
+ }
75
77
  /**
76
- * @internal
78
+ * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
79
+ * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
77
80
  */
78
81
  constructor(options = {}) {
79
82
  ow(options, ow.object.exactShape({
@@ -81,23 +84,23 @@ export class Statistics {
81
84
  logMessage: ow.optional.string,
82
85
  log: ow.optional.object,
83
86
  keyValueStore: ow.optional.object,
84
- config: ow.optional.object,
85
87
  persistenceOptions: ow.optional.object,
86
88
  saveErrorSnapshots: ow.optional.boolean,
89
+ id: ow.optional.any(ow.number, ow.string),
87
90
  }));
88
- const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = Configuration.getGlobalConfig(), persistenceOptions = {
91
+ const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
89
92
  enable: true,
90
- }, saveErrorSnapshots = false, } = options;
91
- this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
93
+ }, saveErrorSnapshots = false, id, } = options;
94
+ this.id = id ?? String(Statistics.id++);
95
+ this.persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
96
+ this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
92
97
  this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
93
98
  this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
94
- this.logIntervalMillis = logIntervalSecs * 1000;
95
- this.logMessage = logMessage;
99
+ this.#logIntervalMillis = logIntervalSecs * 1000;
100
+ this.#logMessage = logMessage;
96
101
  this.keyValueStore = keyValueStore;
97
- this.listener = this.persistState.bind(this);
98
- this.events = config.getEventManager();
99
- this.config = config;
100
- this.persistenceOptions = persistenceOptions;
102
+ this.#listener = this.persistState.bind(this);
103
+ this.#persistenceOptions = persistenceOptions;
101
104
  // initialize by "resetting"
102
105
  this.reset();
103
106
  }
@@ -126,15 +129,15 @@ export class Statistics {
126
129
  retryErrors: this.errorTrackerRetry.result,
127
130
  };
128
131
  this.requestRetryHistogram.length = 0;
129
- this.requestsInProgress.clear();
130
- this.instanceStart = Date.now();
131
- this._teardown();
132
+ this.#requestsInProgress.clear();
133
+ this.#instanceStart = Date.now();
134
+ this.teardown();
132
135
  }
133
136
  /**
134
137
  * @param options - Override the persistence options provided in the constructor
135
138
  */
136
139
  async resetStore(options) {
137
- if (!this.persistenceOptions.enable && !options?.enable) {
140
+ if (!this.#persistenceOptions.enable && !options?.enable) {
138
141
  return;
139
142
  }
140
143
  if (!this.keyValueStore) {
@@ -157,49 +160,57 @@ export class Statistics {
157
160
  * @ignore
158
161
  */
159
162
  startJob(id) {
160
- let job = this.requestsInProgress.get(id);
163
+ let job = this.#requestsInProgress.get(id);
161
164
  if (!job)
162
165
  job = new Job();
163
166
  job.run();
164
- this.requestsInProgress.set(id, job);
167
+ this.#requestsInProgress.set(id, job);
165
168
  }
166
169
  /**
167
170
  * Mark job as finished and sets the state
168
171
  * @ignore
169
172
  */
170
173
  finishJob(id, retryCount) {
171
- const job = this.requestsInProgress.get(id);
174
+ const job = this.#requestsInProgress.get(id);
172
175
  if (!job)
173
176
  return;
174
177
  const jobDurationMillis = job.finish();
175
178
  this.state.requestsFinished++;
176
179
  this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
177
- this._saveRetryCountForJob(retryCount);
180
+ this.saveRetryCountForJob(retryCount);
178
181
  if (jobDurationMillis < this.state.requestMinDurationMillis)
179
182
  this.state.requestMinDurationMillis = jobDurationMillis;
180
183
  if (jobDurationMillis > this.state.requestMaxDurationMillis)
181
184
  this.state.requestMaxDurationMillis = jobDurationMillis;
182
- this.requestsInProgress.delete(id);
185
+ this.#requestsInProgress.delete(id);
183
186
  }
184
187
  /**
185
188
  * Mark job as failed and sets the state
186
189
  * @ignore
187
190
  */
188
191
  failJob(id, retryCount) {
189
- const job = this.requestsInProgress.get(id);
192
+ const job = this.#requestsInProgress.get(id);
190
193
  if (!job)
191
194
  return;
192
195
  this.state.requestTotalFailedDurationMillis += job.finish();
193
196
  this.state.requestsFailed++;
194
- this._saveRetryCountForJob(retryCount);
195
- this.requestsInProgress.delete(id);
197
+ this.saveRetryCountForJob(retryCount);
198
+ this.#requestsInProgress.delete(id);
199
+ }
200
+ /**
201
+ * Discards a started job without affecting the finished/failed counters, e.g. when a request
202
+ * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
203
+ * @ignore
204
+ */
205
+ discardJob(id) {
206
+ this.#requestsInProgress.delete(id);
196
207
  }
197
208
  /**
198
209
  * Calculate the current statistics
199
210
  */
200
211
  calculate() {
201
212
  const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
202
- const totalMillis = Date.now() - this.instanceStart;
213
+ const totalMillis = Date.now() - this.#instanceStart;
203
214
  const totalMinutes = totalMillis / 1000 / 60;
204
215
  return {
205
216
  requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
@@ -216,30 +227,35 @@ export class Statistics {
216
227
  * displaying the current state in predefined intervals
217
228
  */
218
229
  async startCapturing() {
219
- this.keyValueStore ??= await KeyValueStore.open(null, { config: this.config });
230
+ // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
231
+ // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
232
+ if (this.#logInterval) {
233
+ throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
234
+ }
235
+ this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
220
236
  if (this.state.crawlerStartedAt === null) {
221
237
  this.state.crawlerStartedAt = new Date();
222
238
  }
223
- if (this.persistenceOptions.enable) {
224
- await this._maybeLoadStatistics();
225
- this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
239
+ if (this.#persistenceOptions.enable) {
240
+ await this.maybeLoadStatistics();
241
+ this.events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
226
242
  }
227
- this.logInterval = setInterval(() => {
228
- this.log.info(this.logMessage, {
243
+ this.#logInterval = setInterval(() => {
244
+ this.log.info(this.#logMessage, {
229
245
  ...this.calculate(),
230
246
  retryHistogram: this.requestRetryHistogram,
231
247
  });
232
- }, this.logIntervalMillis);
248
+ }, this.#logIntervalMillis);
233
249
  }
234
250
  /**
235
251
  * Stops logging and remove event listeners, then persist
236
252
  */
237
253
  async stopCapturing() {
238
- this._teardown();
254
+ this.teardown();
239
255
  this.state.crawlerFinishedAt = new Date();
240
256
  await this.persistState();
241
257
  }
242
- _saveRetryCountForJob(retryCount) {
258
+ saveRetryCountForJob(retryCount) {
243
259
  if (retryCount > 0)
244
260
  this.state.requestsRetries++;
245
261
  this.requestRetryHistogram[retryCount] ??= 0;
@@ -250,7 +266,7 @@ export class Statistics {
250
266
  * @param options - Override the persistence options provided in the constructor
251
267
  */
252
268
  async persistState(options) {
253
- if (!this.persistenceOptions.enable && !options?.enable) {
269
+ if (!this.#persistenceOptions.enable && !options?.enable) {
254
270
  return;
255
271
  }
256
272
  // this might be called before startCapturing was called without using await, should not crash
@@ -258,20 +274,14 @@ export class Statistics {
258
274
  return;
259
275
  }
260
276
  this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
261
- // use half the interval of `persistState` to avoid race conditions
262
- const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
263
- const timeoutSecs = persistStateIntervalMillis / 2_000;
264
277
  await this.keyValueStore
265
- .setValue(this.persistStateKey, this.toJSON(), {
266
- timeoutSecs,
267
- doNotRetryTimeouts: true,
268
- })
278
+ .setValue(this.persistStateKey, this.toJSON())
269
279
  .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
270
280
  }
271
281
  /**
272
282
  * Loads the current statistic from the key value store if any
273
283
  */
274
- async _maybeLoadStatistics() {
284
+ async maybeLoadStatistics() {
275
285
  // this might be called before startCapturing was called without using await, should not crash
276
286
  if (!this.keyValueStore) {
277
287
  return;
@@ -303,15 +313,17 @@ export class Statistics {
303
313
  this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
304
314
  this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
305
315
  this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
306
- this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
316
+ this.#instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
307
317
  this.log.debug('Loaded from KeyValueStore');
308
318
  }
309
- _teardown() {
319
+ teardown() {
310
320
  // this can be called before a call to startCapturing happens (or in a 'finally' block)
311
- this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
312
- if (this.logInterval) {
313
- clearInterval(this.logInterval);
314
- this.logInterval = null;
321
+ // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
322
+ // (e.g. during the constructor's reset() call, which would capture the wrong context)
323
+ this.#events?.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
324
+ if (this.#logInterval) {
325
+ clearInterval(this.#logInterval);
326
+ this.#logInterval = null;
315
327
  }
316
328
  }
317
329
  /**
@@ -324,7 +336,7 @@ export class Statistics {
324
336
  // omit duplicated information
325
337
  const result = {
326
338
  ...this.state,
327
- crawlerLastStartTimestamp: this.instanceStart,
339
+ crawlerLastStartTimestamp: this.#instanceStart,
328
340
  crawlerFinishedAt: this.state.crawlerFinishedAt
329
341
  ? new Date(this.state.crawlerFinishedAt).toISOString()
330
342
  : null,
@@ -343,4 +355,3 @@ export class Statistics {
343
355
  return result;
344
356
  }
345
357
  }
346
- //# sourceMappingURL=statistics.js.map
package/debug.d.ts ADDED
@@ -0,0 +1,36 @@
1
+ import type { IncomingMessage } from 'node:http';
2
+ import type { Dictionary } from '@crawlee/types';
3
+ import type { Request } from './request.js';
4
+ interface BrowserResponseLike {
5
+ status(): number;
6
+ }
7
+ /**
8
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
9
+ *
10
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
11
+ * @param [response]
12
+ * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
13
+ * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
14
+ * @param [additionalFields] Object containing additional fields to be added.
15
+ *
16
+ * @internal
17
+ */
18
+ export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
19
+ /**
20
+ * Returns a human-readable label for an unknown value,
21
+ * suitable for embedding in error messages and log output.
22
+ *
23
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
24
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
25
+ *
26
+ * @internal
27
+ */
28
+ export declare function inspectValue(value: unknown): string;
29
+ /**
30
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
31
+ * by their constructor name. Used for building validation error messages.
32
+ *
33
+ * @internal
34
+ */
35
+ export declare function getObjectType(value: unknown): string;
36
+ export {};
package/debug.js ADDED
@@ -0,0 +1,70 @@
1
+ import { inspect } from 'node:util';
2
+ import ow from 'ow';
3
+ /**
4
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
5
+ *
6
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
7
+ * @param [response]
8
+ * Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
9
+ * or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
10
+ * @param [additionalFields] Object containing additional fields to be added.
11
+ *
12
+ * @internal
13
+ */
14
+ export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
15
+ ow(request, ow.object);
16
+ ow(response, ow.object);
17
+ ow(additionalFields, ow.object);
18
+ return {
19
+ requestId: request.id,
20
+ url: request.url,
21
+ loadedUrl: request.loadedUrl,
22
+ method: request.method,
23
+ retryCount: request.retryCount,
24
+ errorMessages: request.errorMessages,
25
+ // Puppeteer response has .status() function and NodeJS response, statusCode property.
26
+ statusCode: 'status' in response && response.status instanceof Function
27
+ ? response.status()
28
+ : response.statusCode,
29
+ ...additionalFields,
30
+ };
31
+ }
32
+ /**
33
+ * Returns a human-readable label for an unknown value,
34
+ * suitable for embedding in error messages and log output.
35
+ *
36
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
37
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
38
+ *
39
+ * @internal
40
+ */
41
+ export function inspectValue(value) {
42
+ if (typeof value === 'object' && value !== null && value.constructor?.name) {
43
+ return value.constructor.name;
44
+ }
45
+ return inspect(value, {
46
+ depth: 0,
47
+ compact: true,
48
+ maxStringLength: 64,
49
+ breakLength: Infinity,
50
+ colors: false,
51
+ });
52
+ }
53
+ /**
54
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
55
+ * by their constructor name. Used for building validation error messages.
56
+ *
57
+ * @internal
58
+ */
59
+ export function getObjectType(value) {
60
+ const simple = typeof value;
61
+ if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
62
+ return simple;
63
+ }
64
+ const objectType = Object.prototype.toString.call(value);
65
+ const type = /\[object (\w+)]/.exec(objectType)[1];
66
+ if (type === 'Uint8Array') {
67
+ return 'Buffer';
68
+ }
69
+ return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
70
+ }