@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +75 -167
  5. package/configuration.js +164 -140
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +40 -19
  9. package/errors.js +52 -30
  10. package/events/{event_manager.d.ts → event-manager.d.ts} +29 -6
  11. package/events/{event_manager.js → event-manager.js} +10 -9
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/{local_event_manager.d.ts → local-event-manager.d.ts} +3 -10
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -13
  19. package/index.js +10 -12
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +108 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +23 -23
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -101
  50. package/request.js +139 -249
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/{service_locator.d.ts → service-locator.d.ts} +66 -37
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +39 -0
  56. package/storages/batched-adds.js +81 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -151
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +32 -171
  68. package/storages/{request_list.js → request-list.js} +276 -253
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +34 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +127 -161
  76. package/storages/request-queue.js +871 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -17
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -298
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -527
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -179
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -279
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -178
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -168
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -204
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -350
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -254
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -268
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -84
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -219
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js +0 -93
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/proxy_configuration.d.ts +0 -169
  179. package/proxy_configuration.d.ts.map +0 -1
  180. package/proxy_configuration.js +0 -247
  181. package/proxy_configuration.js.map +0 -1
  182. package/recoverable_state.d.ts +0 -122
  183. package/recoverable_state.d.ts.map +0 -1
  184. package/recoverable_state.js +0 -138
  185. package/recoverable_state.js.map +0 -1
  186. package/request.d.ts.map +0 -1
  187. package/request.js.map +0 -1
  188. package/router.d.ts +0 -134
  189. package/router.d.ts.map +0 -1
  190. package/router.js +0 -164
  191. package/router.js.map +0 -1
  192. package/serialization.d.ts.map +0 -1
  193. package/serialization.js.map +0 -1
  194. package/service_locator.d.ts.map +0 -1
  195. package/service_locator.js +0 -252
  196. package/service_locator.js.map +0 -1
  197. package/session_pool/consts.d.ts +0 -4
  198. package/session_pool/consts.d.ts.map +0 -1
  199. package/session_pool/consts.js +0 -4
  200. package/session_pool/consts.js.map +0 -1
  201. package/session_pool/errors.d.ts +0 -8
  202. package/session_pool/errors.d.ts.map +0 -1
  203. package/session_pool/errors.js +0 -12
  204. package/session_pool/errors.js.map +0 -1
  205. package/session_pool/events.d.ts +0 -3
  206. package/session_pool/events.d.ts.map +0 -1
  207. package/session_pool/events.js +0 -3
  208. package/session_pool/events.js.map +0 -1
  209. package/session_pool/index.d.ts +0 -6
  210. package/session_pool/index.d.ts.map +0 -1
  211. package/session_pool/index.js +0 -6
  212. package/session_pool/index.js.map +0 -1
  213. package/session_pool/session.d.ts +0 -186
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js +0 -278
  216. package/session_pool/session.js.map +0 -1
  217. package/session_pool/session_pool.d.ts +0 -256
  218. package/session_pool/session_pool.d.ts.map +0 -1
  219. package/session_pool/session_pool.js +0 -386
  220. package/session_pool/session_pool.js.map +0 -1
  221. package/storages/access_checking.d.ts +0 -13
  222. package/storages/access_checking.d.ts.map +0 -1
  223. package/storages/access_checking.js +0 -14
  224. package/storages/access_checking.js.map +0 -1
  225. package/storages/dataset.d.ts.map +0 -1
  226. package/storages/dataset.js.map +0 -1
  227. package/storages/index.d.ts.map +0 -1
  228. package/storages/index.js.map +0 -1
  229. package/storages/key_value_store.d.ts.map +0 -1
  230. package/storages/key_value_store.js +0 -513
  231. package/storages/key_value_store.js.map +0 -1
  232. package/storages/request_list.d.ts.map +0 -1
  233. package/storages/request_list.js.map +0 -1
  234. package/storages/request_list_adapter.d.ts +0 -58
  235. package/storages/request_list_adapter.d.ts.map +0 -1
  236. package/storages/request_list_adapter.js +0 -81
  237. package/storages/request_list_adapter.js.map +0 -1
  238. package/storages/request_manager_tandem.d.ts +0 -68
  239. package/storages/request_manager_tandem.d.ts.map +0 -1
  240. package/storages/request_manager_tandem.js +0 -124
  241. package/storages/request_manager_tandem.js.map +0 -1
  242. package/storages/request_provider.d.ts.map +0 -1
  243. package/storages/request_provider.js +0 -592
  244. package/storages/request_provider.js.map +0 -1
  245. package/storages/request_queue.d.ts +0 -128
  246. package/storages/request_queue.d.ts.map +0 -1
  247. package/storages/request_queue.js +0 -320
  248. package/storages/request_queue.js.map +0 -1
  249. package/storages/request_queue_v2.d.ts +0 -87
  250. package/storages/request_queue_v2.d.ts.map +0 -1
  251. package/storages/request_queue_v2.js +0 -437
  252. package/storages/request_queue_v2.js.map +0 -1
  253. package/storages/sitemap_request_list.d.ts +0 -245
  254. package/storages/sitemap_request_list.d.ts.map +0 -1
  255. package/storages/sitemap_request_list.js +0 -431
  256. package/storages/sitemap_request_list.js.map +0 -1
  257. package/storages/storage_manager.d.ts +0 -60
  258. package/storages/storage_manager.d.ts.map +0 -1
  259. package/storages/storage_manager.js +0 -95
  260. package/storages/storage_manager.js.map +0 -1
  261. package/storages/utils.d.ts.map +0 -1
  262. package/storages/utils.js.map +0 -1
  263. package/typedefs.d.ts.map +0 -1
  264. package/typedefs.js.map +0 -1
  265. package/validators.d.ts.map +0 -1
  266. package/validators.js.map +0 -1
@@ -1,350 +0,0 @@
1
- import ow from 'ow';
2
- import { log as defaultLog } from '../log.js';
3
- import { serviceLocator } from '../service_locator.js';
4
- import { KeyValueStore } from '../storages/key_value_store.js';
5
- import { ErrorTracker } from './error_tracker.js';
6
- /**
7
- * @ignore
8
- */
9
- class Job {
10
- lastRunAt = null;
11
- durationMillis;
12
- run() {
13
- this.lastRunAt = Date.now();
14
- }
15
- finish() {
16
- this.durationMillis = Date.now() - this.lastRunAt;
17
- return this.durationMillis;
18
- }
19
- }
20
- const errorTrackerConfig = {
21
- showErrorCode: true,
22
- showErrorName: true,
23
- showStackTrace: true,
24
- showFullStack: false,
25
- showErrorMessage: true,
26
- showFullMessage: false,
27
- };
28
- /**
29
- * The statistics class provides an interface to collecting and logging run
30
- * statistics for requests.
31
- *
32
- * All statistic information is saved on key value store
33
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
34
- * migrations and abort/resurrect
35
- *
36
- * @category Crawlers
37
- */
38
- export class Statistics {
39
- static id = 0;
40
- /**
41
- * An error tracker for final retry errors.
42
- */
43
- errorTracker;
44
- /**
45
- * An error tracker for retry errors prior to the final retry.
46
- */
47
- errorTrackerRetry;
48
- /**
49
- * Statistic instance id.
50
- */
51
- id;
52
- /**
53
- * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
54
- */
55
- state;
56
- /**
57
- * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
58
- */
59
- requestRetryHistogram = [];
60
- keyValueStore = undefined;
61
- persistStateKey;
62
- logIntervalMillis;
63
- logMessage;
64
- listener;
65
- requestsInProgress = new Map();
66
- log;
67
- instanceStart;
68
- logInterval;
69
- _events;
70
- persistenceOptions;
71
- get events() {
72
- if (!this._events) {
73
- this._events = serviceLocator.getEventManager();
74
- }
75
- return this._events;
76
- }
77
- /**
78
- * @internal
79
- */
80
- constructor(options = {}) {
81
- ow(options, ow.object.exactShape({
82
- logIntervalSecs: ow.optional.number,
83
- logMessage: ow.optional.string,
84
- log: ow.optional.object,
85
- keyValueStore: ow.optional.object,
86
- persistenceOptions: ow.optional.object,
87
- saveErrorSnapshots: ow.optional.boolean,
88
- id: ow.optional.any(ow.number, ow.string),
89
- }));
90
- const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, persistenceOptions = {
91
- enable: true,
92
- }, saveErrorSnapshots = false, id, } = options;
93
- this.id = id ?? String(Statistics.id++);
94
- this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
95
- this.log = (options.log ?? defaultLog).child({ prefix: 'Statistics' });
96
- this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
97
- this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
98
- this.logIntervalMillis = logIntervalSecs * 1000;
99
- this.logMessage = logMessage;
100
- this.keyValueStore = keyValueStore;
101
- this.listener = this.persistState.bind(this);
102
- this.persistenceOptions = persistenceOptions;
103
- // initialize by "resetting"
104
- this.reset();
105
- }
106
- /**
107
- * Set the current statistic instance to pristine values
108
- */
109
- reset() {
110
- this.errorTracker.reset();
111
- this.errorTrackerRetry.reset();
112
- this.state = {
113
- requestsFinished: 0,
114
- requestsFailed: 0,
115
- requestsRetries: 0,
116
- requestsFailedPerMinute: 0,
117
- requestsFinishedPerMinute: 0,
118
- requestMinDurationMillis: Infinity,
119
- requestMaxDurationMillis: 0,
120
- requestTotalFailedDurationMillis: 0,
121
- requestTotalFinishedDurationMillis: 0,
122
- crawlerStartedAt: null,
123
- crawlerFinishedAt: null,
124
- statsPersistedAt: null,
125
- crawlerRuntimeMillis: 0,
126
- requestsWithStatusCode: {},
127
- errors: this.errorTracker.result,
128
- retryErrors: this.errorTrackerRetry.result,
129
- };
130
- this.requestRetryHistogram.length = 0;
131
- this.requestsInProgress.clear();
132
- this.instanceStart = Date.now();
133
- this._teardown();
134
- }
135
- /**
136
- * @param options - Override the persistence options provided in the constructor
137
- */
138
- async resetStore(options) {
139
- if (!this.persistenceOptions.enable && !options?.enable) {
140
- return;
141
- }
142
- if (!this.keyValueStore) {
143
- return;
144
- }
145
- await this.keyValueStore.setValue(this.persistStateKey, null);
146
- }
147
- /**
148
- * Increments the status code counter.
149
- */
150
- registerStatusCode(code) {
151
- const s = String(code);
152
- if (this.state.requestsWithStatusCode[s] === undefined) {
153
- this.state.requestsWithStatusCode[s] = 0;
154
- }
155
- this.state.requestsWithStatusCode[s]++;
156
- }
157
- /**
158
- * Starts a job
159
- * @ignore
160
- */
161
- startJob(id) {
162
- let job = this.requestsInProgress.get(id);
163
- if (!job)
164
- job = new Job();
165
- job.run();
166
- this.requestsInProgress.set(id, job);
167
- }
168
- /**
169
- * Mark job as finished and sets the state
170
- * @ignore
171
- */
172
- finishJob(id, retryCount) {
173
- const job = this.requestsInProgress.get(id);
174
- if (!job)
175
- return;
176
- const jobDurationMillis = job.finish();
177
- this.state.requestsFinished++;
178
- this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
179
- this._saveRetryCountForJob(retryCount);
180
- if (jobDurationMillis < this.state.requestMinDurationMillis)
181
- this.state.requestMinDurationMillis = jobDurationMillis;
182
- if (jobDurationMillis > this.state.requestMaxDurationMillis)
183
- this.state.requestMaxDurationMillis = jobDurationMillis;
184
- this.requestsInProgress.delete(id);
185
- }
186
- /**
187
- * Mark job as failed and sets the state
188
- * @ignore
189
- */
190
- failJob(id, retryCount) {
191
- const job = this.requestsInProgress.get(id);
192
- if (!job)
193
- return;
194
- this.state.requestTotalFailedDurationMillis += job.finish();
195
- this.state.requestsFailed++;
196
- this._saveRetryCountForJob(retryCount);
197
- this.requestsInProgress.delete(id);
198
- }
199
- /**
200
- * Calculate the current statistics
201
- */
202
- calculate() {
203
- const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
204
- const totalMillis = Date.now() - this.instanceStart;
205
- const totalMinutes = totalMillis / 1000 / 60;
206
- return {
207
- requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
208
- requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
209
- requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
210
- requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
211
- requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
212
- requestsTotal: requestsFailed + requestsFinished,
213
- crawlerRuntimeMillis: totalMillis,
214
- };
215
- }
216
- /**
217
- * Initializes the key value store for persisting the statistics,
218
- * displaying the current state in predefined intervals
219
- */
220
- async startCapturing() {
221
- this.keyValueStore ??= await KeyValueStore.open(null, { config: serviceLocator.getConfiguration() });
222
- if (this.state.crawlerStartedAt === null) {
223
- this.state.crawlerStartedAt = new Date();
224
- }
225
- if (this.persistenceOptions.enable) {
226
- await this._maybeLoadStatistics();
227
- this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
228
- }
229
- this.logInterval = setInterval(() => {
230
- this.log.info(this.logMessage, {
231
- ...this.calculate(),
232
- retryHistogram: this.requestRetryHistogram,
233
- });
234
- }, this.logIntervalMillis);
235
- }
236
- /**
237
- * Stops logging and remove event listeners, then persist
238
- */
239
- async stopCapturing() {
240
- this._teardown();
241
- this.state.crawlerFinishedAt = new Date();
242
- await this.persistState();
243
- }
244
- _saveRetryCountForJob(retryCount) {
245
- if (retryCount > 0)
246
- this.state.requestsRetries++;
247
- this.requestRetryHistogram[retryCount] ??= 0;
248
- this.requestRetryHistogram[retryCount]++;
249
- }
250
- /**
251
- * Persist internal state to the key value store
252
- * @param options - Override the persistence options provided in the constructor
253
- */
254
- async persistState(options) {
255
- if (!this.persistenceOptions.enable && !options?.enable) {
256
- return;
257
- }
258
- // this might be called before startCapturing was called without using await, should not crash
259
- if (!this.keyValueStore) {
260
- return;
261
- }
262
- this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
263
- // use half the interval of `persistState` to avoid race conditions
264
- const persistStateIntervalMillis = serviceLocator.getConfiguration().get('persistStateIntervalMillis');
265
- const timeoutSecs = persistStateIntervalMillis / 2_000;
266
- await this.keyValueStore
267
- .setValue(this.persistStateKey, this.toJSON(), {
268
- timeoutSecs,
269
- doNotRetryTimeouts: true,
270
- })
271
- .catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
272
- }
273
- /**
274
- * Loads the current statistic from the key value store if any
275
- */
276
- async _maybeLoadStatistics() {
277
- // this might be called before startCapturing was called without using await, should not crash
278
- if (!this.keyValueStore) {
279
- return;
280
- }
281
- const savedState = await this.keyValueStore.getValue(this.persistStateKey);
282
- if (!savedState)
283
- return;
284
- // We saw a run where the requestRetryHistogram was not iterable and crashed
285
- // the crawler. Adding some logging to monitor this problem in the future.
286
- if (!Array.isArray(savedState.requestRetryHistogram)) {
287
- this.log.warning('Received invalid state from Key-value store.', {
288
- persistStateKey: this.persistStateKey,
289
- state: savedState,
290
- });
291
- }
292
- this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });
293
- // the `requestRetryHistogram` array might be very large, we could end up with
294
- // `RangeError: Maximum call stack size exceeded` if we use `a.push(...b)`
295
- savedState.requestRetryHistogram.forEach((idx) => this.requestRetryHistogram.push(idx));
296
- this.state.requestsFinished = savedState.requestsFinished;
297
- this.state.requestsFailed = savedState.requestsFailed;
298
- this.state.requestsRetries = savedState.requestsRetries;
299
- this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;
300
- this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;
301
- this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;
302
- this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;
303
- // persisted state uses ISO date strings
304
- this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;
305
- this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
306
- this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
307
- this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
308
- this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
309
- this.log.debug('Loaded from KeyValueStore');
310
- }
311
- _teardown() {
312
- // this can be called before a call to startCapturing happens (or in a 'finally' block)
313
- // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
314
- // (e.g. during the constructor's reset() call, which would capture the wrong context)
315
- this._events?.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
316
- if (this.logInterval) {
317
- clearInterval(this.logInterval);
318
- this.logInterval = null;
319
- }
320
- }
321
- /**
322
- * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
323
- * or through `keyValueStore.setValue('KEY', statsInstance)`
324
- */
325
- toJSON() {
326
- // merge all the current state information that can be used from the outside
327
- // without the need to reconstruct for the sake of stats.calculate()
328
- // omit duplicated information
329
- const result = {
330
- ...this.state,
331
- crawlerLastStartTimestamp: this.instanceStart,
332
- crawlerFinishedAt: this.state.crawlerFinishedAt
333
- ? new Date(this.state.crawlerFinishedAt).toISOString()
334
- : null,
335
- crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
336
- requestRetryHistogram: this.requestRetryHistogram,
337
- statsId: this.id,
338
- statsPersistedAt: new Date().toISOString(),
339
- ...this.calculate(),
340
- };
341
- Reflect.deleteProperty(result, 'requestsWithStatusCode');
342
- Reflect.deleteProperty(result, 'errors');
343
- Reflect.deleteProperty(result, 'retryErrors');
344
- result.requestsWithStatusCode = this.state.requestsWithStatusCode;
345
- result.errors = this.state.errors;
346
- result.retryErrors = this.state.retryErrors;
347
- return result;
348
- }
349
- }
350
- //# sourceMappingURL=statistics.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAMpB,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;GAEG;AACH,MAAM,GAAG;IACG,SAAS,GAAkB,IAAI,CAAC;IAChC,cAAc,CAAU;IAEhC,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAM,OAAO,UAAU;IACX,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAEtB;;OAEG;IACH,YAAY,CAAe;IAE3B;;OAEG;IACH,iBAAiB,CAAe;IAEhC;;OAEG;IACM,EAAE,CAAS;IAEpB;;OAEG;IACH,KAAK,CAAkB;IAEvB;;OAEG;IACM,qBAAqB,GAAa,EAAE,CAAC;IAEpC,aAAa,GAAmB,SAAS,CAAC;IAC1C,eAAe,CAAS;IAC1B,iBAAiB,CAAS;IAC1B,UAAU,CAAS;IACnB,QAAQ,CAAsB;IAC9B,kBAAkB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5C,GAAG,CAAM;IAClB,aAAa,CAAU;IACvB,WAAW,CAAU;IACrB,OAAO,CAAgB;IACvB,kBAAkB,CAAqB;IAE/C,IAAY,MAAM;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,YAAY,UAA6B,EAAE;QACvC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YACvC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;SAC5C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,EAC1B,EAAE,GACL,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,0BAA0B,IAAI,CAAC,EAAE,EAAE,CAAC;QAE3D,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB,EAAE,UAAkB;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB,EAAE,UAAkB;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAK,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,UAAkB;QAC9C,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QACxG,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;YAC3C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uCAAuC,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7F,CAAC;IACV,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,sFAAsF;QACtF,sFAAsF;QACtF,IAAI,CAAC,OAAO,EAAE,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC"}
@@ -1,254 +0,0 @@
1
- import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
2
- import { type RobotsTxtFile } from '@crawlee/utils';
3
- import type { SetRequired } from 'type-fest';
4
- import { Request } from '../request.js';
5
- import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
6
- import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
7
- export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
8
- /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
9
- limit?: number;
10
- /** An array of URLs to enqueue. */
11
- urls?: readonly string[];
12
- /** A request queue to which the URLs will be enqueued. */
13
- requestQueue?: RequestProvider;
14
- /** A CSS selector matching links to be enqueued. */
15
- selector?: string;
16
- /** Sets {@link Request.userData} for newly enqueued requests. */
17
- userData?: Dictionary;
18
- /**
19
- * Sets {@link Request.label} for newly enqueued requests.
20
- *
21
- * This option has the lowest priority and can be overwritten by request options
22
- * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
23
- */
24
- label?: string;
25
- /**
26
- * If set to `true`, tells the crawler to skip navigation and process the request directly.
27
- * @default false
28
- */
29
- skipNavigation?: boolean;
30
- /**
31
- * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
32
- * since the relative URL resolution is done inside the browser automatically.
33
- */
34
- baseUrl?: string;
35
- /**
36
- * An array of glob pattern strings or plain objects
37
- * containing glob pattern strings matching the URLs to be enqueued.
38
- *
39
- * The plain objects must include at least the `glob` property, which holds the glob pattern string.
40
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
41
- *
42
- * The matching is always case-insensitive.
43
- * If you need case-sensitive matching, use `regexps` property directly.
44
- *
45
- * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
46
- * enqueues the links with the same subdomain.
47
- */
48
- globs?: readonly GlobInput[];
49
- /**
50
- * An array of glob pattern strings, regexp patterns or plain objects
51
- * containing patterns matching URLs that will **never** be enqueued.
52
- *
53
- * The plain objects must include either the `glob` property or the `regexp` property.
54
- *
55
- * Glob matching is always case-insensitive.
56
- * If you need case-sensitive matching, provide a regexp.
57
- */
58
- exclude?: readonly (GlobInput | RegExpInput)[];
59
- /**
60
- * An array of regular expressions or plain objects
61
- * containing regular expressions matching the URLs to be enqueued.
62
- *
63
- * The plain objects must include at least the `regexp` property, which holds the regular expression.
64
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
65
- *
66
- * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
67
- * enqueues the links with the same subdomain.
68
- */
69
- regexps?: readonly RegExpInput[];
70
- /**
71
- * *NOTE:* In future versions of SDK the options will be removed.
72
- * Please use `globs` or `regexps` instead.
73
- *
74
- * An array of {@link PseudoUrl} strings or plain objects
75
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
76
- *
77
- * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
78
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
79
- *
80
- * With a pseudo-URL string, the matching is always case-insensitive.
81
- * If you need case-sensitive matching, use `regexps` property directly.
82
- *
83
- * If `pseudoUrls` is an empty array or `undefined`, then the function
84
- * enqueues the links with the same subdomain.
85
- *
86
- * @deprecated prefer using `globs` or `regexps` instead
87
- */
88
- pseudoUrls?: readonly PseudoUrlInput[];
89
- /**
90
- * After request options are filtered by patterns, this function can be used
91
- * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
92
- * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
93
- * or to dynamically update or create `userData`.
94
- *
95
- * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
96
- * when `uniqueKey` is computed.
97
- *
98
- * **Example:**
99
- * ```javascript
100
- * {
101
- * transformRequestFunction: (request) => {
102
- * request.userData.foo = 'bar';
103
- * request.keepUrlFragment = true;
104
- * return request;
105
- * }
106
- * }
107
- * ```
108
- *
109
- * Note that `transformRequestFunction` has the highest priority and can overwrite request options
110
- * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
111
- *
112
- * The function receives a {@link RequestOptions} object and can return either:
113
- * - The modified {@link RequestOptions} object
114
- * - `'unchanged'` to keep the original options as-is
115
- * - A falsy value or `'skip'` to exclude the request from the queue
116
- */
117
- transformRequestFunction?: RequestTransform;
118
- /**
119
- * The strategy to use when enqueueing the urls.
120
- *
121
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
122
- *
123
- * ```md
124
- * Protocol Domain
125
- * ┌────┐ ┌─────────┐
126
- * https://example.crawlee.dev/...
127
- * │ └─────────────────┤
128
- * │ Hostname │
129
- * │ │
130
- * └─────────────────────────┘
131
- * Origin
132
- *```
133
- *
134
- * @default EnqueueStrategy.SameHostname
135
- */
136
- strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
137
- /**
138
- * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
139
- * You can use this option to wait for adding all of them.
140
- */
141
- waitForAllRequestsToBeAdded?: boolean;
142
- /**
143
- * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
144
- * If provided, disallowed URLs will be ignored.
145
- */
146
- robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
147
- /**
148
- * When a request is skipped for some reason, you can use this callback to act on it.
149
- * This is currently fired for requests skipped
150
- * 1. based on robots.txt file,
151
- * 2. because they don't match enqueueLinks filters,
152
- * 3. or because the maxRequestsPerCrawl limit has been reached
153
- */
154
- onSkippedRequest?: SkippedRequestCallback;
155
- }
156
- /**
157
- * The different enqueueing strategies available.
158
- *
159
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
160
- *
161
- * ```md
162
- * Protocol Domain
163
- * ┌────┐ ┌─────────┐
164
- * https://example.crawlee.dev/...
165
- * │ └─────────────────┤
166
- * │ Hostname │
167
- * │ │
168
- * └─────────────────────────┘
169
- * Origin
170
- *```
171
- *
172
- * - The `Protocol` is usually `http` or `https`
173
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
174
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
175
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
176
- */
177
- export declare enum EnqueueStrategy {
178
- /**
179
- * Matches any URLs found
180
- */
181
- All = "all",
182
- /**
183
- * Matches any URLs that have the same hostname.
184
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
185
- * `https://example.com/hello` will not be matched.
186
- *
187
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
188
- */
189
- SameHostname = "same-hostname",
190
- /**
191
- * Matches any URLs that have the same domain as the base URL.
192
- * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
193
- * `https://example.com`.
194
- *
195
- * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
196
- */
197
- SameDomain = "same-domain",
198
- /**
199
- * Matches any URLs that have the same hostname and protocol.
200
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
201
- * `http://wow.example.com/hello` will not be matched.
202
- *
203
- * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
204
- */
205
- SameOrigin = "same-origin"
206
- }
207
- /**
208
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
209
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
210
- *
211
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
212
- * and override settings of the enqueued {@link Request} objects.
213
- *
214
- * **Example usage**
215
- *
216
- * ```javascript
217
- * await enqueueLinks({
218
- * urls: aListOfFoundUrls,
219
- * requestQueue,
220
- * selector: 'a.product-detail',
221
- * globs: [
222
- * 'https://www.example.com/handbags/*',
223
- * 'https://www.example.com/purses/*'
224
- * ],
225
- * });
226
- * ```
227
- *
228
- * @param options All `enqueueLinks()` parameters are passed via an options object.
229
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
230
- */
231
- export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestQueue'>, 'urls'> & {
232
- requestQueue: {
233
- addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
234
- };
235
- }): Promise<BatchAddRequestsResult>;
236
- /**
237
- * @internal
238
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
239
- * - If a user provides a base url, we always return it
240
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
241
- * request domain, or a redirected one
242
- * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
243
- */
244
- export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
245
- /**
246
- * @internal
247
- */
248
- export interface ResolveBaseUrl {
249
- userProvidedBaseUrl?: string;
250
- enqueueStrategy?: EnqueueLinksOptions['strategy'];
251
- originalRequestUrl: string;
252
- finalRequestUrl?: string;
253
- }
254
- //# sourceMappingURL=enqueue_links.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAK7C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,4BAA4B,EAC/B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACR,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,sBAAsB,EAGzB,MAAM,aAAa,CAAC;AAUrB,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;;OAGG;IACH,aAAa,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAEjD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC7C;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,GAAG;IACtE,YAAY,EAAE;QACV,kBAAkB,EAAE,CAChB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,KACjC,OAAO,CAAC,wBAAwB,CAAC,CAAC;KAC1C,CAAC;CACL,GACF,OAAO,CAAC,sBAAsB,CAAC,CA6NjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}