@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +75 -167
  5. package/configuration.js +164 -140
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +40 -19
  9. package/errors.js +52 -30
  10. package/events/{event_manager.d.ts → event-manager.d.ts} +29 -6
  11. package/events/{event_manager.js → event-manager.js} +10 -9
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/{local_event_manager.d.ts → local-event-manager.d.ts} +3 -10
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -13
  19. package/index.js +10 -12
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +108 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +23 -23
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -101
  50. package/request.js +139 -249
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/{service_locator.d.ts → service-locator.d.ts} +66 -37
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +39 -0
  56. package/storages/batched-adds.js +81 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -151
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +32 -171
  68. package/storages/{request_list.js → request-list.js} +276 -253
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +34 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +127 -161
  76. package/storages/request-queue.js +871 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -17
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -298
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -527
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -179
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -279
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -178
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -168
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -204
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -350
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -254
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -268
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -84
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -219
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts.map +0 -1
  168. package/events/event_manager.js.map +0 -1
  169. package/events/index.d.ts.map +0 -1
  170. package/events/index.js.map +0 -1
  171. package/events/local_event_manager.d.ts.map +0 -1
  172. package/events/local_event_manager.js +0 -93
  173. package/events/local_event_manager.js.map +0 -1
  174. package/index.d.ts.map +0 -1
  175. package/index.js.map +0 -1
  176. package/log.d.ts.map +0 -1
  177. package/log.js.map +0 -1
  178. package/proxy_configuration.d.ts +0 -169
  179. package/proxy_configuration.d.ts.map +0 -1
  180. package/proxy_configuration.js +0 -247
  181. package/proxy_configuration.js.map +0 -1
  182. package/recoverable_state.d.ts +0 -122
  183. package/recoverable_state.d.ts.map +0 -1
  184. package/recoverable_state.js +0 -138
  185. package/recoverable_state.js.map +0 -1
  186. package/request.d.ts.map +0 -1
  187. package/request.js.map +0 -1
  188. package/router.d.ts +0 -134
  189. package/router.d.ts.map +0 -1
  190. package/router.js +0 -164
  191. package/router.js.map +0 -1
  192. package/serialization.d.ts.map +0 -1
  193. package/serialization.js.map +0 -1
  194. package/service_locator.d.ts.map +0 -1
  195. package/service_locator.js +0 -252
  196. package/service_locator.js.map +0 -1
  197. package/session_pool/consts.d.ts +0 -4
  198. package/session_pool/consts.d.ts.map +0 -1
  199. package/session_pool/consts.js +0 -4
  200. package/session_pool/consts.js.map +0 -1
  201. package/session_pool/errors.d.ts +0 -8
  202. package/session_pool/errors.d.ts.map +0 -1
  203. package/session_pool/errors.js +0 -12
  204. package/session_pool/errors.js.map +0 -1
  205. package/session_pool/events.d.ts +0 -3
  206. package/session_pool/events.d.ts.map +0 -1
  207. package/session_pool/events.js +0 -3
  208. package/session_pool/events.js.map +0 -1
  209. package/session_pool/index.d.ts +0 -6
  210. package/session_pool/index.d.ts.map +0 -1
  211. package/session_pool/index.js +0 -6
  212. package/session_pool/index.js.map +0 -1
  213. package/session_pool/session.d.ts +0 -186
  214. package/session_pool/session.d.ts.map +0 -1
  215. package/session_pool/session.js +0 -278
  216. package/session_pool/session.js.map +0 -1
  217. package/session_pool/session_pool.d.ts +0 -256
  218. package/session_pool/session_pool.d.ts.map +0 -1
  219. package/session_pool/session_pool.js +0 -386
  220. package/session_pool/session_pool.js.map +0 -1
  221. package/storages/access_checking.d.ts +0 -13
  222. package/storages/access_checking.d.ts.map +0 -1
  223. package/storages/access_checking.js +0 -14
  224. package/storages/access_checking.js.map +0 -1
  225. package/storages/dataset.d.ts.map +0 -1
  226. package/storages/dataset.js.map +0 -1
  227. package/storages/index.d.ts.map +0 -1
  228. package/storages/index.js.map +0 -1
  229. package/storages/key_value_store.d.ts.map +0 -1
  230. package/storages/key_value_store.js +0 -513
  231. package/storages/key_value_store.js.map +0 -1
  232. package/storages/request_list.d.ts.map +0 -1
  233. package/storages/request_list.js.map +0 -1
  234. package/storages/request_list_adapter.d.ts +0 -58
  235. package/storages/request_list_adapter.d.ts.map +0 -1
  236. package/storages/request_list_adapter.js +0 -81
  237. package/storages/request_list_adapter.js.map +0 -1
  238. package/storages/request_manager_tandem.d.ts +0 -68
  239. package/storages/request_manager_tandem.d.ts.map +0 -1
  240. package/storages/request_manager_tandem.js +0 -124
  241. package/storages/request_manager_tandem.js.map +0 -1
  242. package/storages/request_provider.d.ts.map +0 -1
  243. package/storages/request_provider.js +0 -592
  244. package/storages/request_provider.js.map +0 -1
  245. package/storages/request_queue.d.ts +0 -128
  246. package/storages/request_queue.d.ts.map +0 -1
  247. package/storages/request_queue.js +0 -320
  248. package/storages/request_queue.js.map +0 -1
  249. package/storages/request_queue_v2.d.ts +0 -87
  250. package/storages/request_queue_v2.d.ts.map +0 -1
  251. package/storages/request_queue_v2.js +0 -437
  252. package/storages/request_queue_v2.js.map +0 -1
  253. package/storages/sitemap_request_list.d.ts +0 -245
  254. package/storages/sitemap_request_list.d.ts.map +0 -1
  255. package/storages/sitemap_request_list.js +0 -431
  256. package/storages/sitemap_request_list.js.map +0 -1
  257. package/storages/storage_manager.d.ts +0 -60
  258. package/storages/storage_manager.d.ts.map +0 -1
  259. package/storages/storage_manager.js +0 -95
  260. package/storages/storage_manager.js.map +0 -1
  261. package/storages/utils.d.ts.map +0 -1
  262. package/storages/utils.js.map +0 -1
  263. package/typedefs.d.ts.map +0 -1
  264. package/typedefs.js.map +0 -1
  265. package/validators.d.ts.map +0 -1
  266. package/validators.js.map +0 -1
@@ -1,309 +0,0 @@
1
- import { inspect } from 'node:util';
2
- import { ErrorSnapshotter } from './error_snapshotter.js';
3
- const extractPathFromStackTraceLine = (line) => {
4
- const lastStartingRoundBracketIndex = line.lastIndexOf('(');
5
- if (lastStartingRoundBracketIndex !== -1) {
6
- const closingRoundBracketIndex = line.indexOf(')', lastStartingRoundBracketIndex);
7
- if (closingRoundBracketIndex !== -1) {
8
- return line.slice(lastStartingRoundBracketIndex + 1, closingRoundBracketIndex);
9
- }
10
- }
11
- return line;
12
- };
13
- // https://v8.dev/docs/stack-trace-api#appendix%3A-stack-trace-format
14
- const getPathFromStackTrace = (stack) => {
15
- for (const line of stack) {
16
- const path = extractPathFromStackTraceLine(line);
17
- if (path.startsWith('node:') || path.includes('/node_modules/') || path.includes('\\node_modules\\')) {
18
- continue;
19
- }
20
- return path;
21
- }
22
- return extractPathFromStackTraceLine(stack[0]);
23
- };
24
- const getStackTraceGroup = (error, storage, showFullStack) => {
25
- const stack = error.stack?.split('\n').map((line) => line.trim());
26
- let sliceAt = -1;
27
- if (stack) {
28
- for (let i = 0; i < stack.length; i++) {
29
- if (stack[i].startsWith('at ') || stack[i].startsWith('eval at ')) {
30
- sliceAt = i;
31
- break;
32
- }
33
- }
34
- }
35
- let normalizedStackTrace = null;
36
- if (sliceAt !== -1) {
37
- normalizedStackTrace = showFullStack
38
- ? stack
39
- .slice(sliceAt)
40
- .map((x) => x.trim())
41
- .join('\n')
42
- : getPathFromStackTrace(stack.slice(sliceAt));
43
- }
44
- if (!normalizedStackTrace) {
45
- normalizedStackTrace = 'missing stack trace';
46
- }
47
- if (!(normalizedStackTrace in storage)) {
48
- storage[normalizedStackTrace] = Object.create(null);
49
- }
50
- return storage[normalizedStackTrace];
51
- };
52
- const getErrorCodeGroup = (error, storage) => {
53
- let { code } = error;
54
- if (code === undefined) {
55
- code = 'missing error code';
56
- }
57
- if (!(code in storage)) {
58
- storage[code] = Object.create(null);
59
- }
60
- return storage[String(code)];
61
- };
62
- const getErrorNameGroup = (error, storage) => {
63
- const { name } = error;
64
- if (!(name in storage)) {
65
- storage[name] = Object.create(null);
66
- }
67
- return storage[name];
68
- };
69
- const findBiggestWordIntersection = (a, b) => {
70
- let maxStreak = 0;
71
- let bStreakIndex = -1;
72
- let aStreakIndex = -1;
73
- for (let aIndex = 0; aIndex < a.length; aIndex++) {
74
- let bIndex = -1;
75
- do {
76
- let aWalkIndex = aIndex;
77
- bIndex = b.indexOf(a[aIndex], bIndex + 1);
78
- let bWalkIndex = bIndex;
79
- let streak = 0;
80
- while (aWalkIndex < a.length && bWalkIndex < b.length && b[bWalkIndex++] === a[aWalkIndex++]) {
81
- streak++;
82
- }
83
- if (streak > maxStreak) {
84
- maxStreak = streak;
85
- aStreakIndex = aIndex;
86
- bStreakIndex = bIndex;
87
- }
88
- } while (bIndex !== -1);
89
- }
90
- return {
91
- maxStreak,
92
- aStreakIndex,
93
- bStreakIndex,
94
- };
95
- };
96
- const arrayCount = (array, target) => {
97
- let result = 0;
98
- for (const item of array) {
99
- if (item === target) {
100
- result++;
101
- }
102
- }
103
- return result;
104
- };
105
- const calculatePlaceholder = (a, b) => {
106
- const { maxStreak, aStreakIndex, bStreakIndex } = findBiggestWordIntersection(a, b);
107
- if (maxStreak === 0) {
108
- return ['_'];
109
- }
110
- const leftA = a.slice(0, aStreakIndex);
111
- const leftB = b.slice(0, bStreakIndex);
112
- const rightA = a.slice(aStreakIndex + maxStreak);
113
- const rightB = b.slice(bStreakIndex + maxStreak);
114
- const output = [];
115
- if (leftA.length !== 0 || leftB.length !== 0) {
116
- output.push(...calculatePlaceholder(leftA, leftB));
117
- }
118
- output.push(...a.slice(aStreakIndex, aStreakIndex + maxStreak));
119
- if (rightA.length !== 0 || rightB.length !== 0) {
120
- output.push(...calculatePlaceholder(rightA, rightB));
121
- }
122
- return output;
123
- };
124
- const normalizedCalculatePlaceholder = (a, b) => {
125
- const output = calculatePlaceholder(a, b);
126
- // We can't be too general
127
- if (arrayCount(output, '_') / output.length >= 0.5) {
128
- return ['_'];
129
- }
130
- return output;
131
- };
132
- // Merge A (missing placeholders) into B (can contain placeholders but does not have to)
133
- const mergeMessages = (a, b, storage) => {
134
- const placeholder = normalizedCalculatePlaceholder(a.split(' '), b.split(' ')).join(' ');
135
- if (placeholder === '_') {
136
- return undefined;
137
- }
138
- const count = storage[a].count + storage[b].count;
139
- delete storage[a];
140
- delete storage[b];
141
- storage[placeholder] = Object.assign(Object.create(null), {
142
- count,
143
- });
144
- return placeholder;
145
- };
146
- const getErrorMessageGroup = (error, storage, showFullMessage) => {
147
- let { message } = error;
148
- if (!message) {
149
- try {
150
- message =
151
- typeof error === 'string'
152
- ? error
153
- : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
154
- }
155
- catch {
156
- message = `Unknown error message. Received non-error object, and could not stringify it: ${inspect(error, {
157
- depth: 0,
158
- })}`;
159
- }
160
- }
161
- if (!showFullMessage) {
162
- const newLineIndex = message.indexOf('\n');
163
- message = message.slice(0, newLineIndex === -1 ? undefined : newLineIndex);
164
- }
165
- if (!(message in storage)) {
166
- storage[message] = Object.assign(Object.create(null), {
167
- count: 0,
168
- });
169
- // This actually safe, since we Object.create(null) so no prototype pollution can happen.
170
- // eslint-disable-next-line no-restricted-syntax, guard-for-in
171
- for (const existingMessage in storage) {
172
- const newMessage = mergeMessages(message, existingMessage, storage);
173
- if (newMessage) {
174
- message = newMessage;
175
- break;
176
- }
177
- }
178
- }
179
- return storage[message];
180
- };
181
- const increaseCount = (group) => {
182
- if (!('count' in group)) {
183
- // In case users don't want to display error message
184
- group.count = 0;
185
- }
186
- group.count++;
187
- };
188
- /**
189
- * This class tracks errors and computes a summary of information like:
190
- * - where the errors happened
191
- * - what the error names are
192
- * - what the error codes are
193
- * - what is the general error message
194
- *
195
- * This is extremely useful when there are dynamic error messages, such as argument validation.
196
- *
197
- * Since the structure of the `tracker.result` object differs when using different options,
198
- * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
199
- *
200
- * It's possible to get the total amount of errors via the `tracker.total` property.
201
- */
202
- export class ErrorTracker {
203
- #options;
204
- result;
205
- total;
206
- errorSnapshotter;
207
- constructor(options = {}) {
208
- this.#options = {
209
- showErrorCode: true,
210
- showErrorName: true,
211
- showStackTrace: true,
212
- showFullStack: false,
213
- showErrorMessage: true,
214
- showFullMessage: false,
215
- saveErrorSnapshots: false,
216
- ...options,
217
- };
218
- if (this.#options.saveErrorSnapshots) {
219
- this.errorSnapshotter = new ErrorSnapshotter();
220
- }
221
- this.result = Object.create(null);
222
- this.total = 0;
223
- }
224
- updateGroup(error) {
225
- let group = this.result;
226
- if (this.#options.showStackTrace) {
227
- group = getStackTraceGroup(error, group, this.#options.showFullStack);
228
- }
229
- if (this.#options.showErrorCode) {
230
- group = getErrorCodeGroup(error, group);
231
- }
232
- if (this.#options.showErrorName) {
233
- group = getErrorNameGroup(error, group);
234
- }
235
- if (this.#options.showErrorMessage) {
236
- group = getErrorMessageGroup(error, group, this.#options.showFullMessage);
237
- }
238
- increaseCount(group);
239
- return group;
240
- }
241
- add(error) {
242
- this.total++;
243
- this.updateGroup(error);
244
- if (typeof error.cause === 'object' && error.cause !== null) {
245
- this.add(error.cause);
246
- }
247
- }
248
- /**
249
- * This method is async, because it captures a snapshot of the error context.
250
- * We added this new method to avoid breaking changes.
251
- */
252
- async addAsync(error, context) {
253
- this.total++;
254
- const group = this.updateGroup(error);
255
- // Capture a snapshot (screenshot and HTML) on the first occurrence of an error
256
- if (group.count === 1 && context) {
257
- await this.captureSnapshot(group, error, context).catch(() => { });
258
- }
259
- if (typeof error.cause === 'object' && error.cause !== null) {
260
- await this.addAsync(error.cause);
261
- }
262
- }
263
- getUniqueErrorCount() {
264
- let count = 0;
265
- const goDeeper = (group) => {
266
- if ('count' in group) {
267
- count++;
268
- return;
269
- }
270
- // eslint-disable-next-line guard-for-in, no-restricted-syntax
271
- for (const key in group) {
272
- goDeeper(group[key]);
273
- }
274
- };
275
- goDeeper(this.result);
276
- return count;
277
- }
278
- getMostPopularErrors(count) {
279
- const result = [];
280
- const goDeeper = (group, path) => {
281
- if ('count' in group) {
282
- result.push([group.count, path]);
283
- return;
284
- }
285
- // eslint-disable-next-line guard-for-in, no-restricted-syntax
286
- for (const key in group) {
287
- goDeeper(group[key], [...path, key]);
288
- }
289
- };
290
- goDeeper(this.result, []);
291
- return result.sort((a, b) => b[0] - a[0]).slice(0, count);
292
- }
293
- async captureSnapshot(storage, error, context) {
294
- if (!this.errorSnapshotter) {
295
- return;
296
- }
297
- const { screenshotFileUrl, htmlFileUrl } = await this.errorSnapshotter.captureSnapshot(error, context);
298
- storage.firstErrorScreenshotUrl = screenshotFileUrl;
299
- storage.firstErrorHtmlUrl = htmlFileUrl;
300
- }
301
- reset() {
302
- // This actually safe, since we Object.create(null) so no prototype pollution can happen.
303
- // eslint-disable-next-line no-restricted-syntax, guard-for-in
304
- for (const key in this.result) {
305
- delete this.result[key];
306
- }
307
- }
308
- }
309
- //# sourceMappingURL=error_tracker.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"error_tracker.js","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAwB1D,MAAM,6BAA6B,GAAG,CAAC,IAAY,EAAE,EAAE;IACnD,MAAM,6BAA6B,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE5D,IAAI,6BAA6B,KAAK,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,6BAA6B,CAAC,CAAC;QAElF,IAAI,wBAAwB,KAAK,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,EAAE,wBAAwB,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,qBAAqB,GAAG,CAAC,KAAe,EAAE,EAAE;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnG,SAAS;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,aAAsB,EAAE,EAAE;IAC3G,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAElE,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IAEjB,IAAI,KAAK,EAAE,CAAC;QACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChE,OAAO,GAAG,CAAC,CAAC;gBACZ,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,oBAAoB,GAAG,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,oBAAoB,GAAG,aAAa;YAChC,CAAC,CAAC,KAAM;iBACD,KAAK,CAAC,OAAO,CAAC;iBACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,IAAI,CAAC,IAAI,CAAC;YACjB,CAAC,CAAC,qBAAqB,CAAC,KAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACxB,oBAAoB,GAAG,qBAAqB,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,CAAC,oBAAoB,IAAI,OAAO,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,oBAAoB,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,OAAO,CAAC,oBAAoB,CAA4B,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,IAAI,GAAG,oBAAoB,CAAC;IAChC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAEvB,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAC7D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;QAEhB,GAAG,CAAC;YACA,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YAE1C,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,CAAC;YACb,CAAC;YAED,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;gBACrB,SAAS,GAAG,MAAM,CAAC;gBACnB,YAAY,GAAG,MAAM,CAAC;gBACtB,YAAY,GAAG,MAAM,CAAC;YAC1B,CAAC;QACL,CAAC,QAAQ,MAAM,KAAK,CAAC,CAAC,EAAE;IAC5B,CAAC;IAED,OAAO;QACH,SAAS;QACT,YAAY;QACZ,YAAY;KACf,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,KAAgB,EAAE,MAAe,EAAE,EAAE;IACrD,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IACtD,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC;IAEhE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,8BAA8B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAChE,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE1C,0BAA0B;IAC1B,IAAI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,OAAgC,EAAE,EAAE;IAC7E,MAAM,WAAW,GAAG,8BAA8B,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzF,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAMD,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,CAAC;IAE9E,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAClB,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAElB,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtD,KAAK;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,eAAwB,EAAE,EAAE;IAC/G,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAExB,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,IAAI,CAAC;YACD,OAAO;gBACH,OAAO,KAAK,KAAK,QAAQ;oBACrB,CAAC,CAAC,KAAK;oBACP,CAAC,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,iFAAiF,OAAO,CAAC,KAAK,EAAE;gBACtG,KAAK,EAAE,CAAC;aACX,CAAC,EAAE,CAAC;QACT,CAAC;IACL,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,eAAe,IAAI,OAAO,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACpE,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,GAAG,UAAU,CAAC;gBACrB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAA4B,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAyB,EAAE,EAAE;IAChD,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,oDAAoD;QACpD,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAM,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,YAAY;IACrB,QAAQ,CAAsB;IAE9B,MAAM,CAA0B;IAEhC,KAAK,CAAS;IAEd,gBAAgB,CAAoB;IAEpC,YAAY,UAAwC,EAAE;QAClD,IAAI,CAAC,QAAQ,GAAG;YACZ,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,KAAK;YACtB,kBAAkB,EAAE,KAAK;YACzB,GAAG,OAAO;SACb,CAAC;QAEF,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAqB;QACrC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,aAAa,CAAC,KAA0B,CAAC,CAAC;QAE1C,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,KAAqB;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAqB,EAAE,OAAyB;QAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAEtC,+EAA+E;QAC/E,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;IAED,mBAAmB;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAQ,EAAE;YACtD,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAC;gBACR,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC,CAAC;YACpD,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,oBAAoB,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAE,IAAc,EAAQ,EAAE;YACtE,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,CAAE,KAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1C,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,eAAe,CACjB,OAAgC,EAChC,KAAqB,EACrB,OAAkD;QAElD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEvG,OAAO,CAAC,uBAAuB,GAAG,iBAAiB,CAAC;QACpD,OAAO,CAAC,iBAAiB,GAAG,WAAW,CAAC;IAC5C,CAAC;IAED,KAAK;QACD,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ"}
@@ -1,7 +0,0 @@
1
- export * from './context_pipeline.js';
2
- export * from './crawler_commons.js';
3
- export * from './crawler_utils.js';
4
- export * from './statistics.js';
5
- export * from './error_tracker.js';
6
- export * from './error_snapshotter.js';
7
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
package/crawlers/index.js DELETED
@@ -1,7 +0,0 @@
1
- export * from './context_pipeline.js';
2
- export * from './crawler_commons.js';
3
- export * from './crawler_utils.js';
4
- export * from './statistics.js';
5
- export * from './error_tracker.js';
6
- export * from './error_snapshotter.js';
7
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,wBAAwB,CAAC"}
@@ -1,8 +0,0 @@
1
- export interface BrowserPage {
2
- content: () => Promise<string>;
3
- }
4
- export interface SnapshottableProperties {
5
- body?: unknown;
6
- page?: BrowserPage;
7
- }
8
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/crawlers/internals/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IACxB,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,uBAAuB;IACpC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,WAAW,CAAC;CACtB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/crawlers/internals/types.ts"],"names":[],"mappings":""}
@@ -1,204 +0,0 @@
1
- import type { Log } from '@apify/log';
2
- import { KeyValueStore } from '../storages/key_value_store.js';
3
- import { ErrorTracker } from './error_tracker.js';
4
- /**
5
- * Persistence-related options to control how and when crawler's data gets persisted.
6
- */
7
- export interface PersistenceOptions {
8
- /**
9
- * Use this flag to disable or enable periodic persistence to key value store.
10
- * @default true
11
- */
12
- enable?: boolean;
13
- }
14
- /**
15
- * The statistics class provides an interface to collecting and logging run
16
- * statistics for requests.
17
- *
18
- * All statistic information is saved on key value store
19
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
20
- * migrations and abort/resurrect
21
- *
22
- * @category Crawlers
23
- */
24
- export declare class Statistics {
25
- private static id;
26
- /**
27
- * An error tracker for final retry errors.
28
- */
29
- errorTracker: ErrorTracker;
30
- /**
31
- * An error tracker for retry errors prior to the final retry.
32
- */
33
- errorTrackerRetry: ErrorTracker;
34
- /**
35
- * Statistic instance id.
36
- */
37
- readonly id: string;
38
- /**
39
- * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
40
- */
41
- state: StatisticState;
42
- /**
43
- * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
44
- */
45
- readonly requestRetryHistogram: number[];
46
- protected keyValueStore?: KeyValueStore;
47
- protected persistStateKey: string;
48
- private logIntervalMillis;
49
- private logMessage;
50
- private listener;
51
- private requestsInProgress;
52
- private readonly log;
53
- private instanceStart;
54
- private logInterval;
55
- private _events?;
56
- private persistenceOptions;
57
- private get events();
58
- /**
59
- * @internal
60
- */
61
- constructor(options?: StatisticsOptions);
62
- /**
63
- * Set the current statistic instance to pristine values
64
- */
65
- reset(): void;
66
- /**
67
- * @param options - Override the persistence options provided in the constructor
68
- */
69
- resetStore(options?: PersistenceOptions): Promise<void>;
70
- /**
71
- * Increments the status code counter.
72
- */
73
- registerStatusCode(code: number): void;
74
- /**
75
- * Starts a job
76
- * @ignore
77
- */
78
- startJob(id: number | string): void;
79
- /**
80
- * Mark job as finished and sets the state
81
- * @ignore
82
- */
83
- finishJob(id: number | string, retryCount: number): void;
84
- /**
85
- * Mark job as failed and sets the state
86
- * @ignore
87
- */
88
- failJob(id: number | string, retryCount: number): void;
89
- /**
90
- * Calculate the current statistics
91
- */
92
- calculate(): {
93
- requestAvgFailedDurationMillis: number;
94
- requestAvgFinishedDurationMillis: number;
95
- requestsFinishedPerMinute: number;
96
- requestsFailedPerMinute: number;
97
- requestTotalDurationMillis: number;
98
- requestsTotal: number;
99
- crawlerRuntimeMillis: number;
100
- };
101
- /**
102
- * Initializes the key value store for persisting the statistics,
103
- * displaying the current state in predefined intervals
104
- */
105
- startCapturing(): Promise<void>;
106
- /**
107
- * Stops logging and remove event listeners, then persist
108
- */
109
- stopCapturing(): Promise<void>;
110
- protected _saveRetryCountForJob(retryCount: number): void;
111
- /**
112
- * Persist internal state to the key value store
113
- * @param options - Override the persistence options provided in the constructor
114
- */
115
- persistState(options?: PersistenceOptions): Promise<void>;
116
- /**
117
- * Loads the current statistic from the key value store if any
118
- */
119
- protected _maybeLoadStatistics(): Promise<void>;
120
- protected _teardown(): void;
121
- /**
122
- * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
123
- * or through `keyValueStore.setValue('KEY', statsInstance)`
124
- */
125
- toJSON(): StatisticPersistedState;
126
- }
127
- /**
128
- * Configuration for the {@link Statistics} instance used by the crawler
129
- */
130
- export interface StatisticsOptions {
131
- /**
132
- * Interval in seconds to log the current statistics
133
- * @default 60
134
- */
135
- logIntervalSecs?: number;
136
- /**
137
- * Message to log with the current statistics
138
- * @default 'Statistics'
139
- */
140
- logMessage?: string;
141
- /**
142
- * Parent logger instance, the statistics will create a child logger from this.
143
- * @default crawler.log
144
- */
145
- log?: Log;
146
- /**
147
- * Key value store instance to persist the statistics.
148
- * If not provided, the default one will be used when capturing starts
149
- */
150
- keyValueStore?: KeyValueStore;
151
- /**
152
- * Control how and when to persist the statistics.
153
- */
154
- persistenceOptions?: PersistenceOptions;
155
- /**
156
- * Save HTML snapshot (and a screenshot if possible) when an error occurs.
157
- * @default false
158
- */
159
- saveErrorSnapshots?: boolean;
160
- /**
161
- * A unique identifier for this statistics instance. This ID is used for persistence
162
- * to the key value store, ensuring the same statistics can be loaded after script restarts.
163
- *
164
- * If not provided, an auto-incremented ID will be used for backward compatibility.
165
- * This means statistics may not persist correctly across script restarts
166
- * if crawler creation order changes.
167
- */
168
- id?: string;
169
- }
170
- /**
171
- * Format of the persisted stats
172
- */
173
- export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
174
- requestRetryHistogram: number[];
175
- statsId: string;
176
- requestAvgFailedDurationMillis: number;
177
- requestAvgFinishedDurationMillis: number;
178
- requestTotalDurationMillis: number;
179
- requestsTotal: number;
180
- crawlerLastStartTimestamp: number;
181
- statsPersistedAt: string;
182
- }
183
- /**
184
- * Contains the statistics state
185
- */
186
- export interface StatisticState {
187
- requestsFinished: number;
188
- requestsFailed: number;
189
- requestsRetries: number;
190
- requestsFailedPerMinute: number;
191
- requestsFinishedPerMinute: number;
192
- requestMinDurationMillis: number;
193
- requestMaxDurationMillis: number;
194
- requestTotalFailedDurationMillis: number;
195
- requestTotalFinishedDurationMillis: number;
196
- crawlerStartedAt: Date | string | null;
197
- crawlerFinishedAt: Date | string | null;
198
- crawlerRuntimeMillis: number;
199
- statsPersistedAt: Date | string | null;
200
- errors: Record<string, unknown>;
201
- retryErrors: Record<string, unknown>;
202
- requestsWithStatusCode: Record<string, number>;
203
- }
204
- //# sourceMappingURL=statistics.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAMtC,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AA4BlD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;OAEG;IACH,iBAAiB,EAAE,YAAY,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAe;IAC/B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C,OAAO,KAAK,MAAM,GAKjB;IAED;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAyC3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAcjD;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAS/C;;OAEG;IACH,SAAS;;;;;;;;;IAsBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM;IAMlD;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAyB/C;;OAEG;cACa,oBAAoB;IA0CpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAY3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CA2BpC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;;;;;OAOG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}