@crawlee/core 3.13.3-beta.10 → 3.13.3-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/crawlers/statistics.d.ts
CHANGED
|
@@ -2,17 +2,6 @@ import type { Log } from '@apify/log';
|
|
|
2
2
|
import { Configuration } from '../configuration';
|
|
3
3
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
4
4
|
import { ErrorTracker } from './error_tracker';
|
|
5
|
-
/**
|
|
6
|
-
* @ignore
|
|
7
|
-
*/
|
|
8
|
-
declare class Job {
|
|
9
|
-
private lastRunAt;
|
|
10
|
-
private runs;
|
|
11
|
-
private durationMillis?;
|
|
12
|
-
run(): number;
|
|
13
|
-
finish(): number;
|
|
14
|
-
retryCount(): number;
|
|
15
|
-
}
|
|
16
5
|
/**
|
|
17
6
|
* Persistence-related options to control how and when crawler's data gets persisted.
|
|
18
7
|
*/
|
|
@@ -95,12 +84,12 @@ export declare class Statistics {
|
|
|
95
84
|
* Mark job as finished and sets the state
|
|
96
85
|
* @ignore
|
|
97
86
|
*/
|
|
98
|
-
finishJob(id: number | string): void;
|
|
87
|
+
finishJob(id: number | string, retryCount: number): void;
|
|
99
88
|
/**
|
|
100
89
|
* Mark job as failed and sets the state
|
|
101
90
|
* @ignore
|
|
102
91
|
*/
|
|
103
|
-
failJob(id: number | string): void;
|
|
92
|
+
failJob(id: number | string, retryCount: number): void;
|
|
104
93
|
/**
|
|
105
94
|
* Calculate the current statistics
|
|
106
95
|
*/
|
|
@@ -122,7 +111,7 @@ export declare class Statistics {
|
|
|
122
111
|
* Stops logging and remove event listeners, then persist
|
|
123
112
|
*/
|
|
124
113
|
stopCapturing(): Promise<void>;
|
|
125
|
-
protected _saveRetryCountForJob(
|
|
114
|
+
protected _saveRetryCountForJob(retryCount: number): void;
|
|
126
115
|
/**
|
|
127
116
|
* Persist internal state to the key value store
|
|
128
117
|
* @param options - Override the persistence options provided in the constructor
|
|
@@ -212,5 +201,4 @@ export interface StatisticState {
|
|
|
212
201
|
retryErrors: Record<string, unknown>;
|
|
213
202
|
requestsWithStatusCode: Record<string, number>;
|
|
214
203
|
}
|
|
215
|
-
export {};
|
|
216
204
|
//# sourceMappingURL=statistics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AA4B/C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;OAEG;IACH,iBAAiB,EAAE,YAAY,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,SAAuC;IAChE,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAwC3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAcjD;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAS/C;;OAEG;IACH,SAAS;;;;;;;;;IAsBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM;IAMlD;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAe/C;;OAEG;cACa,oBAAoB;IA0CpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAU3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CA2BpC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}
|
package/crawlers/statistics.js
CHANGED
|
@@ -18,12 +18,6 @@ class Job {
|
|
|
18
18
|
writable: true,
|
|
19
19
|
value: null
|
|
20
20
|
});
|
|
21
|
-
Object.defineProperty(this, "runs", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: 0
|
|
26
|
-
});
|
|
27
21
|
Object.defineProperty(this, "durationMillis", {
|
|
28
22
|
enumerable: true,
|
|
29
23
|
configurable: true,
|
|
@@ -33,15 +27,11 @@ class Job {
|
|
|
33
27
|
}
|
|
34
28
|
run() {
|
|
35
29
|
this.lastRunAt = Date.now();
|
|
36
|
-
return ++this.runs;
|
|
37
30
|
}
|
|
38
31
|
finish() {
|
|
39
32
|
this.durationMillis = Date.now() - this.lastRunAt;
|
|
40
33
|
return this.durationMillis;
|
|
41
34
|
}
|
|
42
|
-
retryCount() {
|
|
43
|
-
return Math.max(0, this.runs - 1);
|
|
44
|
-
}
|
|
45
35
|
}
|
|
46
36
|
const errorTrackerConfig = {
|
|
47
37
|
showErrorCode: true,
|
|
@@ -277,14 +267,14 @@ class Statistics {
|
|
|
277
267
|
* Mark job as finished and sets the state
|
|
278
268
|
* @ignore
|
|
279
269
|
*/
|
|
280
|
-
finishJob(id) {
|
|
270
|
+
finishJob(id, retryCount) {
|
|
281
271
|
const job = this.requestsInProgress.get(id);
|
|
282
272
|
if (!job)
|
|
283
273
|
return;
|
|
284
274
|
const jobDurationMillis = job.finish();
|
|
285
275
|
this.state.requestsFinished++;
|
|
286
276
|
this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
|
|
287
|
-
this._saveRetryCountForJob(
|
|
277
|
+
this._saveRetryCountForJob(retryCount);
|
|
288
278
|
if (jobDurationMillis < this.state.requestMinDurationMillis)
|
|
289
279
|
this.state.requestMinDurationMillis = jobDurationMillis;
|
|
290
280
|
if (jobDurationMillis > this.state.requestMaxDurationMillis)
|
|
@@ -295,13 +285,13 @@ class Statistics {
|
|
|
295
285
|
* Mark job as failed and sets the state
|
|
296
286
|
* @ignore
|
|
297
287
|
*/
|
|
298
|
-
failJob(id) {
|
|
288
|
+
failJob(id, retryCount) {
|
|
299
289
|
const job = this.requestsInProgress.get(id);
|
|
300
290
|
if (!job)
|
|
301
291
|
return;
|
|
302
292
|
this.state.requestTotalFailedDurationMillis += job.finish();
|
|
303
293
|
this.state.requestsFailed++;
|
|
304
|
-
this._saveRetryCountForJob(
|
|
294
|
+
this._saveRetryCountForJob(retryCount);
|
|
305
295
|
this.requestsInProgress.delete(id);
|
|
306
296
|
}
|
|
307
297
|
/**
|
|
@@ -349,13 +339,12 @@ class Statistics {
|
|
|
349
339
|
this.state.crawlerFinishedAt = new Date();
|
|
350
340
|
await this.persistState();
|
|
351
341
|
}
|
|
352
|
-
_saveRetryCountForJob(
|
|
353
|
-
|
|
342
|
+
_saveRetryCountForJob(retryCount) {
|
|
343
|
+
var _a;
|
|
354
344
|
if (retryCount > 0)
|
|
355
345
|
this.state.requestsRetries++;
|
|
356
|
-
this.requestRetryHistogram[retryCount]
|
|
357
|
-
|
|
358
|
-
: 1;
|
|
346
|
+
(_a = this.requestRetryHistogram)[retryCount] ?? (_a[retryCount] = 0);
|
|
347
|
+
this.requestRetryHistogram[retryCount]++;
|
|
359
348
|
}
|
|
360
349
|
/**
|
|
361
350
|
* Persist internal state to the key value store
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAIpB,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAC5D,mDAA+C;AAE/C;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;mBAAO,CAAC;WAAC;QACT;;;;;WAAwB;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;;WAA2B;QAE3B;;WAEG;QACH;;;;;WAAgC;QAEhC;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;QAE7B;;;;mBAAgC,SAAS;WAAC;QAC1C;;;;mBAAkB,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QACxD;;;;;WAA0B;QAC1B;;;;;WAAmB;QACnB;;;;;WAA8B;QAC9B;;;;mBAAqB,IAAI,GAAG,EAAwB;WAAC;QAC5C;;;;;WAAS;QAClB;;;;;WAAuB;QACvB;;;;;WAAqB;QACrB;;;;;WAAqB;QACrB;;;;;WAAuC;QAM3C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SAC1C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,GAC7B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,SAAU,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AA3WL,gCA4WC;AA3WkB;;;;WAAK,CAAC;EAAJ,CAAK"}
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAIpB,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAC5D,mDAA+C;AAE/C;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;;WAAwB;IAUpC,CAAC;IARG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;;WAA2B;QAE3B;;WAEG;QACH;;;;;WAAgC;QAEhC;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;QAE7B;;;;mBAAgC,SAAS;WAAC;QAC1C;;;;mBAAkB,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QACxD;;;;;WAA0B;QAC1B;;;;;WAAmB;QACnB;;;;;WAA8B;QAC9B;;;;mBAAqB,IAAI,GAAG,EAAwB;WAAC;QAC5C;;;;;WAAS;QAClB;;;;;WAAuB;QACvB;;;;;WAAqB;QACrB;;;;;WAAqB;QACrB;;;;;WAAuC;QAM3C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SAC1C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,GAC7B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,SAAU,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB,EAAE,UAAkB;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB,EAAE,UAAkB;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,UAAkB;;QAC9C,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,MAAA,IAAI,CAAC,qBAAqB,EAAC,UAAU,SAAV,UAAU,IAAM,CAAC,EAAC;QAC7C,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AAzWL,gCA0WC;AAzWkB;;;;WAAK,CAAC;EAAJ,CAAK"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.13.3-beta.
|
|
3
|
+
"version": "3.13.3-beta.12",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.13.3-beta.
|
|
63
|
-
"@crawlee/types": "3.13.3-beta.
|
|
64
|
-
"@crawlee/utils": "3.13.3-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.13.3-beta.12",
|
|
63
|
+
"@crawlee/types": "3.13.3-beta.12",
|
|
64
|
+
"@crawlee/utils": "3.13.3-beta.12",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
67
67
|
"csv-stringify": "^6.2.0",
|
|
@@ -83,5 +83,5 @@
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
},
|
|
86
|
-
"gitHead": "
|
|
86
|
+
"gitHead": "fac14d5b8671e2984cff9dddb6031f6ed5b1a372"
|
|
87
87
|
}
|