@crawlee/core 3.10.4-beta.9 → 3.10.5-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/statistics.d.ts +6 -0
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +3 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +5 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +6 -2
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/package.json +5 -5
- package/tsconfig.build.tsbuildinfo +1 -1
package/crawlers/statistics.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Log } from '@apify/log';
|
|
1
2
|
import { ErrorTracker } from './error_tracker';
|
|
2
3
|
import { Configuration } from '../configuration';
|
|
3
4
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
@@ -152,6 +153,11 @@ export interface StatisticsOptions {
|
|
|
152
153
|
* @default 'Statistics'
|
|
153
154
|
*/
|
|
154
155
|
logMessage?: string;
|
|
156
|
+
/**
|
|
157
|
+
* Parent logger instance, the statistics will create a child logger from this.
|
|
158
|
+
* @default crawler.log
|
|
159
|
+
*/
|
|
160
|
+
log?: Log;
|
|
155
161
|
/**
|
|
156
162
|
* Key value store instance to persist the statistics.
|
|
157
163
|
* If not provided, the default one will be used when capturing starts
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D;;GAEG;AACH,cAAM,GAAG;IACL,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,cAAc,CAAC,CAAS;IAEhC,GAAG;IAKH,MAAM;IAKN,UAAU;CAGb;AAWD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;OAEG;IACH,iBAAiB,EAAE,YAAY,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,SAAuC;IAChE,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAwC3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAc7B;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAS3B;;OAEG;IACH,SAAS;;;;;;;;;IAsBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG;IAQxC;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAe/C;;OAEG;cACa,oBAAoB;IA0CpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAU3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CA2BpC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}
|
package/crawlers/statistics.js
CHANGED
|
@@ -160,7 +160,7 @@ class Statistics {
|
|
|
160
160
|
enumerable: true,
|
|
161
161
|
configurable: true,
|
|
162
162
|
writable: true,
|
|
163
|
-
value:
|
|
163
|
+
value: void 0
|
|
164
164
|
});
|
|
165
165
|
Object.defineProperty(this, "instanceStart", {
|
|
166
166
|
enumerable: true,
|
|
@@ -189,6 +189,7 @@ class Statistics {
|
|
|
189
189
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
190
190
|
logIntervalSecs: ow_1.default.optional.number,
|
|
191
191
|
logMessage: ow_1.default.optional.string,
|
|
192
|
+
log: ow_1.default.optional.object,
|
|
192
193
|
keyValueStore: ow_1.default.optional.object,
|
|
193
194
|
config: ow_1.default.optional.object,
|
|
194
195
|
persistenceOptions: ow_1.default.optional.object,
|
|
@@ -197,6 +198,7 @@ class Statistics {
|
|
|
197
198
|
const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = configuration_1.Configuration.getGlobalConfig(), persistenceOptions = {
|
|
198
199
|
enable: true,
|
|
199
200
|
}, saveErrorSnapshots = false, } = options;
|
|
201
|
+
this.log = (options.log ?? log_1.log).child({ prefix: 'Statistics' });
|
|
200
202
|
this.errorTracker = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
201
203
|
this.errorTrackerRetry = new error_tracker_1.ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
|
|
202
204
|
this.logIntervalMillis = logIntervalSecs * 1000;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AAEpB,mDAA+C;AAC/C,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAE5D;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;mBAAO,CAAC;WAAC;QACT;;;;;WAAwB;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;;WAA2B;QAE3B;;WAEG;QACH;;;;;WAAgC;QAEhC;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;QAE7B;;;;mBAAgC,SAAS;WAAC;QAC1C;;;;mBAAkB,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QACxD;;;;;WAA0B;QAC1B;;;;;WAAmB;QACnB;;;;;WAA8B;QAC9B;;;;mBAAqB,IAAI,GAAG,EAAwB;WAAC;QAC5C;;;;mBAAM,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;WAAC;QAC1D;;;;;WAAuB;QACvB;;;;;WAAqB;QACrB;;;;;WAAqB;QACrB;;;;;WAAuC;QAM3C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SAC1C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,GAC7B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AAzWL,gCA0WC;AAzWkB;;;;WAAK,CAAC;EAAJ,CAAK"}
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AACA,oDAAoB;AAEpB,mDAA+C;AAC/C,oDAAiD;AAGjD,gCAA2C;AAC3C,iEAA4D;AAE5D;;GAEG;AACH,MAAM,GAAG;IAAT;QACY;;;;mBAA2B,IAAI;WAAC;QAChC;;;;mBAAO,CAAC;WAAC;QACT;;;;;WAAwB;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6CnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7C3C;;WAEG;QACH;;;;;WAA2B;QAE3B;;WAEG;QACH;;;;;WAAgC;QAEhC;;WAEG;QACM;;;;mBAAK,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACM;;;;mBAAkC,EAAE;WAAC;QAE9C;;WAEG;QACc;;;;;WAAsB;QAE7B;;;;mBAAgC,SAAS;WAAC;QAC1C;;;;mBAAkB,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QACxD;;;;;WAA0B;QAC1B;;;;;WAAmB;QACnB;;;;;WAA8B;QAC9B;;;;mBAAqB,IAAI,GAAG,EAAwB;WAAC;QAC5C;;;;;WAAS;QAClB;;;;;WAAuB;QACvB;;;;;WAAqB;QACrB;;;;;WAAqB;QACrB;;;;;WAAuC;QAM3C,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SAC1C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,GAC7B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,SAAU,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,4BAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAC;QAE/E,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AA3WL,gCA4WC;AA3WkB;;;;WAAK,CAAC;EAAJ,CAAK"}
|
|
@@ -124,6 +124,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
124
124
|
* @default EnqueueStrategy.SameHostname
|
|
125
125
|
*/
|
|
126
126
|
strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
|
|
127
|
+
/**
|
|
128
|
+
* By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
|
|
129
|
+
* You can use this option to wait for adding all of them.
|
|
130
|
+
*/
|
|
131
|
+
waitForAllRequestsToBeAdded?: boolean;
|
|
127
132
|
}
|
|
128
133
|
/**
|
|
129
134
|
* The different enqueueing strategies available.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1B,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IAE9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IAEhD;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAElC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1B,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;IAE9B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;IAEhD;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAElC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,GACnE,OAAO,CAAC,sBAAsB,CAAC,CA0JjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -105,8 +105,9 @@ async function enqueueLinks(options) {
|
|
|
105
105
|
regexps: ow_1.default.optional.array.ofType(ow_1.default.any(ow_1.default.regExp, ow_1.default.object.hasKeys('regexp'))),
|
|
106
106
|
transformRequestFunction: ow_1.default.optional.function,
|
|
107
107
|
strategy: ow_1.default.optional.string.oneOf(Object.values(EnqueueStrategy)),
|
|
108
|
+
waitForAllRequestsToBeAdded: ow_1.default.optional.boolean,
|
|
108
109
|
}));
|
|
109
|
-
const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront } = options;
|
|
110
|
+
const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront, waitForAllRequestsToBeAdded, } = options;
|
|
110
111
|
const urlExcludePatternObjects = [];
|
|
111
112
|
const urlPatternObjects = [];
|
|
112
113
|
if (exclude?.length) {
|
|
@@ -187,7 +188,10 @@ async function enqueueLinks(options) {
|
|
|
187
188
|
let requests = createFilteredRequests();
|
|
188
189
|
if (limit)
|
|
189
190
|
requests = requests.slice(0, limit);
|
|
190
|
-
const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
|
|
191
|
+
const { addedRequests } = await requestQueue.addRequestsBatched(requests, {
|
|
192
|
+
forefront,
|
|
193
|
+
waitForAllRequestsToBeAdded,
|
|
194
|
+
});
|
|
191
195
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
192
196
|
}
|
|
193
197
|
exports.enqueueLinks = enqueueLinks;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAE7B,oDAAoB;AACpB,iCAAkC;AAIlC,qCAOkB;
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAE7B,oDAAoB;AACpB,iCAAkC;AAIlC,qCAOkB;AAmJlB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,+BAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAC9B,OAAkE;IAElE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,+GAA+G;SAClH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,GAC9B,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IAChD,CAAC;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAA,uBAAc,EAAC,cAAc,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/G,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,IAAA,uBAAc,EACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,+EAA+E;QAC/E,OAAO,IAAA,iCAAwB,EAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AA5JD,oCA4JC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.10.
|
|
3
|
+
"version": "3.10.5-beta.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.10.
|
|
63
|
-
"@crawlee/types": "3.10.
|
|
64
|
-
"@crawlee/utils": "3.10.
|
|
62
|
+
"@crawlee/memory-storage": "^3.10.5-beta.0",
|
|
63
|
+
"@crawlee/types": "^3.10.5-beta.0",
|
|
64
|
+
"@crawlee/utils": "^3.10.5-beta.0",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@types/tough-cookie": "^4.0.2",
|
|
67
67
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "c2c47db87da4748ff81f7fd1f790a8150a041471"
|
|
89
89
|
}
|