@crawlee/core 4.0.0-beta.60 → 4.0.0-beta.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -7
- package/crawlers/statistics.js.map +1 -1
- package/package.json +5 -5
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +1 -7
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +33 -34
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +74 -113
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +37 -25
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +82 -51
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_provider.d.ts +12 -0
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +20 -1
- package/storages/request_provider.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +1 -0
- package/storages/storage_instance_manager.d.ts.map +1 -1
- package/storages/storage_instance_manager.js +25 -0
- package/storages/storage_instance_manager.js.map +1 -1
- package/storages/utils.d.ts +22 -0
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +51 -0
- package/storages/utils.js.map +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AA4BlD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;OAEG;IACH,iBAAiB,EAAE,YAAY,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAgB;IACpC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAe;IAC/B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C,OAAO,KAAK,MAAM,GAKjB;IAED;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAyC3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAcjD;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAS/C;;OAEG;IACH,SAAS;;;;;;;;;IAsBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM;IAMlD;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AA4BlD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAC;IAE3B;;OAEG;IACH,iBAAiB,EAAE,YAAY,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAa;IACpD,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAgB;IACpC,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,OAAO,CAAC,CAAe;IAC/B,OAAO,CAAC,kBAAkB,CAAqB;IAE/C,OAAO,KAAK,MAAM,GAKjB;IAED;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAyC3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAY7C;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAcjD;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM;IAS/C;;OAEG;IACH,SAAS;;;;;;;;;IAsBT;;;OAGG;IACG,cAAc;IAoBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM;IAMlD;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAmB/C;;OAEG;cACa,oBAAoB;IA0CpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAY3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CA2BpC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAC9B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAE7B;;;;;;;OAOG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}
|
package/crawlers/statistics.js
CHANGED
|
@@ -259,14 +259,8 @@ export class Statistics {
|
|
|
259
259
|
return;
|
|
260
260
|
}
|
|
261
261
|
this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
|
|
262
|
-
// use half the interval of `persistState` to avoid race conditions
|
|
263
|
-
const persistStateIntervalMillis = serviceLocator.getConfiguration().persistStateIntervalMillis;
|
|
264
|
-
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
265
262
|
await this.keyValueStore
|
|
266
|
-
.setValue(this.persistStateKey, this.toJSON()
|
|
267
|
-
timeoutSecs,
|
|
268
|
-
doNotRetryTimeouts: true,
|
|
269
|
-
})
|
|
263
|
+
.setValue(this.persistStateKey, this.toJSON())
|
|
270
264
|
.catch((error) => this.log.warning(`Failed to persist the statistics to ${this.persistStateKey}`, { error }));
|
|
271
265
|
}
|
|
272
266
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;GAEG;AACH,MAAM,GAAG;IACG,SAAS,GAAkB,IAAI,CAAC;IAChC,cAAc,CAAU;IAEhC,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAM,OAAO,UAAU;IACX,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAEtB;;OAEG;IACH,YAAY,CAAe;IAE3B;;OAEG;IACH,iBAAiB,CAAe;IAEhC;;OAEG;IACM,EAAE,CAAS;IAEpB;;OAEG;IACH,KAAK,CAAkB;IAEvB;;OAEG;IACM,qBAAqB,GAAa,EAAE,CAAC;IAEpC,aAAa,GAAmB,SAAS,CAAC;IAC1C,eAAe,CAAS;IAC1B,iBAAiB,CAAS;IAC1B,UAAU,CAAS;IACnB,QAAQ,CAAsB;IAC9B,kBAAkB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5C,GAAG,CAAgB;IAC5B,aAAa,CAAU;IACvB,WAAW,CAAU;IACrB,OAAO,CAAgB;IACvB,kBAAkB,CAAqB;IAE/C,IAAY,MAAM;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,YAAY,UAA6B,EAAE;QACvC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YACvC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;SAC5C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,EAC1B,EAAE,GACL,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,0BAA0B,IAAI,CAAC,EAAE,EAAE,CAAC;QAE3D,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB,EAAE,UAAkB;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB,EAAE,UAAkB;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAK,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,UAAkB;QAC9C,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,0BAA0B,CAAC;QAChG,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE;YAC3C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uCAAuC,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7F,CAAC;IACV,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,sFAAsF;QACtF,sFAAsF;QACtF,IAAI,CAAC,OAAO,EAAE,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC"}
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD;;GAEG;AACH,MAAM,GAAG;IACG,SAAS,GAAkB,IAAI,CAAC;IAChC,cAAc,CAAU;IAEhC,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAaF;;;;;;;;;GASG;AACH,MAAM,OAAO,UAAU;IACX,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAEtB;;OAEG;IACH,YAAY,CAAe;IAE3B;;OAEG;IACH,iBAAiB,CAAe;IAEhC;;OAEG;IACM,EAAE,CAAS;IAEpB;;OAEG;IACH,KAAK,CAAkB;IAEvB;;OAEG;IACM,qBAAqB,GAAa,EAAE,CAAC;IAEpC,aAAa,GAAmB,SAAS,CAAC;IAC1C,eAAe,CAAS;IAC1B,iBAAiB,CAAS;IAC1B,UAAU,CAAS;IACnB,QAAQ,CAAsB;IAC9B,kBAAkB,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5C,GAAG,CAAgB;IAC5B,aAAa,CAAU;IACvB,WAAW,CAAU;IACrB,OAAO,CAAgB;IACvB,kBAAkB,CAAqB;IAE/C,IAAY,MAAM;QACd,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,YAAY,UAA6B,EAAE;QACvC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YACvC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;SAC5C,CAAC,CACL,CAAC;QAEF,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,kBAAkB,GAAG,KAAK,EAC1B,EAAE,GACL,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,0BAA0B,IAAI,CAAC,EAAE,EAAE,CAAC;QAE3D,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,iBAAiB,GAAG,IAAI,YAAY,CAAC,EAAE,GAAG,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB,EAAE,UAAkB;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YACvD,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QAC5D,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB,EAAE,UAAkB;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACvC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAC5B,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YACjF,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAK,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7C,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,UAAkB;QAC9C,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;aAC7C,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uCAAuC,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC7F,CAAC;IACV,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,8EAA8E;QAC9E,0EAA0E;QAC1E,UAAU,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,sFAAsF;QACtF,sFAAsF;QACtF,IAAI,CAAC,OAAO,EAAE,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB;gBAC3C,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE;gBACtD,CAAC,CAAC,IAAI;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.62",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.62",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.62",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.62",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
@@ -76,5 +76,5 @@
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
},
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "74568b7b82e943056f6b22675d94c2ce12c198a5"
|
|
80
80
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAI/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,QAAA,MAAM,wBAAwB,yDAA0D,CAAC;AACzF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,CAAC,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/E;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAE5C,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,qBAAa,WAAY,YAAW,YAAY;IAC5C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAK;IAE1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,CAAC;IAErD,OAAO,CAAC,WAAW,CAAC,CAAgB;IACpC,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,GAAE,kBAAuB;IAqD5C;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK7C;;;OAGG;cACa,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAOpC,SAAS;IAyBvB;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;;OAMG;IACG,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;IAyBlE;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAS7C;;;OAGG;IACG,QAAQ;;;;;IASd;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAI/D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,QAAA,MAAM,wBAAwB,yDAA0D,CAAC;AACzF,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,CAAC,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/E;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAE5C,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,qBAAa,WAAY,YAAW,YAAY;IAC5C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAK;IAE1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,oBAAoB,EAAE,oBAAoB,CAAC;IAErD,OAAO,CAAC,WAAW,CAAC,CAAgB;IACpC,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,eAAe,CAAK;gBAEhB,OAAO,GAAE,kBAAuB;IAqD5C;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK5C;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAK7C;;;OAGG;cACa,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;YAOpC,SAAS;IAyBvB;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;;OAMG;IACG,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC;IAyBlE;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAS7C;;;OAGG;IACG,QAAQ;;;;;IASd;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;OAKG;cACa,6BAA6B,CAAC,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAOlH;;;OAGG;YACW,4BAA4B;IAK1C;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO,GAAG,SAAS;IAmB7C;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;CAuBzD"}
|
|
@@ -250,14 +250,8 @@ export class SessionPool {
|
|
|
250
250
|
persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
|
|
251
251
|
persistStateKey: this.persistStateKey,
|
|
252
252
|
});
|
|
253
|
-
// use half the interval of `persistState` to avoid race conditions
|
|
254
|
-
const persistStateIntervalMillis = serviceLocator.getConfiguration().persistStateIntervalMillis;
|
|
255
|
-
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
256
253
|
await this.keyValueStore
|
|
257
|
-
?.setValue(this.persistStateKey, await this.getState()
|
|
258
|
-
timeoutSecs,
|
|
259
|
-
doNotRetryTimeouts: true,
|
|
260
|
-
})
|
|
254
|
+
?.setValue(this.persistStateKey, await this.getState())
|
|
261
255
|
.catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
|
|
262
256
|
}
|
|
263
257
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAMpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,mBAAmB,CAAU,CAAC;AAgEzF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,MAAM,OAAO,WAAW;IACZ,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAEjB,EAAE,CAAS;IACV,GAAG,CAAgB;IACnB,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACrB,kBAAkB,CAAqB;IACvC,oBAAoB,CAAuB;IAE7C,WAAW,CAAiB;IAC5B,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IACzB,eAAe,GAAG,CAAC,CAAC;IAE5B,YAAY,UAA8B,EAAE;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;YACzC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,wBAAwB,CAAC,CAAC;SAChF,CAAC,CACL,CAAC;QAEF,MAAM,EACF,EAAE,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,EACf,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,EAChC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,oBAAoB,GAAG,QAAQ,GAClC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAC/C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,8FAA8F;QAC9F,qGAAqG;QACrG,2BAA2B;QAC3B,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,GAAG,iBAAiB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACrB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QACtB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CACzC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,IAAI,EAClF;YACI,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE;SAC5C,CACJ,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;QAC3G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,cAA+B;QAC5C,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,cAAc,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7B,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,EAAE,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBACxC,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa;gBAAE,OAAO,aAAa,CAAC;YAExC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO;YACH,mBAAmB,EAAE,MAAM,IAAI,CAAC,mBAAmB,EAAE;YACrD,oBAAoB,EAAE,MAAM,IAAI,CAAC,oBAAoB,EAAE;YACvD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAMpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,wBAAwB,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,mBAAmB,CAAU,CAAC;AAgEzF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACH,MAAM,OAAO,WAAW;IACZ,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAEjB,EAAE,CAAS;IACV,GAAG,CAAgB;IACnB,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACrB,kBAAkB,CAAqB;IACvC,oBAAoB,CAAuB;IAE7C,WAAW,CAAiB;IAC5B,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IACzB,eAAe,GAAG,CAAC,CAAC;IAE5B,YAAY,UAA8B,EAAE;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC;YACzC,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,wBAAwB,CAAC,CAAC;SAChF,CAAC,CACL,CAAC;QAEF,MAAM,EACF,EAAE,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,EACf,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,EAChC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,EACD,oBAAoB,GAAG,QAAQ,GAClC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAC/C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,8FAA8F;QAC9F,qGAAqG;QACrG,2BAA2B;QAC3B,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,GAAG,iBAAiB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACrB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QACtB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CACzC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,IAAI,EAClF;YACI,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE;SAC5C,CACJ,CAAC;QAEF,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;QAC3G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,cAA+B;QAC5C,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,cAAc,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7B,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC;YACD,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,EAAE,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBACxC,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa;gBAAE,OAAO,aAAa,CAAC;YAExC,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO;YACH,mBAAmB,EAAE,MAAM,IAAI,CAAC,mBAAmB,EAAE;YACrD,oBAAoB,EAAE,MAAM,IAAI,CAAC,oBAAoB,EAAE;YACvD,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,aAAa;YACpB,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;aACtD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,+CAA+C,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CACrG,CAAC;IACV,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO;QAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,6BAA6B,CAAC,UAA+C,EAAE;QAC3F,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAExC,OAAO,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,4BAA4B,CAAC,cAA+B;QACtE,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,OAAO,IAAI,CAAC,qBAAqB,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,4BAA4B,EAAE,CAAC;QAC7D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,IAAI,IAAI,CAAC,oBAAoB,KAAK,mBAAmB,IAAI,IAAI,CAAC,mBAAmB,EAAE;YAAE,OAAO,SAAS,CAAC;QAEtG,IAAI,IAAI,CAAC,oBAAoB,KAAK,mBAAmB,EAAE,CAAC;YACpD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,MAAe,CAAC;QACpB,IAAI,IAAI,CAAC,oBAAoB,KAAK,aAAa,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC1D,IAAI,CAAC,eAAe,GAAG,KAAK,GAAG,CAAC,CAAC;YACjC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE/G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,aAAa,CAAC,CAAC;YAEhF,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,4CAA4C,CAAC,CAAC;IACxF,CAAC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DatasetClient, DatasetInfo, Dictionary
|
|
1
|
+
import type { DatasetClient, DatasetInfo, Dictionary } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { CrawleeLogger } from '../log.js';
|
|
4
4
|
import type { Awaitable } from '../typedefs.js';
|
|
@@ -7,22 +7,14 @@ import type { StorageOpenOptions } from './utils.js';
|
|
|
7
7
|
/** @internal */
|
|
8
8
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* in an array to provide better error messages. Returns serialized object.
|
|
13
|
-
* @ignore
|
|
14
|
-
*/
|
|
15
|
-
export declare function checkAndSerialize<T>(item: T, limitBytes: number, index?: number): string;
|
|
16
|
-
/**
|
|
17
|
-
* Takes an array of JSONs (payloads) as input and produces an array of JSON strings
|
|
18
|
-
* where each string is a JSON array of payloads with a maximum size of limitBytes per one
|
|
19
|
-
* JSON array. Fits as many payloads as possible into a single JSON array and then moves
|
|
20
|
-
* on to the next, preserving item order.
|
|
10
|
+
* Validates that the given value is a plain JSON-serializable object
|
|
11
|
+
* (not an array, not a primitive, not circular).
|
|
21
12
|
*
|
|
22
|
-
*
|
|
13
|
+
* @param item The value to validate.
|
|
14
|
+
* @param index Optional index for error messages when validating inside an array.
|
|
23
15
|
* @ignore
|
|
24
16
|
*/
|
|
25
|
-
export declare function
|
|
17
|
+
export declare function assertJsonSerializable<T>(item: T, index?: number): void;
|
|
26
18
|
export interface DatasetDataOptions {
|
|
27
19
|
/**
|
|
28
20
|
* Number of array elements that should be skipped at the start.
|
|
@@ -164,21 +156,8 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
164
156
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
165
157
|
* otherwise the crawler process might finish before the data is stored!
|
|
166
158
|
*
|
|
167
|
-
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
168
|
-
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
169
|
-
* none of the included objects
|
|
170
|
-
* may be larger than 9MB, but the array itself may be of any size.
|
|
171
|
-
*
|
|
172
|
-
* The function internally
|
|
173
|
-
* chunks the array into separate items and pushes them sequentially.
|
|
174
|
-
* The chunking process is stable (keeps order of data), but it does not provide a transaction
|
|
175
|
-
* safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
|
|
176
|
-
* the function's Promise will reject and the dataset will be left in a state where some of
|
|
177
|
-
* the items have already been saved to the dataset while other items from the source array were not.
|
|
178
|
-
* To overcome this limitation, the developer may, for example, read the last item saved in the dataset
|
|
179
|
-
* and re-attempt the save of the data from this item onwards to prevent duplicates.
|
|
180
159
|
* @param data Object or array of objects containing data to be stored in the default dataset.
|
|
181
|
-
* The objects must be serializable to JSON
|
|
160
|
+
* The objects must be serializable to JSON.
|
|
182
161
|
*/
|
|
183
162
|
pushData(data: Data | Data[]): Promise<void>;
|
|
184
163
|
/**
|
|
@@ -327,36 +306,56 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
327
306
|
* @param [options] An object containing extra options for `reduce()`
|
|
328
307
|
*/
|
|
329
308
|
reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
|
|
309
|
+
private fetchEntryPages;
|
|
310
|
+
private fetchPages;
|
|
330
311
|
/**
|
|
331
|
-
*
|
|
332
|
-
*
|
|
312
|
+
* Returns dataset items.
|
|
313
|
+
*
|
|
314
|
+
* When awaited (`await dataset.values()`), returns all items as a flat `Data[]` array.
|
|
315
|
+
* When used as an async iterable (`for await...of`), iterates over all items across pages
|
|
316
|
+
* without loading everything into memory at once.
|
|
333
317
|
*
|
|
334
318
|
* **Example usage:**
|
|
335
319
|
* ```javascript
|
|
336
320
|
* const dataset = await Dataset.open('my-results');
|
|
321
|
+
*
|
|
322
|
+
* // Iterate over all items (memory-efficient for large datasets)
|
|
337
323
|
* for await (const item of dataset.values()) {
|
|
338
324
|
* console.log(item);
|
|
339
325
|
* }
|
|
326
|
+
*
|
|
327
|
+
* // Or fetch all items at once
|
|
328
|
+
* const items = await dataset.values();
|
|
329
|
+
* console.log(items);
|
|
340
330
|
* ```
|
|
341
331
|
*
|
|
342
332
|
* @param options Options for the iteration.
|
|
343
333
|
*/
|
|
344
|
-
values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<
|
|
334
|
+
values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
|
|
345
335
|
/**
|
|
346
|
-
*
|
|
347
|
-
*
|
|
336
|
+
* Returns dataset entries (index-value pairs).
|
|
337
|
+
*
|
|
338
|
+
* When awaited (`await dataset.entries()`), returns all entries as a flat `[index, item][]` array.
|
|
339
|
+
* When used as an async iterable (`for await...of`), iterates over all entries across pages
|
|
340
|
+
* without loading everything into memory at once.
|
|
348
341
|
*
|
|
349
342
|
* **Example usage:**
|
|
350
343
|
* ```javascript
|
|
351
344
|
* const dataset = await Dataset.open('my-results');
|
|
345
|
+
*
|
|
346
|
+
* // Iterate over all entries
|
|
352
347
|
* for await (const [index, item] of dataset.entries()) {
|
|
353
348
|
* console.log(`Item at ${index}: ${JSON.stringify(item)}`);
|
|
354
349
|
* }
|
|
350
|
+
*
|
|
351
|
+
* // Or fetch all at once
|
|
352
|
+
* const entries = await dataset.entries();
|
|
353
|
+
* console.log(entries);
|
|
355
354
|
* ```
|
|
356
355
|
*
|
|
357
356
|
* @param options Options for the iteration.
|
|
358
357
|
*/
|
|
359
|
-
entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<
|
|
358
|
+
entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
|
|
360
359
|
/**
|
|
361
360
|
* Default async iterator for the dataset, iterating over items.
|
|
362
361
|
* Allows using the dataset directly in a `for await...of` loop.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAiB,MAAM,gBAAgB,CAAC;AAI5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAErD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAcvE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtF;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAChD,kBAAkB,EAClB,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAC5D;IACG,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;IACzC,KAAK,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAWjD,QAAQ,CAAC,MAAM;IAVnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,EAAE,aAAa,CAAC;IAEnB;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAQrD;;;;;;;;;;OAUG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAclD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAYjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAiCpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;OAgBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;YAyB1F,eAAe;YAUf,UAAU;IAqBzB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,OAAO,GAAE,sBAA2B,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IASnF;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,OAAO,GAAE,sBAA2B,GAAG,aAAa,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;IASxG;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC;IAItE;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;;;;;;;;;;;;;OAcG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IA0BzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
package/storages/dataset.js
CHANGED
|
@@ -1,80 +1,34 @@
|
|
|
1
1
|
import { stringify } from 'csv-stringify/sync';
|
|
2
2
|
import ow from 'ow';
|
|
3
|
-
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
|
|
4
3
|
import { Configuration } from '../configuration.js';
|
|
5
4
|
import { serviceLocator } from '../service_locator.js';
|
|
6
5
|
import { checkStorageAccess } from './access_checking.js';
|
|
7
6
|
import { KeyValueStore } from './key_value_store.js';
|
|
8
7
|
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
9
|
-
import { purgeDefaultStorages } from './utils.js';
|
|
8
|
+
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
10
9
|
/** @internal */
|
|
11
10
|
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
12
|
-
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
13
11
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
12
|
+
* Validates that the given value is a plain JSON-serializable object
|
|
13
|
+
* (not an array, not a primitive, not circular).
|
|
14
|
+
*
|
|
15
|
+
* @param item The value to validate.
|
|
16
|
+
* @param index Optional index for error messages when validating inside an array.
|
|
17
17
|
* @ignore
|
|
18
18
|
*/
|
|
19
|
-
export function
|
|
19
|
+
export function assertJsonSerializable(item, index) {
|
|
20
20
|
const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
|
|
21
21
|
const isItemObject = item && typeof item === 'object' && !Array.isArray(item);
|
|
22
22
|
if (!isItemObject) {
|
|
23
23
|
throw new Error(`Data item${s}is not an object. You can push only objects into a dataset.`);
|
|
24
24
|
}
|
|
25
|
-
let payload;
|
|
26
25
|
try {
|
|
27
|
-
|
|
26
|
+
JSON.stringify(item);
|
|
28
27
|
}
|
|
29
28
|
catch (e) {
|
|
30
29
|
const err = e;
|
|
31
30
|
throw new Error(`Data item${s}is not serializable to JSON.\nCause: ${err.message}`);
|
|
32
31
|
}
|
|
33
|
-
const bytes = Buffer.byteLength(payload);
|
|
34
|
-
if (bytes > limitBytes) {
|
|
35
|
-
throw new Error(`Data item${s}is too large (size: ${bytes} bytes, limit: ${limitBytes} bytes)`);
|
|
36
|
-
}
|
|
37
|
-
return payload;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Takes an array of JSONs (payloads) as input and produces an array of JSON strings
|
|
41
|
-
* where each string is a JSON array of payloads with a maximum size of limitBytes per one
|
|
42
|
-
* JSON array. Fits as many payloads as possible into a single JSON array and then moves
|
|
43
|
-
* on to the next, preserving item order.
|
|
44
|
-
*
|
|
45
|
-
* The function assumes that none of the items is larger than limitBytes and does not validate.
|
|
46
|
-
* @ignore
|
|
47
|
-
*/
|
|
48
|
-
export function chunkBySize(items, limitBytes) {
|
|
49
|
-
if (!items.length)
|
|
50
|
-
return [];
|
|
51
|
-
if (items.length === 1)
|
|
52
|
-
return items;
|
|
53
|
-
// Split payloads into buckets of valid size.
|
|
54
|
-
let lastChunkBytes = 2; // Add 2 bytes for [] wrapper.
|
|
55
|
-
const chunks = [];
|
|
56
|
-
for (const payload of items) {
|
|
57
|
-
const bytes = Buffer.byteLength(payload);
|
|
58
|
-
if (bytes <= limitBytes && bytes + 2 > limitBytes) {
|
|
59
|
-
// Handle cases where wrapping with [] would fail, but solo object is fine.
|
|
60
|
-
chunks.push(payload);
|
|
61
|
-
lastChunkBytes = bytes;
|
|
62
|
-
}
|
|
63
|
-
else if (lastChunkBytes + bytes <= limitBytes) {
|
|
64
|
-
// ensure array
|
|
65
|
-
if (!Array.isArray(chunks[chunks.length - 1])) {
|
|
66
|
-
chunks.push([]);
|
|
67
|
-
}
|
|
68
|
-
chunks[chunks.length - 1].push(payload);
|
|
69
|
-
lastChunkBytes += bytes + 1; // Add 1 byte for ',' separator.
|
|
70
|
-
}
|
|
71
|
-
else {
|
|
72
|
-
chunks.push([payload]);
|
|
73
|
-
lastChunkBytes = bytes + 2; // Add 2 bytes for [] wrapper.
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
// Stringify array chunks.
|
|
77
|
-
return chunks.map((chunk) => (typeof chunk === 'string' ? chunk : `[${chunk.join(',')}]`));
|
|
78
32
|
}
|
|
79
33
|
/**
|
|
80
34
|
* The `Dataset` class represents a store for structured data where each object stored has the same attributes,
|
|
@@ -151,40 +105,18 @@ export class Dataset {
|
|
|
151
105
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
152
106
|
* otherwise the crawler process might finish before the data is stored!
|
|
153
107
|
*
|
|
154
|
-
* The size of the data is limited by the receiving API and therefore `pushData()` will only
|
|
155
|
-
* allow objects whose JSON representation is smaller than 9MB. When an array is passed,
|
|
156
|
-
* none of the included objects
|
|
157
|
-
* may be larger than 9MB, but the array itself may be of any size.
|
|
158
|
-
*
|
|
159
|
-
* The function internally
|
|
160
|
-
* chunks the array into separate items and pushes them sequentially.
|
|
161
|
-
* The chunking process is stable (keeps order of data), but it does not provide a transaction
|
|
162
|
-
* safety mechanism. Therefore, in the event of an uploading error (after several automatic retries),
|
|
163
|
-
* the function's Promise will reject and the dataset will be left in a state where some of
|
|
164
|
-
* the items have already been saved to the dataset while other items from the source array were not.
|
|
165
|
-
* To overcome this limitation, the developer may, for example, read the last item saved in the dataset
|
|
166
|
-
* and re-attempt the save of the data from this item onwards to prevent duplicates.
|
|
167
108
|
* @param data Object or array of objects containing data to be stored in the default dataset.
|
|
168
|
-
* The objects must be serializable to JSON
|
|
109
|
+
* The objects must be serializable to JSON.
|
|
169
110
|
*/
|
|
170
111
|
async pushData(data) {
|
|
171
112
|
checkStorageAccess();
|
|
172
113
|
ow(data, 'data', ow.object);
|
|
173
|
-
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const payload = checkAndSerialize(data, limit);
|
|
178
|
-
await dispatch(payload);
|
|
179
|
-
return;
|
|
180
|
-
}
|
|
181
|
-
// Handle Arrays
|
|
182
|
-
const payloads = data.map((item, index) => checkAndSerialize(item, limit, index));
|
|
183
|
-
const chunks = chunkBySize(payloads, limit);
|
|
184
|
-
// Invoke client in series to preserve order of data
|
|
185
|
-
for (const chunk of chunks) {
|
|
186
|
-
await dispatch(chunk);
|
|
114
|
+
// Normalize to array and validate each item
|
|
115
|
+
const items = Array.isArray(data) ? data : [data];
|
|
116
|
+
for (let i = 0; i < items.length; i++) {
|
|
117
|
+
assertJsonSerializable(items[i], i);
|
|
187
118
|
}
|
|
119
|
+
await this.client.pushData(items);
|
|
188
120
|
}
|
|
189
121
|
/**
|
|
190
122
|
* Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
@@ -192,7 +124,7 @@ export class Dataset {
|
|
|
192
124
|
async getData(options = {}) {
|
|
193
125
|
checkStorageAccess();
|
|
194
126
|
try {
|
|
195
|
-
return await this.client.
|
|
127
|
+
return await this.client.getData(options);
|
|
196
128
|
}
|
|
197
129
|
catch (e) {
|
|
198
130
|
const error = e;
|
|
@@ -209,18 +141,9 @@ export class Dataset {
|
|
|
209
141
|
async export(options = {}) {
|
|
210
142
|
checkStorageAccess();
|
|
211
143
|
const items = [];
|
|
212
|
-
const
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if (value.count === 0) {
|
|
216
|
-
return;
|
|
217
|
-
}
|
|
218
|
-
items.push(...value.items);
|
|
219
|
-
if (value.total > offset + value.count) {
|
|
220
|
-
await fetchNextChunk(offset + value.count);
|
|
221
|
-
}
|
|
222
|
-
};
|
|
223
|
-
await fetchNextChunk();
|
|
144
|
+
for await (const page of this.fetchPages(options)) {
|
|
145
|
+
items.push(...page.items);
|
|
146
|
+
}
|
|
224
147
|
return items;
|
|
225
148
|
}
|
|
226
149
|
/**
|
|
@@ -390,52 +313,90 @@ export class Dataset {
|
|
|
390
313
|
await this.forEach(wrappedFunc, options);
|
|
391
314
|
return currentMemo;
|
|
392
315
|
}
|
|
316
|
+
async *fetchEntryPages(options) {
|
|
317
|
+
let index = options.offset ?? 0;
|
|
318
|
+
for await (const page of this.fetchPages(options)) {
|
|
319
|
+
yield {
|
|
320
|
+
...page,
|
|
321
|
+
items: page.items.map((item) => [index++, item]),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
async *fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
|
|
326
|
+
let offset = options.offset ?? 0;
|
|
327
|
+
const totalLimit = options.limit;
|
|
328
|
+
let yielded = 0;
|
|
329
|
+
while (true) {
|
|
330
|
+
const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
|
|
331
|
+
if (fetchLimit <= 0)
|
|
332
|
+
break;
|
|
333
|
+
const page = await this.client.getData({ ...options, offset, limit: fetchLimit });
|
|
334
|
+
yield page;
|
|
335
|
+
yielded += page.items.length;
|
|
336
|
+
if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
|
|
337
|
+
break;
|
|
338
|
+
offset += page.items.length;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
393
341
|
/**
|
|
394
|
-
*
|
|
395
|
-
*
|
|
342
|
+
* Returns dataset items.
|
|
343
|
+
*
|
|
344
|
+
* When awaited (`await dataset.values()`), returns all items as a flat `Data[]` array.
|
|
345
|
+
* When used as an async iterable (`for await...of`), iterates over all items across pages
|
|
346
|
+
* without loading everything into memory at once.
|
|
396
347
|
*
|
|
397
348
|
* **Example usage:**
|
|
398
349
|
* ```javascript
|
|
399
350
|
* const dataset = await Dataset.open('my-results');
|
|
351
|
+
*
|
|
352
|
+
* // Iterate over all items (memory-efficient for large datasets)
|
|
400
353
|
* for await (const item of dataset.values()) {
|
|
401
354
|
* console.log(item);
|
|
402
355
|
* }
|
|
356
|
+
*
|
|
357
|
+
* // Or fetch all items at once
|
|
358
|
+
* const items = await dataset.values();
|
|
359
|
+
* console.log(items);
|
|
403
360
|
* ```
|
|
404
361
|
*
|
|
405
362
|
* @param options Options for the iteration.
|
|
406
363
|
*/
|
|
407
364
|
values(options = {}) {
|
|
408
365
|
checkStorageAccess();
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
throw new Error('Resource client "listItems" method does not return an async iterable.');
|
|
414
|
-
},
|
|
415
|
-
});
|
|
416
|
-
}
|
|
417
|
-
return result;
|
|
366
|
+
return createDualIterable({
|
|
367
|
+
createPages: () => this.fetchPages(options),
|
|
368
|
+
extractItems: (page) => page.items,
|
|
369
|
+
});
|
|
418
370
|
}
|
|
419
371
|
/**
|
|
420
|
-
*
|
|
421
|
-
*
|
|
372
|
+
* Returns dataset entries (index-value pairs).
|
|
373
|
+
*
|
|
374
|
+
* When awaited (`await dataset.entries()`), returns all entries as a flat `[index, item][]` array.
|
|
375
|
+
* When used as an async iterable (`for await...of`), iterates over all entries across pages
|
|
376
|
+
* without loading everything into memory at once.
|
|
422
377
|
*
|
|
423
378
|
* **Example usage:**
|
|
424
379
|
* ```javascript
|
|
425
380
|
* const dataset = await Dataset.open('my-results');
|
|
381
|
+
*
|
|
382
|
+
* // Iterate over all entries
|
|
426
383
|
* for await (const [index, item] of dataset.entries()) {
|
|
427
384
|
* console.log(`Item at ${index}: ${JSON.stringify(item)}`);
|
|
428
385
|
* }
|
|
386
|
+
*
|
|
387
|
+
* // Or fetch all at once
|
|
388
|
+
* const entries = await dataset.entries();
|
|
389
|
+
* console.log(entries);
|
|
429
390
|
* ```
|
|
430
391
|
*
|
|
431
392
|
* @param options Options for the iteration.
|
|
432
393
|
*/
|
|
433
394
|
entries(options = {}) {
|
|
434
395
|
checkStorageAccess();
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
396
|
+
return createDualIterable({
|
|
397
|
+
createPages: () => this.fetchEntryPages(options),
|
|
398
|
+
extractItems: (page) => page.items,
|
|
399
|
+
});
|
|
439
400
|
}
|
|
440
401
|
/**
|
|
441
402
|
* Default async iterator for the dataset, iterating over items.
|
|
@@ -458,7 +419,7 @@ export class Dataset {
|
|
|
458
419
|
*/
|
|
459
420
|
async drop() {
|
|
460
421
|
checkStorageAccess();
|
|
461
|
-
await this.client.
|
|
422
|
+
await this.client.drop();
|
|
462
423
|
serviceLocator.getStorageInstanceManager().removeFromCache(this);
|
|
463
424
|
}
|
|
464
425
|
/**
|