@crawlee/core 3.13.3 → 3.13.4-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +10 -1
- package/autoscaling/snapshotter.js.map +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +1 -5
- package/events/local_event_manager.js.map +1 -1
- package/package.json +5 -5
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAMlD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAMlD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkC5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IA2BvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EACpF,GAAG,EAAE,IAAI;CAUhB"}
|
|
@@ -183,7 +183,16 @@ class Snapshotter {
|
|
|
183
183
|
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
184
184
|
}
|
|
185
185
|
else {
|
|
186
|
-
|
|
186
|
+
let totalBytes;
|
|
187
|
+
if (this.config.get('systemInfoV2')) {
|
|
188
|
+
const containerized = this.config.get('containerized', await (0, utils_1.isContainerized)());
|
|
189
|
+
const memInfo = await (0, utils_1.getMemoryInfoV2)(containerized);
|
|
190
|
+
totalBytes = memInfo.totalBytes;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
const memInfo = await (0, utils_1.getMemoryInfo)();
|
|
194
|
+
totalBytes = memInfo.totalBytes;
|
|
195
|
+
}
|
|
187
196
|
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
|
|
188
197
|
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
189
198
|
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,0CAAiF;AACjF,oDAAoB;AAIpB,gDAA0E;AAE1E,oDAAiD;AAGjD,gCAA2C;AAG3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA4ElD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,IAAI,UAAkB,CAAC;YAEvB,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,IAAA,uBAAe,GAAE,CAAC,CAAC;gBAChF,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAe,EAAC,aAAa,CAAC,CAAC;gBACrD,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,MAAM,OAAO,GAAG,MAAM,IAAA,qBAAa,GAAE,CAAC;gBACtC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YACpC,CAAC;YAED,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;YACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO;gBACrF,sGAAsG,CAC7G,CAAC;QACN,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EACtC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAClC,IAAI,CAAC,+BAA+B,CACvC,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,MAAM;YACV,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IACI,IAAI,CAAC,kCAAkC;YACvC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAE3F,OAAO;QAEX,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,mCAAmC;gBAC/B,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CACzC,IAAI,CAAC,cAAe,CACvB,QAAQ,cAAc,2CAA2C,CACzE,CAAC;YACF,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CACrB,SAAoF,EACpF,GAAS;QAET,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;QACf,CAAC;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;CACJ;AA1TD,kCA0TC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAa,MAAM,iBAAiB,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,aAAa,CAAyB;IAE9C;;;OAGG;IACY,IAAI;IAYnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAI5B,OAAO,CAAC,kBAAkB;IAc1B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YAoBb,gBAAgB;CAiBjC"}
|
|
@@ -54,11 +54,7 @@ class LocalEventManager extends event_manager_1.EventManager {
|
|
|
54
54
|
* @internal
|
|
55
55
|
*/
|
|
56
56
|
async isContainerizedWrapper() {
|
|
57
|
-
|
|
58
|
-
if (config !== undefined) {
|
|
59
|
-
return config;
|
|
60
|
-
}
|
|
61
|
-
return (0, utils_1.isContainerized)();
|
|
57
|
+
return this.config.get('containerized', await (0, utils_1.isContainerized)());
|
|
62
58
|
}
|
|
63
59
|
getCurrentCpuTicks() {
|
|
64
60
|
const cpus = node_os_1.default.cpus();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AAEzB,0CAAuG;AAEvG,6DAA6B;AAC7B,gDAA0E;AAG1E,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACY;;;;mBAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":";;;;AAAA,8DAAyB;AAEzB,0CAAuG;AAEvG,6DAA6B;AAC7B,gDAA0E;AAG1E,mDAA0D;AAE1D,MAAa,iBAAkB,SAAQ,4BAAY;IAAnD;;QACY;;;;mBAAgB,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;WAAC;IA8GlD,CAAC;IA5GG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,MAAM,wBAAwB,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAE,CAAC;QAC9E,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACjH,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,IAAA,uBAAe,GAAE,CAAC,CAAC;IACrE,CAAC;IAEO,kBAAkB;QACtB,MAAM,IAAI,GAAG,iBAAE,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,MAAM,CACd,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI;gBAC/B,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC;aAC9D,CAAC;QACN,CAAC,EACD,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,YAAY,GAAG,MAAM,IAAA,4BAAoB,EAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACrF,OAAO;gBACH,eAAe,EAAE,YAAY,GAAG,GAAG;gBACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;aAC1D,CAAC;QACN,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,aAAc,CAAC,IAAI,CAAC;QAC7D,MAAM,eAAe,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC;QAChE,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAChF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAEzC,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAe,EAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;gBAC3E,OAAO;oBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;iBAC1E,CAAC;YACN,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAA,qBAAa,GAAE,CAAC;YACtC,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,aAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ;AA/GD,8CA+GC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.13.
|
|
3
|
+
"version": "3.13.4-beta.1",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.13.
|
|
63
|
-
"@crawlee/types": "3.13.
|
|
64
|
-
"@crawlee/utils": "3.13.
|
|
62
|
+
"@crawlee/memory-storage": "3.13.4-beta.1",
|
|
63
|
+
"@crawlee/types": "3.13.4-beta.1",
|
|
64
|
+
"@crawlee/utils": "3.13.4-beta.1",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
67
67
|
"csv-stringify": "^6.2.0",
|
|
@@ -83,5 +83,5 @@
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
},
|
|
86
|
-
"gitHead": "
|
|
86
|
+
"gitHead": "78ec3cacb6edf5948d460b04c6e63f0fad31205f"
|
|
87
87
|
}
|