@crawlee/core 4.0.0-beta.47 → 4.0.0-beta.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +3 -3
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +89 -158
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -130
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +4 -4
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -12
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/errors.d.ts.map +1 -1
- package/errors.js +2 -7
- package/errors.js.map +1 -1
- package/events/local_event_manager.js +4 -4
- package/events/local_event_manager.js.map +1 -1
- package/package.json +8 -8
- package/recoverable_state.d.ts.map +1 -1
- package/recoverable_state.js +8 -3
- package/recoverable_state.js.map +1 -1
- package/service_locator.js +1 -1
- package/service_locator.js.map +1 -1
- package/session_pool/session_pool.d.ts +16 -3
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +28 -10
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +13 -19
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -17
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +8 -9
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +9 -11
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_provider.d.ts +11 -15
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +25 -23
- package/storages/request_provider.js.map +1 -1
- package/storages/storage_manager.d.ts +15 -12
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +48 -32
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.js +2 -2
- package/storages/utils.js.map +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAIzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMrD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,aAAa,CAAC;IACnB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IAsC5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAIzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMrD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,aAAa,CAAC;IACnB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IAsC5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgC5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IA2BvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EACpF,GAAG,EAAE,IAAI;CAUhB"}
|
|
@@ -75,18 +75,18 @@ export class Snapshotter {
|
|
|
75
75
|
* Starts capturing snapshots at configured intervals.
|
|
76
76
|
*/
|
|
77
77
|
async start() {
|
|
78
|
-
const memoryMbytes = serviceLocator.getConfiguration().
|
|
78
|
+
const memoryMbytes = serviceLocator.getConfiguration().memoryMbytes ?? 0;
|
|
79
79
|
if (memoryMbytes > 0) {
|
|
80
80
|
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
81
81
|
}
|
|
82
82
|
else {
|
|
83
|
-
const containerized = serviceLocator.getConfiguration().
|
|
83
|
+
const containerized = serviceLocator.getConfiguration().containerized ?? (await isContainerized());
|
|
84
84
|
const memInfo = await getMemoryInfo({
|
|
85
85
|
containerized,
|
|
86
86
|
logger: serviceLocator.getLogger(),
|
|
87
87
|
});
|
|
88
88
|
const totalBytes = memInfo.totalBytes;
|
|
89
|
-
this.maxMemoryBytes = Math.ceil(totalBytes * serviceLocator.getConfiguration().
|
|
89
|
+
this.maxMemoryBytes = Math.ceil(totalBytes * serviceLocator.getConfiguration().availableMemoryRatio);
|
|
90
90
|
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
91
91
|
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
92
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAGvD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AAsElD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,OAAO,WAAW;IACpB,GAAG,CAAgB;IACnB,+BAA+B,CAAS;IACxC,4BAA4B,CAAS;IACrC,qBAAqB,CAAS;IAC9B,gBAAgB,CAAS;IACzB,kBAAkB,CAAS;IAC3B,eAAe,CAAS;IACxB,cAAc,CAAU;IAExB,YAAY,GAAkB,EAAE,CAAC;IACjC,kBAAkB,GAAwB,EAAE,CAAC;IAC7C,eAAe,GAAqB,EAAE,CAAC;IACvC,eAAe,GAAqB,EAAE,CAAC;IAEvC,iBAAiB,GAAqB,IAAK,CAAC;IAC5C,cAAc,GAAqB,IAAK,CAAC;IAEzC,kCAAkC,GAAgB,IAAI,CAAC;IAEvD;;OAEG;IACH,YAAY,UAA8B,EAAE;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,GACnC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAGvD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AAsElD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,OAAO,WAAW;IACpB,GAAG,CAAgB;IACnB,+BAA+B,CAAS;IACxC,4BAA4B,CAAS;IACrC,qBAAqB,CAAS;IAC9B,gBAAgB,CAAS;IACzB,kBAAkB,CAAS;IAC3B,eAAe,CAAS;IACxB,cAAc,CAAU;IAExB,YAAY,GAAkB,EAAE,CAAC;IACjC,kBAAkB,GAAwB,EAAE,CAAC;IAC7C,eAAe,GAAqB,EAAE,CAAC;IACvC,eAAe,GAAqB,EAAE,CAAC;IAEvC,iBAAiB,GAAqB,IAAK,CAAC;IAC5C,cAAc,GAAqB,IAAK,CAAC;IAEzC,kCAAkC,GAAgB,IAAI,CAAC;IAEvD;;OAEG;IACH,YAAY,UAA8B,EAAE;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,GACnC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAEhD,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,YAAY,IAAI,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,aAAa,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,aAAa,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;YACnG,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC;gBAChC,aAAa;gBACb,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE;aACrC,CAAC,CAAC;YACH,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YAEtC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,oBAAoB,CAAC,CAAC;YACrG,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO;gBACrF,sGAAsG,CAC7G,CAAC;QACN,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CACtC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAClC,IAAI,CAAC,+BAA+B,CACvC,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAE5G,MAAM,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,mBAAmB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAExD,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,MAAM;YACV,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IACI,IAAI,CAAC,kCAAkC;YACvC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAE3F,OAAO;QAEX,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,mCAAmC;gBAC/B,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CACzC,IAAI,CAAC,cAAe,CACvB,QAAQ,cAAc,2CAA2C,CACzE,CAAC;YACF,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QAC/H,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CACrB,SAAoF,EACpF,GAAS;QAET,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;QACf,CAAC;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;CACJ"}
|
package/configuration.d.ts
CHANGED
|
@@ -1,126 +1,57 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { z } from 'zod';
|
|
2
2
|
import { LogLevel } from './log.js';
|
|
3
|
-
export interface
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*
|
|
56
|
-
* Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
|
|
57
|
-
* @default 60_000
|
|
58
|
-
*/
|
|
59
|
-
persistStateIntervalMillis?: number;
|
|
60
|
-
/**
|
|
61
|
-
Defines the interval of emitting the `systemInfo` event.
|
|
62
|
-
@default 1_000
|
|
63
|
-
*/
|
|
64
|
-
systemInfoIntervalMillis?: number;
|
|
65
|
-
/**
|
|
66
|
-
* Defines the default input key, i.e. the key that is used to get the crawler input value
|
|
67
|
-
* from the default {@link KeyValueStore} associated with the current crawler run.
|
|
68
|
-
*
|
|
69
|
-
* Alternative to `CRAWLEE_INPUT_KEY` environment variable.
|
|
70
|
-
* @default 'INPUT'
|
|
71
|
-
*/
|
|
72
|
-
inputKey?: string;
|
|
73
|
-
/**
|
|
74
|
-
* Defines whether web browsers launched by Crawlee will run in the headless mode.
|
|
75
|
-
*
|
|
76
|
-
* Alternative to `CRAWLEE_HEADLESS` environment variable.
|
|
77
|
-
* @default true
|
|
78
|
-
*/
|
|
79
|
-
headless?: boolean;
|
|
80
|
-
/**
|
|
81
|
-
* Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
|
|
82
|
-
*
|
|
83
|
-
* Alternative to `CRAWLEE_XVFB` environment variable.
|
|
84
|
-
* @default false
|
|
85
|
-
*/
|
|
86
|
-
xvfb?: boolean;
|
|
87
|
-
/**
|
|
88
|
-
* Defines a path to Chrome executable.
|
|
89
|
-
*
|
|
90
|
-
* Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
|
|
91
|
-
*/
|
|
92
|
-
chromeExecutablePath?: string;
|
|
93
|
-
/**
|
|
94
|
-
* Defines a path to default browser executable.
|
|
95
|
-
*
|
|
96
|
-
* Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
|
|
97
|
-
*/
|
|
98
|
-
defaultBrowserPath?: string;
|
|
99
|
-
/**
|
|
100
|
-
* Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
|
|
101
|
-
*
|
|
102
|
-
* Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
|
|
103
|
-
*/
|
|
104
|
-
disableBrowserSandbox?: boolean;
|
|
105
|
-
/**
|
|
106
|
-
* Sets the log level to the given value.
|
|
107
|
-
*
|
|
108
|
-
* Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
|
|
109
|
-
* @default 'INFO'
|
|
110
|
-
*/
|
|
111
|
-
logLevel?: LogLevel | LogLevel[keyof LogLevel];
|
|
112
|
-
/**
|
|
113
|
-
* Defines whether the storage client used should persist the data it stores.
|
|
114
|
-
*
|
|
115
|
-
* Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
|
|
116
|
-
*/
|
|
117
|
-
persistStorage?: boolean;
|
|
118
|
-
/**
|
|
119
|
-
* Used in place of `isContainerized()` when collecting system metrics.
|
|
120
|
-
*
|
|
121
|
-
* Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
|
|
122
|
-
*/
|
|
123
|
-
containerized?: boolean;
|
|
3
|
+
export interface ConfigField<T extends z.ZodType = z.ZodType> {
|
|
4
|
+
schema: T;
|
|
5
|
+
envVar?: string | string[];
|
|
6
|
+
}
|
|
7
|
+
export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
|
|
8
|
+
/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
|
|
9
|
+
export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
|
|
10
|
+
export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
|
|
11
|
+
export declare const crawleeConfigFields: {
|
|
12
|
+
/** @default 'default' */
|
|
13
|
+
defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
14
|
+
/** @default true */
|
|
15
|
+
purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
16
|
+
/** @default 'default' */
|
|
17
|
+
defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
18
|
+
/** @default 'default' */
|
|
19
|
+
defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
20
|
+
/** @default 0.95 */
|
|
21
|
+
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
22
|
+
/** @default 0.25 */
|
|
23
|
+
availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
24
|
+
memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
25
|
+
/** @default 60_000 */
|
|
26
|
+
persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
27
|
+
/** @default 1_000 */
|
|
28
|
+
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
29
|
+
/** @default 'INPUT' */
|
|
30
|
+
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
31
|
+
/** @default true */
|
|
32
|
+
headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
33
|
+
/** @default false */
|
|
34
|
+
xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
35
|
+
chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
36
|
+
defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
37
|
+
/** @default false */
|
|
38
|
+
disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
39
|
+
logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
|
|
40
|
+
/** @default true */
|
|
41
|
+
persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
42
|
+
containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
43
|
+
};
|
|
44
|
+
export type FieldsInput<F extends Record<string, ConfigField>> = {
|
|
45
|
+
[K in keyof F]?: z.output<F[K]['schema']>;
|
|
46
|
+
};
|
|
47
|
+
export type FieldsOutput<F extends Record<string, ConfigField>> = {
|
|
48
|
+
[K in keyof F]: z.output<F[K]['schema']>;
|
|
49
|
+
};
|
|
50
|
+
export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
|
|
51
|
+
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
|
|
52
|
+
/** @deprecated Use {@link ConfigurationInput} instead. */
|
|
53
|
+
export type ConfigurationOptions = ConfigurationInput;
|
|
54
|
+
export interface Configuration extends ResolvedConfigValues {
|
|
124
55
|
}
|
|
125
56
|
/**
|
|
126
57
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
@@ -134,13 +65,9 @@ export interface ConfigurationOptions {
|
|
|
134
65
|
*
|
|
135
66
|
* // Get the global configuration
|
|
136
67
|
* const config = Configuration.getGlobalConfig();
|
|
137
|
-
* //
|
|
138
|
-
*
|
|
139
|
-
* config.
|
|
140
|
-
*
|
|
141
|
-
* // No need to pass the configuration to the crawler,
|
|
142
|
-
* // as it's using the global configuration by default
|
|
143
|
-
* const crawler = new BasicCrawler();
|
|
68
|
+
* // Access configuration values directly as properties
|
|
69
|
+
* console.log(config.headless);
|
|
70
|
+
* console.log(config.persistStateIntervalMillis);
|
|
144
71
|
* ```
|
|
145
72
|
*
|
|
146
73
|
* *Using custom configuration:*
|
|
@@ -150,15 +77,14 @@ export interface ConfigurationOptions {
|
|
|
150
77
|
* // Create a new configuration
|
|
151
78
|
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
152
79
|
* // Pass the configuration to the crawler
|
|
153
|
-
* const crawler = new BasicCrawler({ configuration: config
|
|
80
|
+
* const crawler = new BasicCrawler({ configuration: config });
|
|
154
81
|
* ```
|
|
155
82
|
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
* so the options provided in constructor will override those. In other words, the precedence is:
|
|
83
|
+
* Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
|
|
84
|
+
* The priority order for resolving values is (highest to lowest):
|
|
159
85
|
*
|
|
160
86
|
* ```text
|
|
161
|
-
*
|
|
87
|
+
* constructor options > environment variables > crawlee.json > schema defaults
|
|
162
88
|
* ```
|
|
163
89
|
*
|
|
164
90
|
* ## Supported Configuration Options
|
|
@@ -180,45 +106,50 @@ export interface ConfigurationOptions {
|
|
|
180
106
|
* Key | Environment Variable | Default Value
|
|
181
107
|
* ---|---|---
|
|
182
108
|
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
183
|
-
* `xvfb` | `CRAWLEE_XVFB` |
|
|
109
|
+
* `xvfb` | `CRAWLEE_XVFB` | `false`
|
|
184
110
|
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
185
111
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
186
112
|
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
187
113
|
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
188
|
-
* `containerized | `CRAWLEE_CONTAINERIZED | -
|
|
114
|
+
* `containerized` | `CRAWLEE_CONTAINERIZED` | -
|
|
189
115
|
*/
|
|
190
116
|
export declare class Configuration {
|
|
191
117
|
/**
|
|
192
|
-
*
|
|
193
|
-
|
|
194
|
-
protected static ENV_MAP: Dictionary;
|
|
195
|
-
protected static BOOLEAN_VARS: string[];
|
|
196
|
-
protected static INTEGER_VARS: string[];
|
|
197
|
-
protected static COMMA_SEPARATED_LIST_VARS: string[];
|
|
198
|
-
protected static DEFAULTS: Dictionary;
|
|
199
|
-
protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
|
|
200
|
-
/**
|
|
201
|
-
* Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
|
|
202
|
-
*/
|
|
203
|
-
constructor(options?: ConfigurationOptions);
|
|
204
|
-
/**
|
|
205
|
-
* Returns configured value. First checks the environment variables, then provided configuration,
|
|
206
|
-
* fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
|
|
207
|
-
* in the above section.
|
|
118
|
+
* Field definitions for this configuration class.
|
|
119
|
+
* Subclasses override this to register additional fields.
|
|
208
120
|
*/
|
|
209
|
-
|
|
210
|
-
|
|
121
|
+
protected static fields: Record<string, ConfigField>;
|
|
122
|
+
private resolvedValues;
|
|
211
123
|
/**
|
|
212
|
-
*
|
|
213
|
-
*
|
|
124
|
+
* Creates new `Configuration` instance with provided options.
|
|
125
|
+
* Constructor options take precedence over environment variables, which take precedence
|
|
126
|
+
* over crawlee.json values, which take precedence over schema defaults.
|
|
214
127
|
*/
|
|
215
|
-
|
|
128
|
+
constructor(options?: ConfigurationInput);
|
|
216
129
|
/**
|
|
217
130
|
* Returns the global configuration instance. It will respect the environment variables.
|
|
218
131
|
*
|
|
219
132
|
* Delegates to the global ServiceLocator, making it the single source of truth for service management.
|
|
220
133
|
*/
|
|
221
134
|
static getGlobalConfig(): Configuration;
|
|
222
|
-
|
|
135
|
+
/**
|
|
136
|
+
* Resolves all field values once using the priority chain:
|
|
137
|
+
* constructor options > env vars > crawlee.json > schema defaults.
|
|
138
|
+
*/
|
|
139
|
+
private static resolveAll;
|
|
140
|
+
/**
|
|
141
|
+
* Registers getters (and throwing setters) on the instance for each field.
|
|
142
|
+
*/
|
|
143
|
+
private registerAccessors;
|
|
144
|
+
/**
|
|
145
|
+
* Reads the first defined env var value for a field definition.
|
|
146
|
+
* Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
|
|
147
|
+
* (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
|
|
148
|
+
*/
|
|
149
|
+
private static readEnvVar;
|
|
150
|
+
/**
|
|
151
|
+
* Loads config options from crawlee.json in the current working directory.
|
|
152
|
+
*/
|
|
153
|
+
private static loadFileOptions;
|
|
223
154
|
}
|
|
224
155
|
//# sourceMappingURL=configuration.d.ts.map
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAIA,OAAO,
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAO,QAAQ,EAAE,MAAM,UAAU,CAAC;AAWzC,MAAM,WAAW,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;IACxD,MAAM,EAAE,CAAC,CAAC;IACV,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC9B;AAED,wBAAgB,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAEhG;AAID,8DAA8D;AAC9D,eAAO,MAAM,aAAa,2DAKX,CAAC;AAEhB,eAAO,MAAM,YAAY,0DAGX,CAAC;AAcf,eAAO,MAAM,mBAAmB;IAC5B,yBAAyB;;IAEzB,oBAAoB;;IAEpB,yBAAyB;;IAEzB,yBAAyB;;IAEzB,oBAAoB;;IAEpB,oBAAoB;;;IAGpB,sBAAsB;;IAEtB,qBAAqB;;IAErB,uBAAuB;;IAEvB,oBAAoB;;IAEpB,qBAAqB;;;;IAIrB,qBAAqB;;;IAGrB,oBAAoB;;;CAGvB,CAAC;AAIF,MAAM,MAAM,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI;KAC5D,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,YAAY,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI;KAC7D,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;CAC3C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,WAAW,CAAC,OAAO,mBAAmB,CAAC,CAAC;AACzE,MAAM,MAAM,oBAAoB,GAAG,YAAY,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAE5E,0DAA0D;AAC1D,MAAM,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AAOtD,MAAM,WAAW,aAAc,SAAQ,oBAAoB;CAAG;AAE9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AAEH,qBAAa,aAAa;IACtB;;;OAGG;IACH,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAuB;IAE3E,OAAO,CAAC,cAAc,CAA0B;IAEhD;;;;OAIG;gBACS,OAAO,GAAE,kBAAuB;IAa5C;;;;OAIG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IAIvC;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,UAAU;IAmCzB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAkBzB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,UAAU;IAUzB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;CAQjC"}
|