@crawlee/core 3.0.0-beta.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -133
- package/autoscaling/autoscaled_pool.d.ts +5 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +16 -16
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +6 -6
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +12 -8
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +1 -1
- package/autoscaling/system_status.d.ts.map +1 -1
- package/configuration.d.ts +77 -62
- package/configuration.d.ts.map +1 -1
- package/configuration.js +85 -87
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +7 -12
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +25 -23
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +23 -23
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +11 -6
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +7 -2
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +16 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +22 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +12 -8
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +17 -6
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +3 -5
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -5
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -0
- package/index.d.ts.map +1 -1
- package/index.js +5 -0
- package/index.js.map +1 -1
- package/index.mjs +6 -0
- package/package.json +22 -27
- package/proxy_configuration.d.ts +7 -14
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +6 -15
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +11 -6
- package/request.d.ts.map +1 -1
- package/request.js +5 -1
- package/request.js.map +1 -1
- package/router.d.ts +93 -0
- package/router.d.ts.map +1 -0
- package/router.js +137 -0
- package/router.js.map +1 -0
- package/serialization.d.ts.map +1 -1
- package/serialization.js +1 -2
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +10 -15
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +18 -18
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +3 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +4 -5
- package/session_pool/session_pool.js.map +1 -1
- package/session_pool/session_utils.d.ts +6 -5
- package/session_pool/session_utils.d.ts.map +1 -1
- package/session_pool/session_utils.js +3 -0
- package/session_pool/session_utils.js.map +1 -1
- package/storages/dataset.d.ts +37 -15
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +44 -12
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +1 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +121 -23
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +178 -24
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +32 -35
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +32 -24
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +12 -8
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +18 -10
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +9 -5
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +28 -18
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +16 -0
- package/storages/utils.d.ts.map +1 -0
- package/storages/utils.js +26 -0
- package/storages/utils.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts +0 -2
- package/typedefs.d.ts.map +1 -1
- package/typedefs.js.map +1 -1
- package/validators.d.ts +5 -5
- package/validators.d.ts.map +1 -1
- package/validators.js.map +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AAEA,gDAA0E;AAC1E,0CAA+C;AAC/C,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA2C;AAK3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA2DlD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,EAAE;YAClB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;YACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO;kBAClG,sGAAsG,CAAC,CAAC;SACjH;QAED,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE;gBAC3D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC5B;iBAAM;gBACH,MAAM;aACT;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,CAAC,kCAAkC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAAE,OAAO;QAEnJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAe,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC5I,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,SAAoF,EAAE,GAAS;QACrH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AAlSD,kCAkSC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,2CAA2C;IAC3C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,GAAE,mBAAwB;IA6B7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAclE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK3D;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK9D;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAKxD;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAKxE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU;CAoB9H"}
|
package/configuration.d.ts
CHANGED
|
@@ -1,98 +1,114 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
2
|
+
import type { MemoryStorageOptions } from '@crawlee/memory-storage';
|
|
3
|
+
import { MemoryStorage } from '@crawlee/memory-storage';
|
|
3
4
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
4
|
-
import { StorageClient } from '@crawlee/types';
|
|
5
|
-
import {
|
|
6
|
-
import { EventManager } from './events';
|
|
5
|
+
import type { Dictionary, StorageClient } from '@crawlee/types';
|
|
6
|
+
import { LogLevel } from '@apify/log';
|
|
7
|
+
import type { EventManager } from './events';
|
|
7
8
|
export interface ConfigurationOptions {
|
|
8
9
|
storageClient?: StorageClient;
|
|
9
10
|
eventManager?: EventManager;
|
|
10
11
|
storageClientOptions?: Dictionary;
|
|
11
12
|
defaultDatasetId?: string;
|
|
13
|
+
purgeOnStart?: boolean;
|
|
12
14
|
defaultKeyValueStoreId?: string;
|
|
13
15
|
defaultRequestQueueId?: string;
|
|
14
16
|
maxUsedCpuRatio?: number;
|
|
15
17
|
availableMemoryRatio?: number;
|
|
18
|
+
memoryMbytes?: number;
|
|
16
19
|
persistStateIntervalMillis?: number;
|
|
17
20
|
systemInfoIntervalMillis?: number;
|
|
18
|
-
metamorphAfterSleepMillis?: number;
|
|
19
21
|
inputKey?: string;
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
proxyPassword?: string;
|
|
27
|
-
proxyPort?: number;
|
|
22
|
+
headless?: boolean;
|
|
23
|
+
xvfb?: boolean;
|
|
24
|
+
chromeExecutablePath?: string;
|
|
25
|
+
defaultBrowserPath?: string;
|
|
26
|
+
disableBrowserSandbox?: boolean;
|
|
27
|
+
logLevel?: LogLevel | LogLevel[keyof LogLevel];
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
|
-
* `Configuration` is a value object holding
|
|
30
|
+
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
31
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfig()`.
|
|
32
|
+
* Places that depend on a configurable behaviour depend on this class as have the global
|
|
33
|
+
* instance as the default value.
|
|
31
34
|
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* const { Apify } = require('apify');
|
|
35
|
+
* ```js
|
|
36
|
+
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
35
37
|
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
38
|
+
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
39
|
+
* const crawler = new BasicCrawler({ ... }, config);
|
|
40
|
+
* ```
|
|
41
|
+
*
|
|
42
|
+
* The configuration provided via environment variables always takes precedence. We can also
|
|
43
|
+
* define the `crawlee.json` file in the project root directory which will serve as a baseline,
|
|
44
|
+
* so the options provided in constructor will override those.
|
|
45
|
+
*
|
|
46
|
+
* > In other words, the precedence is: crawlee.json < constructor options < environment variables.
|
|
44
47
|
*
|
|
45
48
|
* ## Supported Configuration Options
|
|
46
49
|
*
|
|
47
50
|
* Key | Environment Variable | Default Value
|
|
48
51
|
* ---|---|---
|
|
49
|
-
* `
|
|
50
|
-
* `
|
|
51
|
-
* `
|
|
52
|
-
* `
|
|
52
|
+
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
53
|
+
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
54
|
+
* `headless` | `CRAWLEE_HEADLESS` | -
|
|
55
|
+
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
56
|
+
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
57
|
+
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
58
|
+
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
|
|
53
59
|
*
|
|
54
60
|
* ## Advanced Configuration Options
|
|
55
61
|
*
|
|
56
62
|
* Key | Environment Variable | Default Value
|
|
57
63
|
* ---|---|---
|
|
58
|
-
* `
|
|
59
|
-
* `
|
|
60
|
-
* `
|
|
61
|
-
* `
|
|
62
|
-
* `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
|
|
63
|
-
* `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
|
|
64
|
-
* `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
|
|
65
|
-
* `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
|
|
66
|
-
* `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
|
|
67
|
-
* `proxyPort` | `APIFY_PROXY_PORT` | `8000`
|
|
68
|
-
*
|
|
69
|
-
* ## Not Supported environment variables
|
|
70
|
-
*
|
|
71
|
-
* - `MEMORY_MBYTES`
|
|
72
|
-
* - `HEADLESS`
|
|
73
|
-
* - `XVFB`
|
|
74
|
-
* - `CHROME_EXECUTABLE_PATH`
|
|
64
|
+
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
65
|
+
* `xvfb` | `CRAWLEE_XVFB` | -
|
|
66
|
+
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
67
|
+
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
75
68
|
*/
|
|
76
69
|
export declare class Configuration {
|
|
77
70
|
/**
|
|
78
|
-
* Maps environment variables to config keys (e.g. `
|
|
71
|
+
* Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
|
|
79
72
|
*/
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
73
|
+
protected static ENV_MAP: {
|
|
74
|
+
CRAWLEE_AVAILABLE_MEMORY_RATIO: string;
|
|
75
|
+
CRAWLEE_PURGE_ON_START: string;
|
|
76
|
+
CRAWLEE_MEMORY_MBYTES: string;
|
|
77
|
+
CRAWLEE_DEFAULT_DATASET_ID: string;
|
|
78
|
+
CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: string;
|
|
79
|
+
CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: string;
|
|
80
|
+
CRAWLEE_INPUT_KEY: string;
|
|
81
|
+
CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: string;
|
|
82
|
+
CRAWLEE_HEADLESS: string;
|
|
83
|
+
CRAWLEE_XVFB: string;
|
|
84
|
+
CRAWLEE_CHROME_EXECUTABLE_PATH: string;
|
|
85
|
+
CRAWLEE_DEFAULT_BROWSER_PATH: string;
|
|
86
|
+
CRAWLEE_DISABLE_BROWSER_SANDBOX: string;
|
|
87
|
+
CRAWLEE_LOG_LEVEL: string;
|
|
88
|
+
};
|
|
89
|
+
protected static BOOLEAN_VARS: string[];
|
|
90
|
+
protected static INTEGER_VARS: string[];
|
|
91
|
+
protected static DEFAULTS: {
|
|
92
|
+
defaultKeyValueStoreId: string;
|
|
93
|
+
defaultDatasetId: string;
|
|
94
|
+
defaultRequestQueueId: string;
|
|
95
|
+
inputKey: string;
|
|
96
|
+
maxUsedCpuRatio: number;
|
|
97
|
+
availableMemoryRatio: number;
|
|
98
|
+
storageClientOptions: {};
|
|
99
|
+
purgeOnStart: boolean;
|
|
100
|
+
persistStateIntervalMillis: number;
|
|
101
|
+
systemInfoIntervalMillis: number;
|
|
102
|
+
};
|
|
88
103
|
/**
|
|
89
104
|
* Provides access to the current-instance-scoped Configuration without passing it around in parameters.
|
|
90
105
|
* @internal
|
|
91
106
|
*/
|
|
92
107
|
static storage: AsyncLocalStorage<Configuration>;
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
108
|
+
protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
|
|
109
|
+
protected services: Map<string, unknown>;
|
|
110
|
+
/** @internal */
|
|
111
|
+
static globalConfig?: Configuration;
|
|
96
112
|
/**
|
|
97
113
|
* Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
|
|
98
114
|
*/
|
|
@@ -103,7 +119,7 @@ export declare class Configuration {
|
|
|
103
119
|
* in the above section.
|
|
104
120
|
*/
|
|
105
121
|
get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
|
|
106
|
-
|
|
122
|
+
protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean;
|
|
107
123
|
/**
|
|
108
124
|
* Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
|
|
109
125
|
* To reset a value, we can omit the `value` argument or pass `undefined` there.
|
|
@@ -114,7 +130,7 @@ export declare class Configuration {
|
|
|
114
130
|
* this {@link Configuration} instance. Only first call of this method will create the client, following calls will
|
|
115
131
|
* return the same client instance.
|
|
116
132
|
*
|
|
117
|
-
* Caching works based on the
|
|
133
|
+
* Caching works based on the `storageClientOptions`, so calling this method with different options will return
|
|
118
134
|
* multiple instances, one for each variant of the options.
|
|
119
135
|
* @internal
|
|
120
136
|
*/
|
|
@@ -129,8 +145,6 @@ export declare class Configuration {
|
|
|
129
145
|
useEventManager(events: EventManager): void;
|
|
130
146
|
/**
|
|
131
147
|
* Returns the global configuration instance. It will respect the environment variables.
|
|
132
|
-
* When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
|
|
133
|
-
*
|
|
134
148
|
*/
|
|
135
149
|
static getGlobalConfig(): Configuration;
|
|
136
150
|
/**
|
|
@@ -146,5 +160,6 @@ export declare class Configuration {
|
|
|
146
160
|
* if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
|
|
147
161
|
*/
|
|
148
162
|
static resetGlobalState(): void;
|
|
163
|
+
protected buildOptions(options: ConfigurationOptions): void;
|
|
149
164
|
}
|
|
150
165
|
//# sourceMappingURL=configuration.d.ts.map
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO;;;;;;;;;;;;;;;MAetB;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,CAAiE;IAExG,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ;;;;;;;;;;;MAWvB;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
|
package/configuration.js
CHANGED
|
@@ -1,58 +1,54 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Configuration = void 0;
|
|
4
|
-
const
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
5
|
const memory_storage_1 = require("@crawlee/memory-storage");
|
|
6
|
+
const fs_extra_1 = require("fs-extra");
|
|
7
|
+
const node_path_1 = require("node:path");
|
|
6
8
|
const node_async_hooks_1 = require("node:async_hooks");
|
|
7
9
|
const node_events_1 = require("node:events");
|
|
10
|
+
const log_1 = tslib_1.__importStar(require("@apify/log"));
|
|
8
11
|
const typedefs_1 = require("./typedefs");
|
|
9
12
|
const events_1 = require("./events");
|
|
10
13
|
/**
|
|
11
|
-
* `Configuration` is a value object holding
|
|
14
|
+
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
15
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfig()`.
|
|
16
|
+
* Places that depend on a configurable behaviour depend on this class as have the global
|
|
17
|
+
* instance as the default value.
|
|
12
18
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* const { Apify } = require('apify');
|
|
19
|
+
* ```js
|
|
20
|
+
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
16
21
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
23
|
+
* const crawler = new BasicCrawler({ ... }, config);
|
|
24
|
+
* ```
|
|
25
|
+
*
|
|
26
|
+
* The configuration provided via environment variables always takes precedence. We can also
|
|
27
|
+
* define the `crawlee.json` file in the project root directory which will serve as a baseline,
|
|
28
|
+
* so the options provided in constructor will override those.
|
|
29
|
+
*
|
|
30
|
+
* > In other words, the precedence is: crawlee.json < constructor options < environment variables.
|
|
25
31
|
*
|
|
26
32
|
* ## Supported Configuration Options
|
|
27
33
|
*
|
|
28
34
|
* Key | Environment Variable | Default Value
|
|
29
35
|
* ---|---|---
|
|
30
|
-
* `
|
|
31
|
-
* `
|
|
32
|
-
* `
|
|
33
|
-
* `
|
|
36
|
+
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
37
|
+
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
38
|
+
* `headless` | `CRAWLEE_HEADLESS` | -
|
|
39
|
+
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
40
|
+
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
41
|
+
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
42
|
+
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
|
|
34
43
|
*
|
|
35
44
|
* ## Advanced Configuration Options
|
|
36
45
|
*
|
|
37
46
|
* Key | Environment Variable | Default Value
|
|
38
47
|
* ---|---|---
|
|
39
|
-
* `
|
|
40
|
-
* `
|
|
41
|
-
* `
|
|
42
|
-
* `
|
|
43
|
-
* `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
|
|
44
|
-
* `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
|
|
45
|
-
* `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
|
|
46
|
-
* `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
|
|
47
|
-
* `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
|
|
48
|
-
* `proxyPort` | `APIFY_PROXY_PORT` | `8000`
|
|
49
|
-
*
|
|
50
|
-
* ## Not Supported environment variables
|
|
51
|
-
*
|
|
52
|
-
* - `MEMORY_MBYTES`
|
|
53
|
-
* - `HEADLESS`
|
|
54
|
-
* - `XVFB`
|
|
55
|
-
* - `CHROME_EXECUTABLE_PATH`
|
|
48
|
+
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
49
|
+
* `xvfb` | `CRAWLEE_XVFB` | -
|
|
50
|
+
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
51
|
+
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
56
52
|
*/
|
|
57
53
|
class Configuration {
|
|
58
54
|
/**
|
|
@@ -71,9 +67,15 @@ class Configuration {
|
|
|
71
67
|
writable: true,
|
|
72
68
|
value: new Map()
|
|
73
69
|
});
|
|
74
|
-
this.
|
|
70
|
+
this.buildOptions(options);
|
|
75
71
|
// Increase the global limit for event emitter memory leak warnings.
|
|
76
72
|
node_events_1.EventEmitter.defaultMaxListeners = 50;
|
|
73
|
+
// set the log level to support CRAWLEE_ prefixed env var too
|
|
74
|
+
const logLevel = this.get('logLevel');
|
|
75
|
+
if (logLevel) {
|
|
76
|
+
const level = Number.isFinite(+logLevel) ? +logLevel : log_1.LogLevel[String(logLevel).toUpperCase()];
|
|
77
|
+
log_1.default.setLevel(level);
|
|
78
|
+
}
|
|
77
79
|
}
|
|
78
80
|
/**
|
|
79
81
|
* Returns configured value. First checks the environment variables, then provided configuration,
|
|
@@ -81,9 +83,16 @@ class Configuration {
|
|
|
81
83
|
* in the above section.
|
|
82
84
|
*/
|
|
83
85
|
get(key, defaultValue) {
|
|
84
|
-
// prefer env vars
|
|
85
|
-
|
|
86
|
-
const
|
|
86
|
+
// prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
|
|
87
|
+
let envValue;
|
|
88
|
+
for (const [k, v] of (0, typedefs_1.entries)(Configuration.ENV_MAP)) {
|
|
89
|
+
if (key === v) {
|
|
90
|
+
envValue = process.env[k];
|
|
91
|
+
if (envValue) {
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
87
96
|
if (envValue != null) {
|
|
88
97
|
return this._castEnvValue(key, envValue);
|
|
89
98
|
}
|
|
@@ -116,7 +125,7 @@ class Configuration {
|
|
|
116
125
|
* this {@link Configuration} instance. Only first call of this method will create the client, following calls will
|
|
117
126
|
* return the same client instance.
|
|
118
127
|
*
|
|
119
|
-
* Caching works based on the
|
|
128
|
+
* Caching works based on the `storageClientOptions`, so calling this method with different options will return
|
|
120
129
|
* multiple instances, one for each variant of the options.
|
|
121
130
|
* @internal
|
|
122
131
|
*/
|
|
@@ -159,8 +168,6 @@ class Configuration {
|
|
|
159
168
|
}
|
|
160
169
|
/**
|
|
161
170
|
* Returns the global configuration instance. It will respect the environment variables.
|
|
162
|
-
* When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
|
|
163
|
-
*
|
|
164
171
|
*/
|
|
165
172
|
static getGlobalConfig() {
|
|
166
173
|
if (Configuration.storage.getStore()) {
|
|
@@ -188,81 +195,72 @@ class Configuration {
|
|
|
188
195
|
static resetGlobalState() {
|
|
189
196
|
delete this.globalConfig;
|
|
190
197
|
}
|
|
198
|
+
buildOptions(options) {
|
|
199
|
+
// try to load configuration from crawlee.json as the baseline
|
|
200
|
+
const path = (0, node_path_1.join)(process.cwd(), 'crawlee.json');
|
|
201
|
+
if ((0, fs_extra_1.pathExistsSync)(path)) {
|
|
202
|
+
try {
|
|
203
|
+
const file = (0, fs_extra_1.readFileSync)(path);
|
|
204
|
+
const optionsFromFileConfig = JSON.parse(file.toString());
|
|
205
|
+
Object.assign(options, optionsFromFileConfig);
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// ignore
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
this.options = new Map((0, typedefs_1.entries)(options));
|
|
212
|
+
}
|
|
191
213
|
}
|
|
192
214
|
exports.Configuration = Configuration;
|
|
193
215
|
/**
|
|
194
|
-
* Maps environment variables to config keys (e.g. `
|
|
216
|
+
* Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
|
|
195
217
|
*/
|
|
196
218
|
Object.defineProperty(Configuration, "ENV_MAP", {
|
|
197
219
|
enumerable: true,
|
|
198
220
|
configurable: true,
|
|
199
221
|
writable: true,
|
|
200
222
|
value: {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
APIFY_PROXY_HOSTNAME: 'proxyHostname',
|
|
216
|
-
APIFY_PROXY_PASSWORD: 'proxyPassword',
|
|
217
|
-
APIFY_PROXY_PORT: 'proxyPort',
|
|
218
|
-
// not supported, use env vars directly:
|
|
219
|
-
// APIFY_MEMORY_MBYTES: 'memoryMbytes',
|
|
220
|
-
// APIFY_HEADLESS: 'headless',
|
|
221
|
-
// APIFY_XVFB: 'xvfb',
|
|
222
|
-
// APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
223
|
+
CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
|
|
224
|
+
CRAWLEE_PURGE_ON_START: 'purgeOnStart',
|
|
225
|
+
CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
|
|
226
|
+
CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
|
|
227
|
+
CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
|
|
228
|
+
CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
|
|
229
|
+
CRAWLEE_INPUT_KEY: 'inputKey',
|
|
230
|
+
CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
|
|
231
|
+
CRAWLEE_HEADLESS: 'headless',
|
|
232
|
+
CRAWLEE_XVFB: 'xvfb',
|
|
233
|
+
CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
234
|
+
CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
|
|
235
|
+
CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
|
|
236
|
+
CRAWLEE_LOG_LEVEL: 'logLevel',
|
|
223
237
|
}
|
|
224
238
|
});
|
|
225
|
-
/**
|
|
226
|
-
* Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
|
|
227
|
-
*/
|
|
228
|
-
Object.defineProperty(Configuration, "ENV_MAP_REVERSED", {
|
|
229
|
-
enumerable: true,
|
|
230
|
-
configurable: true,
|
|
231
|
-
writable: true,
|
|
232
|
-
value: (0, typedefs_1.entries)(Configuration.ENV_MAP).reduce((obj, [key, value]) => {
|
|
233
|
-
obj[value] = key;
|
|
234
|
-
return obj;
|
|
235
|
-
}, {})
|
|
236
|
-
});
|
|
237
239
|
Object.defineProperty(Configuration, "BOOLEAN_VARS", {
|
|
238
240
|
enumerable: true,
|
|
239
241
|
configurable: true,
|
|
240
242
|
writable: true,
|
|
241
|
-
value: []
|
|
243
|
+
value: ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox']
|
|
242
244
|
});
|
|
243
245
|
Object.defineProperty(Configuration, "INTEGER_VARS", {
|
|
244
246
|
enumerable: true,
|
|
245
247
|
configurable: true,
|
|
246
248
|
writable: true,
|
|
247
|
-
value: ['
|
|
249
|
+
value: ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis']
|
|
248
250
|
});
|
|
249
251
|
Object.defineProperty(Configuration, "DEFAULTS", {
|
|
250
252
|
enumerable: true,
|
|
251
253
|
configurable: true,
|
|
252
254
|
writable: true,
|
|
253
255
|
value: {
|
|
254
|
-
defaultKeyValueStoreId:
|
|
255
|
-
defaultDatasetId:
|
|
256
|
-
defaultRequestQueueId:
|
|
256
|
+
defaultKeyValueStoreId: 'default',
|
|
257
|
+
defaultDatasetId: 'default',
|
|
258
|
+
defaultRequestQueueId: 'default',
|
|
259
|
+
inputKey: 'INPUT',
|
|
257
260
|
maxUsedCpuRatio: 0.95,
|
|
258
261
|
availableMemoryRatio: 0.25,
|
|
259
262
|
storageClientOptions: {},
|
|
260
|
-
|
|
261
|
-
proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
|
|
262
|
-
proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
|
|
263
|
-
containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
|
|
264
|
-
containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
|
|
265
|
-
metamorphAfterSleepMillis: 300000,
|
|
263
|
+
purgeOnStart: true,
|
|
266
264
|
persistStateIntervalMillis: 60000,
|
|
267
265
|
systemInfoIntervalMillis: 60000,
|
|
268
266
|
}
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAwB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,aAAa;IAkDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAE1B,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AApOL,sCAqOC;AApOG;;GAEG;AACH;;;;WAA2B;QACvB,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;KAChC;GAAC;AAEF;;;;WAA0C,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAExG;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAA4B;QACxB,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
1
|
+
import type { Dictionary } from '@crawlee/types';
|
|
2
|
+
import type { Log } from '../log';
|
|
3
|
+
import type { ProxyInfo } from '../proxy_configuration';
|
|
4
|
+
import type { Request } from '../request';
|
|
5
|
+
import type { Session } from '../session_pool/session';
|
|
6
|
+
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {
|
|
6
7
|
id: string;
|
|
7
8
|
/**
|
|
8
9
|
* The original {@link Request} object.
|
|
9
10
|
*/
|
|
10
|
-
request: Request
|
|
11
|
+
request: Request<UserData>;
|
|
11
12
|
session?: Session;
|
|
12
13
|
/**
|
|
13
14
|
* An object with information about currently used proxy by the crawler
|
|
@@ -16,10 +17,4 @@ export interface CrawlingContext extends Record<PropertyKey, unknown> {
|
|
|
16
17
|
proxyInfo?: ProxyInfo;
|
|
17
18
|
log: Log;
|
|
18
19
|
}
|
|
19
|
-
export interface CrawlerHandleFailedRequestInput extends CrawlingContext {
|
|
20
|
-
/**
|
|
21
|
-
* The Error thrown by `requestHandler`.
|
|
22
|
-
*/
|
|
23
|
-
error: Error;
|
|
24
|
-
}
|
|
25
20
|
//# sourceMappingURL=crawler_commons.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD,MAAM,WAAW,eAAe,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;IAC3G,EAAE,EAAE,MAAM,CAAC;IACX;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,GAAG,EAAE,GAAG,CAAC;CACZ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,SAAK,EAAE,aAAa,SAAK,GAAG,MAAM,CAmBvF"}
|