@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +88 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +20 -29
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +16 -16
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +22 -25
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +36 -21
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +47 -26
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +34 -7
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +8 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -2
- package/index.d.ts.map +1 -1
- package/index.js +3 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +84 -12
- package/request.d.ts.map +1 -1
- package/request.js +107 -27
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +239 -0
- package/service_locator.js.map +1 -0
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/fingerprint.d.ts +10 -0
- package/session_pool/fingerprint.d.ts.map +1 -0
- package/session_pool/fingerprint.js +31 -0
- package/session_pool/fingerprint.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +35 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +46 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +71 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +111 -99
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +98 -47
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +166 -123
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +8 -6
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +6 -6
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +163 -33
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +267 -119
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.d.ts +32 -81
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +49 -39
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +97 -0
- package/storages/request_loader.d.ts.map +1 -0
- package/storages/request_loader.js +2 -0
- package/storages/request_loader.js.map +1 -0
- package/storages/request_manager.d.ts +34 -0
- package/storages/request_manager.d.ts.map +1 -0
- package/storages/request_manager.js +2 -0
- package/storages/request_manager.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +107 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +198 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_queue.d.ts +282 -45
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +592 -213
- package/storages/request_queue.js.map +1 -1
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +29 -24
- package/storages/sitemap_request_loader.d.ts.map +1 -0
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +50 -46
- package/storages/sitemap_request_loader.js.map +1 -0
- package/storages/storage_instance_manager.d.ts +92 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +261 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +51 -5
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +60 -8
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +8 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +8 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/router.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAIhD,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAIhD,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAc7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,MAAM;IACE,MAAM,GAAwD,IAAI,GAAG,EAAE,CAAC;IACxE,WAAW,GAA0C,EAAE,CAAC;IAEzE;;;OAGG;IACH,gBAAyB,CAAC;IAE1B;;OAEG;IACH,UAAU,CACN,KAAsB,EACtB,OAA2G;QAE3G,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CACb,OAA2G;QAE3G,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,UAA6C;QAC7C,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;QACnC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,iBAAiB,CACvB,8BAA8B,MAAM,CAAC,KAAK,CAAC,IAAI;YAC3C,6DAA6D;YAC7D,2EAA2E,CAClF,CAAC;IACN,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GACT,KAAK,KAAK,YAAY;gBAClB,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YACnE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,CAGX,MAAwC;QACtC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAElC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,WAAW,OAAgB;YACzC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;YAEpE,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;YAC9B,CAAC;YAED,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;QAEF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEjC,OAAO,IAAyC,CAAC;IACrD,CAAC;CACJ"}
|
package/serialization.js
CHANGED
|
@@ -116,7 +116,7 @@ function createChunkCollector(options = {}) {
|
|
|
116
116
|
}
|
|
117
117
|
function pluckValue(streamArray) {
|
|
118
118
|
const realPush = streamArray.push.bind(streamArray);
|
|
119
|
-
streamArray.push = (obj) => realPush(obj
|
|
119
|
+
streamArray.push = (obj) => realPush(obj?.value ?? null);
|
|
120
120
|
return streamArray;
|
|
121
121
|
}
|
|
122
122
|
//# sourceMappingURL=serialization.js.map
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAE/D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;AAIhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,QAAQ;IAKrB;IAJJ,MAAM,GAAG,CAAC,CAAC;IACF,SAAS,CAAS;IAEnC,YACY,IAAS,EACjB,UAAkC,EAAE;QAEpC,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QAPK,SAAI,GAAJ,IAAI,CAAK;QAQjB,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAEpE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAA4B,cAAmC;IACjG,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAE1G,OAAO,MAAa,CAAC;AACzB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,cAAmC;IACjE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,GAAQ,EAAE,EAAE,CAC3F,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACxB,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CACzB,UAA0C,EAAE;IAE5C,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,QAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAE/D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;AAIhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,QAAQ;IAKrB;IAJJ,MAAM,GAAG,CAAC,CAAC;IACF,SAAS,CAAS;IAEnC,YACY,IAAS,EACjB,UAAkC,EAAE;QAEpC,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QAPK,SAAI,GAAJ,IAAI,CAAK;QAQjB,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAEpE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAA4B,cAAmC;IACjG,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAE1G,OAAO,MAAa,CAAC;AACzB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,cAAmC;IACjE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,CAAC,GAAQ,EAAE,EAAE,CAC3F,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACxB,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CACzB,UAA0C,EAAE;IAE5C,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,QAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC;IACzD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import type { StorageBackend } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from './configuration.js';
|
|
3
|
+
import type { EventManager } from './events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from './log.js';
|
|
5
|
+
import { StorageInstanceManager } from './storages/storage_instance_manager.js';
|
|
6
|
+
interface ServiceLocatorInterface {
|
|
7
|
+
/**
|
|
8
|
+
* Get the configuration.
|
|
9
|
+
* Creates a default Configuration instance if none has been set.
|
|
10
|
+
*/
|
|
11
|
+
getConfiguration(): Configuration;
|
|
12
|
+
/**
|
|
13
|
+
* Set the configuration.
|
|
14
|
+
*
|
|
15
|
+
* @param configuration The configuration to set
|
|
16
|
+
* @throws {ServiceConflictError} If a different configuration has already been retrieved
|
|
17
|
+
*/
|
|
18
|
+
setConfiguration(configuration: Configuration): void;
|
|
19
|
+
/**
|
|
20
|
+
* Get the event manager.
|
|
21
|
+
* Creates a default LocalEventManager instance if none has been set.
|
|
22
|
+
*/
|
|
23
|
+
getEventManager(): EventManager;
|
|
24
|
+
/**
|
|
25
|
+
* Set the event manager.
|
|
26
|
+
*
|
|
27
|
+
* @param eventManager The event manager to set
|
|
28
|
+
* @throws {ServiceConflictError} If a different event manager has already been retrieved
|
|
29
|
+
*/
|
|
30
|
+
setEventManager(eventManager: EventManager): void;
|
|
31
|
+
/**
|
|
32
|
+
* Get the storage backend.
|
|
33
|
+
* Creates a default storage backend if none has been set — `FileSystemStorageBackend` when
|
|
34
|
+
* `persistStorage` is enabled (the default), `MemoryStorageBackend` otherwise.
|
|
35
|
+
*/
|
|
36
|
+
getStorageBackend(): StorageBackend;
|
|
37
|
+
/**
|
|
38
|
+
* Set the storage backend.
|
|
39
|
+
*
|
|
40
|
+
* @param storageBackend The storage backend to set
|
|
41
|
+
* @throws {ServiceConflictError} If a different storage backend has already been retrieved
|
|
42
|
+
*/
|
|
43
|
+
setStorageBackend(storageBackend: StorageBackend): void;
|
|
44
|
+
/**
|
|
45
|
+
* Get the logger.
|
|
46
|
+
* Returns the default `@apify/log` logger if none has been set.
|
|
47
|
+
*/
|
|
48
|
+
getLogger(): CrawleeLogger;
|
|
49
|
+
/**
|
|
50
|
+
* Set the logger.
|
|
51
|
+
*
|
|
52
|
+
* @param logger The logger to set
|
|
53
|
+
* @throws {ServiceConflictError} If a different logger has already been retrieved
|
|
54
|
+
*/
|
|
55
|
+
setLogger(logger: CrawleeLogger): void;
|
|
56
|
+
/**
|
|
57
|
+
* Get a child logger with the given prefix.
|
|
58
|
+
* Equivalent to `getLogger().child({ prefix })`.
|
|
59
|
+
*/
|
|
60
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
61
|
+
/**
|
|
62
|
+
* Get the storage instance manager (shared across all storage types).
|
|
63
|
+
*/
|
|
64
|
+
getStorageInstanceManager(): StorageInstanceManager;
|
|
65
|
+
/**
|
|
66
|
+
* Resets the service locator to its initial state.
|
|
67
|
+
* Used mainly for testing purposes.
|
|
68
|
+
* @internal
|
|
69
|
+
*/
|
|
70
|
+
reset(): void;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Service locator for managing the services used by Crawlee.
|
|
74
|
+
*
|
|
75
|
+
* All services are initialized to their default value lazily.
|
|
76
|
+
*
|
|
77
|
+
* There are two primary usage patterns:
|
|
78
|
+
*
|
|
79
|
+
* **1. Global service locator (for default services):**
|
|
80
|
+
* ```typescript
|
|
81
|
+
* import { serviceLocator, BasicCrawler } from 'crawlee';
|
|
82
|
+
*
|
|
83
|
+
* // Optionally configure global services before creating crawlers
|
|
84
|
+
* serviceLocator.setStorageBackend(myCustomClient);
|
|
85
|
+
*
|
|
86
|
+
* // Crawler uses global services
|
|
87
|
+
* const crawler = new BasicCrawler({ ... });
|
|
88
|
+
* ```
|
|
89
|
+
*
|
|
90
|
+
* **2. Per-crawler services (recommended for isolation):**
|
|
91
|
+
* ```typescript
|
|
92
|
+
* import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
|
|
93
|
+
*
|
|
94
|
+
* const crawler = new BasicCrawler({
|
|
95
|
+
* requestHandler: async ({ request }) => { ... },
|
|
96
|
+
* configuration: new Configuration({ ... }), // custom config
|
|
97
|
+
* storageBackend: new MemoryStorageBackend(), // custom storage
|
|
98
|
+
* eventManager: LocalEventManager.fromConfig(), // custom events
|
|
99
|
+
* });
|
|
100
|
+
* // Crawler has its own isolated ServiceLocator instance
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
104
|
+
private configuration?;
|
|
105
|
+
private eventManager?;
|
|
106
|
+
private storageBackend?;
|
|
107
|
+
private logger?;
|
|
108
|
+
/**
|
|
109
|
+
* Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
|
|
110
|
+
* Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
|
|
111
|
+
* Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
|
|
112
|
+
*/
|
|
113
|
+
private static storageInstanceManager?;
|
|
114
|
+
/**
|
|
115
|
+
* Creates a new ServiceLocator instance.
|
|
116
|
+
*
|
|
117
|
+
* @param configuration Optional configuration instance to use
|
|
118
|
+
* @param eventManager Optional event manager instance to use
|
|
119
|
+
* @param storageBackend Optional storage backend instance to use
|
|
120
|
+
* @param logger Optional logger instance to use
|
|
121
|
+
*/
|
|
122
|
+
constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
|
|
123
|
+
getConfiguration(): Configuration;
|
|
124
|
+
setConfiguration(configuration: Configuration): void;
|
|
125
|
+
getEventManager(): EventManager;
|
|
126
|
+
setEventManager(eventManager: EventManager): void;
|
|
127
|
+
getStorageBackend(): StorageBackend;
|
|
128
|
+
setStorageBackend(storageBackend: StorageBackend): void;
|
|
129
|
+
getLogger(): CrawleeLogger;
|
|
130
|
+
setLogger(logger: CrawleeLogger): void;
|
|
131
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
132
|
+
getStorageInstanceManager(): StorageInstanceManager;
|
|
133
|
+
reset(): void;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Wraps all methods on `target` so that any code they invoke will see the given
|
|
137
|
+
* `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
|
|
138
|
+
*
|
|
139
|
+
* Walks the prototype chain and replaces each method on the *instance* (not the prototype)
|
|
140
|
+
* with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
|
|
141
|
+
*
|
|
142
|
+
* The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
|
|
143
|
+
* wrapped method — including `super` calls, since the prototype methods execute within the
|
|
144
|
+
* context established by the instance-level wrapper.
|
|
145
|
+
*
|
|
146
|
+
* @internal
|
|
147
|
+
* @returns Scope control functions: `run` executes a callback within the scoped context,
|
|
148
|
+
* `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
|
|
149
|
+
*/
|
|
150
|
+
export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
|
|
151
|
+
run: <T>(fn: () => T) => T;
|
|
152
|
+
enterScope: () => void;
|
|
153
|
+
exitScope: () => void;
|
|
154
|
+
};
|
|
155
|
+
export declare const serviceLocator: ServiceLocatorInterface;
|
|
156
|
+
export {};
|
|
157
|
+
//# sourceMappingURL=service_locator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service_locator.d.ts","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAIrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG9C,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AAEhF,UAAU,uBAAuB;IAC7B;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,eAAe,IAAI,YAAY,CAAC;IAEhC;;;;;OAKG;IACH,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD;;;;OAIG;IACH,iBAAiB,IAAI,cAAc,CAAC;IAEpC;;;;;OAKG;IACH,iBAAiB,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IAExD;;;OAGG;IACH,SAAS,IAAI,aAAa,CAAC;IAE3B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAEvC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC;IAE3C;;OAEG;IACH,yBAAyB,IAAI,sBAAsB,CAAC;IAEpD;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,cAAe,YAAW,uBAAuB;IAC1D,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,MAAM,CAAC,CAAgB;IAE/B;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAyB;IAE/D;;;;;;;OAOG;gBAEC,aAAa,CAAC,EAAE,aAAa,EAC7B,YAAY,CAAC,EAAE,YAAY,EAC3B,cAAc,CAAC,EAAE,cAAc,EAC/B,MAAM,CAAC,EAAE,aAAa;IAQ1B,gBAAgB,IAAI,aAAa;IAQjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,eAAe,IAAI,YAAY;IAc/B,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAcjD,iBAAiB,IAAI,cAAc;IAyBnC,iBAAiB,CAAC,cAAc,EAAE,cAAc,GAAG,IAAI;IAcvD,SAAS,IAAI,aAAa;IAO1B,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAYtC,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa;IAI1C,yBAAyB,IAAI,sBAAsB;IAOnD,KAAK,IAAI,IAAI;CAQhB;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,CACvC,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,EAAE,GACX;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,IAAI,CAAA;CAAE,CA4C/E;AAED,eAAO,MAAM,cAAc,yBAczB,CAAC"}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { FileSystemStorageBackend } from '@crawlee/fs-storage';
|
|
3
|
+
import log from '@apify/log';
|
|
4
|
+
import { Configuration } from './configuration.js';
|
|
5
|
+
import { ServiceConflictError } from './errors.js';
|
|
6
|
+
import { LocalEventManager } from './events/local_event_manager.js';
|
|
7
|
+
import { ApifyLogAdapter } from './log.js';
|
|
8
|
+
import { MemoryStorageBackend } from './memory-storage/index.js';
|
|
9
|
+
import { StorageInstanceManager } from './storages/storage_instance_manager.js';
|
|
10
|
+
/**
|
|
11
|
+
* Service locator for managing the services used by Crawlee.
|
|
12
|
+
*
|
|
13
|
+
* All services are initialized to their default value lazily.
|
|
14
|
+
*
|
|
15
|
+
* There are two primary usage patterns:
|
|
16
|
+
*
|
|
17
|
+
* **1. Global service locator (for default services):**
|
|
18
|
+
* ```typescript
|
|
19
|
+
* import { serviceLocator, BasicCrawler } from 'crawlee';
|
|
20
|
+
*
|
|
21
|
+
* // Optionally configure global services before creating crawlers
|
|
22
|
+
* serviceLocator.setStorageBackend(myCustomClient);
|
|
23
|
+
*
|
|
24
|
+
* // Crawler uses global services
|
|
25
|
+
* const crawler = new BasicCrawler({ ... });
|
|
26
|
+
* ```
|
|
27
|
+
*
|
|
28
|
+
* **2. Per-crawler services (recommended for isolation):**
|
|
29
|
+
* ```typescript
|
|
30
|
+
* import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
|
|
31
|
+
*
|
|
32
|
+
* const crawler = new BasicCrawler({
|
|
33
|
+
* requestHandler: async ({ request }) => { ... },
|
|
34
|
+
* configuration: new Configuration({ ... }), // custom config
|
|
35
|
+
* storageBackend: new MemoryStorageBackend(), // custom storage
|
|
36
|
+
* eventManager: LocalEventManager.fromConfig(), // custom events
|
|
37
|
+
* });
|
|
38
|
+
* // Crawler has its own isolated ServiceLocator instance
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
export class ServiceLocator {
|
|
42
|
+
configuration;
|
|
43
|
+
eventManager;
|
|
44
|
+
storageBackend;
|
|
45
|
+
logger;
|
|
46
|
+
/**
|
|
47
|
+
* Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
|
|
48
|
+
* Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
|
|
49
|
+
* Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
|
|
50
|
+
*/
|
|
51
|
+
static storageInstanceManager;
|
|
52
|
+
/**
|
|
53
|
+
* Creates a new ServiceLocator instance.
|
|
54
|
+
*
|
|
55
|
+
* @param configuration Optional configuration instance to use
|
|
56
|
+
* @param eventManager Optional event manager instance to use
|
|
57
|
+
* @param storageBackend Optional storage backend instance to use
|
|
58
|
+
* @param logger Optional logger instance to use
|
|
59
|
+
*/
|
|
60
|
+
constructor(configuration, eventManager, storageBackend, logger) {
|
|
61
|
+
this.configuration = configuration;
|
|
62
|
+
this.eventManager = eventManager;
|
|
63
|
+
this.storageBackend = storageBackend;
|
|
64
|
+
this.logger = logger;
|
|
65
|
+
}
|
|
66
|
+
getConfiguration() {
|
|
67
|
+
if (!this.configuration) {
|
|
68
|
+
this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
|
|
69
|
+
this.configuration = new Configuration();
|
|
70
|
+
}
|
|
71
|
+
return this.configuration;
|
|
72
|
+
}
|
|
73
|
+
setConfiguration(configuration) {
|
|
74
|
+
// Same instance, no need to do anything
|
|
75
|
+
if (this.configuration === configuration) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
// Already have a different configuration that was retrieved
|
|
79
|
+
if (this.configuration) {
|
|
80
|
+
throw new ServiceConflictError('Configuration', configuration, this.configuration);
|
|
81
|
+
}
|
|
82
|
+
this.configuration = configuration;
|
|
83
|
+
}
|
|
84
|
+
getEventManager() {
|
|
85
|
+
if (!this.eventManager) {
|
|
86
|
+
this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
|
|
87
|
+
if (!this.configuration) {
|
|
88
|
+
this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
|
|
89
|
+
'It is advised to explicitly first set the configuration instead.');
|
|
90
|
+
}
|
|
91
|
+
this.eventManager = LocalEventManager.fromConfig(this.getConfiguration());
|
|
92
|
+
}
|
|
93
|
+
return this.eventManager;
|
|
94
|
+
}
|
|
95
|
+
setEventManager(eventManager) {
|
|
96
|
+
// Same instance, no need to do anything
|
|
97
|
+
if (this.eventManager === eventManager) {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
// Already have a different event manager that was retrieved
|
|
101
|
+
if (this.eventManager) {
|
|
102
|
+
throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
|
|
103
|
+
}
|
|
104
|
+
this.eventManager = eventManager;
|
|
105
|
+
}
|
|
106
|
+
getStorageBackend() {
|
|
107
|
+
if (!this.storageBackend) {
|
|
108
|
+
this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
|
|
109
|
+
'(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
|
|
110
|
+
if (!this.configuration) {
|
|
111
|
+
this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
|
|
112
|
+
'It is advised to explicitly first set the configuration instead.');
|
|
113
|
+
}
|
|
114
|
+
const config = this.getConfiguration();
|
|
115
|
+
this.storageBackend = config.persistStorage
|
|
116
|
+
? new FileSystemStorageBackend({
|
|
117
|
+
localDataDirectory: config.storageDir,
|
|
118
|
+
logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
|
|
119
|
+
})
|
|
120
|
+
: new MemoryStorageBackend({
|
|
121
|
+
logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
return this.storageBackend;
|
|
125
|
+
}
|
|
126
|
+
setStorageBackend(storageBackend) {
|
|
127
|
+
// Same instance, no need to do anything
|
|
128
|
+
if (this.storageBackend === storageBackend) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
// Already have a different storage backend that was retrieved
|
|
132
|
+
if (this.storageBackend) {
|
|
133
|
+
throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);
|
|
134
|
+
}
|
|
135
|
+
this.storageBackend = storageBackend;
|
|
136
|
+
}
|
|
137
|
+
getLogger() {
|
|
138
|
+
if (!this.logger) {
|
|
139
|
+
this.logger = new ApifyLogAdapter(log);
|
|
140
|
+
}
|
|
141
|
+
return this.logger;
|
|
142
|
+
}
|
|
143
|
+
setLogger(logger) {
|
|
144
|
+
if (this.logger === logger) {
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (this.logger) {
|
|
148
|
+
throw new ServiceConflictError('Logger', logger, this.logger);
|
|
149
|
+
}
|
|
150
|
+
this.logger = logger;
|
|
151
|
+
}
|
|
152
|
+
getChildLog(prefix) {
|
|
153
|
+
return this.getLogger().child({ prefix });
|
|
154
|
+
}
|
|
155
|
+
getStorageInstanceManager() {
|
|
156
|
+
if (!ServiceLocator.storageInstanceManager) {
|
|
157
|
+
ServiceLocator.storageInstanceManager = new StorageInstanceManager();
|
|
158
|
+
}
|
|
159
|
+
return ServiceLocator.storageInstanceManager;
|
|
160
|
+
}
|
|
161
|
+
reset() {
|
|
162
|
+
this.configuration = undefined;
|
|
163
|
+
this.eventManager = undefined;
|
|
164
|
+
this.storageBackend = undefined;
|
|
165
|
+
this.logger = undefined;
|
|
166
|
+
ServiceLocator.storageInstanceManager?.clearCache();
|
|
167
|
+
ServiceLocator.storageInstanceManager = undefined;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Used as the default service provider when crawlers don't specify custom services.
|
|
172
|
+
*/
|
|
173
|
+
const globalServiceLocator = new ServiceLocator();
|
|
174
|
+
const serviceLocatorStorage = new AsyncLocalStorage();
|
|
175
|
+
/**
|
|
176
|
+
* Wraps all methods on `target` so that any code they invoke will see the given
|
|
177
|
+
* `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
|
|
178
|
+
*
|
|
179
|
+
* Walks the prototype chain and replaces each method on the *instance* (not the prototype)
|
|
180
|
+
* with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
|
|
181
|
+
*
|
|
182
|
+
* The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
|
|
183
|
+
* wrapped method — including `super` calls, since the prototype methods execute within the
|
|
184
|
+
* context established by the instance-level wrapper.
|
|
185
|
+
*
|
|
186
|
+
* @internal
|
|
187
|
+
* @returns Scope control functions: `run` executes a callback within the scoped context,
|
|
188
|
+
* `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
|
|
189
|
+
*/
|
|
190
|
+
export function bindMethodsToServiceLocator(serviceLocator, target) {
|
|
191
|
+
let proto = Object.getPrototypeOf(target);
|
|
192
|
+
while (proto !== null && proto !== Object.prototype) {
|
|
193
|
+
const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
|
|
194
|
+
for (const propertyKey of propertyKeys) {
|
|
195
|
+
const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
|
|
196
|
+
// We use property descriptors rather than accessing target[propertyKey] directly,
|
|
197
|
+
// because that would trigger getters and cause unwanted side effects.
|
|
198
|
+
// Skip getters, setters, and constructors — only wrap regular methods.
|
|
199
|
+
if (propertyKey === 'constructor' ||
|
|
200
|
+
!descriptor ||
|
|
201
|
+
descriptor.get ||
|
|
202
|
+
descriptor.set ||
|
|
203
|
+
typeof descriptor.value !== 'function')
|
|
204
|
+
continue;
|
|
205
|
+
const original = descriptor.value;
|
|
206
|
+
target[propertyKey] = (...args) => {
|
|
207
|
+
return serviceLocatorStorage.run(serviceLocator, () => {
|
|
208
|
+
return original.apply(target, args);
|
|
209
|
+
});
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
proto = Object.getPrototypeOf(proto);
|
|
213
|
+
}
|
|
214
|
+
let previousStore;
|
|
215
|
+
return {
|
|
216
|
+
run: (fn) => serviceLocatorStorage.run(serviceLocator, fn),
|
|
217
|
+
enterScope: () => {
|
|
218
|
+
previousStore = serviceLocatorStorage.getStore();
|
|
219
|
+
serviceLocatorStorage.enterWith(serviceLocator);
|
|
220
|
+
},
|
|
221
|
+
exitScope: () => {
|
|
222
|
+
serviceLocatorStorage.enterWith(previousStore); // casting to any so that `undefined` is accepted - this "unsets" the AsyncLocalStorage
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
export const serviceLocator = new Proxy({}, {
|
|
227
|
+
get(_target, prop) {
|
|
228
|
+
const active = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
229
|
+
const value = Reflect.get(active, prop, active);
|
|
230
|
+
if (typeof value === 'function') {
|
|
231
|
+
return value.bind(active);
|
|
232
|
+
}
|
|
233
|
+
return value;
|
|
234
|
+
},
|
|
235
|
+
set(_target, prop) {
|
|
236
|
+
throw new TypeError(`Cannot set property '${String(prop)}' on serviceLocator directly. Use the setter methods (e.g. setConfiguration(), setStorageBackend()) instead.`);
|
|
237
|
+
},
|
|
238
|
+
});
|
|
239
|
+
//# sourceMappingURL=service_locator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service_locator.js","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAG/D,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AA+EhF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,OAAO,cAAc;IACf,aAAa,CAAiB;IAC9B,YAAY,CAAgB;IAC5B,cAAc,CAAkB;IAChC,MAAM,CAAiB;IAE/B;;;;OAIG;IACK,MAAM,CAAC,sBAAsB,CAA0B;IAE/D;;;;;;;OAOG;IACH,YACI,aAA6B,EAC7B,YAA2B,EAC3B,cAA+B,EAC/B,MAAsB;QAEtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,4EAA4E,CAAC,CAAC;YACrG,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,gBAAgB,CAAC,aAA4B;QACzC,wCAAwC;QACxC,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;YACvC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,eAAe,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,eAAe;QACX,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;YACzG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CACpB,uFAAuF;oBACnF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,IAAI,CAAC,YAAY,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,eAAe,CAAC,YAA0B;QACtC,wCAAwC;QACxC,IAAI,IAAI,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,oBAAoB,CAAC,cAAc,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACpF,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED,iBAAiB;QACb,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAClB,oFAAoF;gBAChF,4FAA4F,CACnG,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,CACpB,yFAAyF;oBACrF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc;gBACvC,CAAC,CAAC,IAAI,wBAAwB,CAAC;oBACzB,kBAAkB,EAAE,MAAM,CAAC,UAAU;oBACrC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;iBACzE,CAAC;gBACJ,CAAC,CAAC,IAAI,oBAAoB,CAAC;oBACrB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;iBACrE,CAAC,CAAC;QACb,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,iBAAiB,CAAC,cAA8B;QAC5C,wCAAwC;QACxC,IAAI,IAAI,CAAC,cAAc,KAAK,cAAc,EAAE,CAAC;YACzC,OAAO;QACX,CAAC;QAED,8DAA8D;QAC9D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,IAAI,oBAAoB,CAAC,gBAAgB,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CAAC,MAAqB;QAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,WAAW,CAAC,MAAc;QACtB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,yBAAyB;QACrB,IAAI,CAAC,cAAc,CAAC,sBAAsB,EAAE,CAAC;YACzC,cAAc,CAAC,sBAAsB,GAAG,IAAI,sBAAsB,EAAE,CAAC;QACzE,CAAC;QACD,OAAO,cAAc,CAAC,sBAAsB,CAAC;IACjD,CAAC;IAED,KAAK;QACD,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,cAAc,CAAC,sBAAsB,EAAE,UAAU,EAAE,CAAC;QACpD,cAAc,CAAC,sBAAsB,GAAG,SAAS,CAAC;IACtD,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,oBAAoB,GAAG,IAAI,cAAc,EAAE,CAAC;AAElD,MAAM,qBAAqB,GAAG,IAAI,iBAAiB,EAA2B,CAAC;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,2BAA2B,CACvC,cAA8B,EAC9B,MAAU;IAEV,IAAI,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;QAClD,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAEvE,kFAAkF;YAClF,sEAAsE;YACtE,uEAAuE;YACvE,IACI,WAAW,KAAK,aAAa;gBAC7B,CAAC,UAAU;gBACX,UAAU,CAAC,GAAG;gBACd,UAAU,CAAC,GAAG;gBACd,OAAO,UAAU,CAAC,KAAK,KAAK,UAAU;gBAEtC,SAAS;YAEb,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC;YACjC,MAA2C,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;gBAC3E,OAAO,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE;oBAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;QACN,CAAC;QAED,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,aAAkD,CAAC;IAEvD,OAAO;QACH,GAAG,EAAE,CAAI,EAAW,EAAK,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QACzE,UAAU,EAAE,GAAG,EAAE;YACb,aAAa,GAAG,qBAAqB,CAAC,QAAQ,EAAE,CAAC;YACjD,qBAAqB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpD,CAAC;QACD,SAAS,EAAE,GAAG,EAAE;YACZ,qBAAqB,CAAC,SAAS,CAAC,aAAoB,CAAC,CAAC,CAAC,uFAAuF;QAClJ,CAAC;KACJ,CAAC;AACN,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,KAAK,CAAC,EAA6B,EAAE;IACnE,GAAG,CAAC,OAAO,EAAE,IAAI;QACb,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACxE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAChD,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,GAAG,CAAC,OAAO,EAAE,IAAI;QACb,MAAM,IAAI,SAAS,CACf,wBAAwB,MAAM,CAAC,IAAI,CAAC,8GAA8G,CACrJ,CAAC;IACN,CAAC;CACJ,CAAC,CAAC"}
|
package/session_pool/consts.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export declare const BLOCKED_STATUS_CODES: number[];
|
|
2
|
-
export declare const PERSIST_STATE_KEY = "
|
|
2
|
+
export declare const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";
|
|
3
3
|
export declare const MAX_POOL_SIZE = 1000;
|
|
4
4
|
//# sourceMappingURL=consts.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../src/session_pool/consts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,oBAAoB,UAAkB,CAAC;AACpD,eAAO,MAAM,iBAAiB,
|
|
1
|
+
{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../src/session_pool/consts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,oBAAoB,UAAkB,CAAC;AACpD,eAAO,MAAM,iBAAiB,+BAA+B,CAAC;AAC9D,eAAO,MAAM,aAAa,OAAO,CAAC"}
|
package/session_pool/consts.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/session_pool/consts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACpD,MAAM,CAAC,MAAM,iBAAiB,GAAG,
|
|
1
|
+
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../src/session_pool/consts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACpD,MAAM,CAAC,MAAM,iBAAiB,GAAG,4BAA4B,CAAC;AAC9D,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SessionFingerprint } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* Build a {@link SessionFingerprint} whose `platform` matches the host OS
|
|
4
|
+
* and whose `browser`/`device` are randomized within the realistic profiles for
|
|
5
|
+
* that platform. Used by {@link SessionPool} as the default fingerprint for
|
|
6
|
+
* freshly created sessions; callers can override by passing their own
|
|
7
|
+
* `fingerprint` in `sessionOptions`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function createDefaultSessionFingerprint(): SessionFingerprint;
|
|
10
|
+
//# sourceMappingURL=fingerprint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../../src/session_pool/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAuBzD;;;;;;GAMG;AACH,wBAAgB,+BAA+B,IAAI,kBAAkB,CAEpE"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* (browser, platform, device) combinations that correspond to setups people
|
|
3
|
+
* actually run. Anything not listed here (e.g. `edge` on android, `safari` on
|
|
4
|
+
* windows, `desktop` mobile platforms) is left out so a randomized default
|
|
5
|
+
* never produces a fingerprint that would itself be a giveaway.
|
|
6
|
+
*/
|
|
7
|
+
const PROFILES_BY_PLATFORM = [
|
|
8
|
+
{ browser: 'chrome', platform: 'windows', device: 'desktop' },
|
|
9
|
+
{ browser: 'firefox', platform: 'windows', device: 'desktop' },
|
|
10
|
+
{ browser: 'edge', platform: 'windows', device: 'desktop' },
|
|
11
|
+
{ browser: 'chrome', platform: 'macos', device: 'desktop' },
|
|
12
|
+
{ browser: 'firefox', platform: 'macos', device: 'desktop' },
|
|
13
|
+
{ browser: 'safari', platform: 'macos', device: 'desktop' },
|
|
14
|
+
{ browser: 'edge', platform: 'macos', device: 'desktop' },
|
|
15
|
+
{ browser: 'chrome', platform: 'linux', device: 'desktop' },
|
|
16
|
+
{ browser: 'firefox', platform: 'linux', device: 'desktop' },
|
|
17
|
+
{ browser: 'chrome', platform: 'android', device: 'mobile' },
|
|
18
|
+
{ browser: 'firefox', platform: 'android', device: 'mobile' },
|
|
19
|
+
{ browser: 'safari', platform: 'ios', device: 'mobile' },
|
|
20
|
+
];
|
|
21
|
+
/**
|
|
22
|
+
* Build a {@link SessionFingerprint} whose `platform` matches the host OS
|
|
23
|
+
* and whose `browser`/`device` are randomized within the realistic profiles for
|
|
24
|
+
* that platform. Used by {@link SessionPool} as the default fingerprint for
|
|
25
|
+
* freshly created sessions; callers can override by passing their own
|
|
26
|
+
* `fingerprint` in `sessionOptions`.
|
|
27
|
+
*/
|
|
28
|
+
export function createDefaultSessionFingerprint() {
|
|
29
|
+
return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=fingerprint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../../src/session_pool/fingerprint.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,oBAAoB,GAAG;IACzB,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;IAC7D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;IAC9D,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;IAC3D,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IAC3D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IAC5D,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IAC3D,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IACzD,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IAC3D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE;IAC5D,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC5D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE;IAC7D,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE;CAClD,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,+BAA+B;IAC3C,OAAO,EAAE,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;AAChG,CAAC"}
|
package/session_pool/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
|
package/session_pool/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
|