@crawlee/core 4.0.0-beta.3 → 4.0.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/autoscaling/autoscaled_pool.d.ts +1 -3
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +1 -7
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +0 -10
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +15 -28
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +4 -77
- package/configuration.d.ts.map +1 -1
- package/configuration.js +5 -101
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -6
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +61 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +99 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +12 -20
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +13 -13
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +20 -16
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +30 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +39 -21
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +24 -7
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +66 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +18 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +29 -0
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +7 -4
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +5 -7
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +26 -37
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +4 -2
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +6 -7
- package/proxy_configuration.d.ts +16 -93
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +15 -51
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +122 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +138 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +39 -5
- package/request.d.ts.map +1 -1
- package/request.js +52 -12
- package/request.js.map +1 -1
- package/service_locator.d.ts +139 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +252 -0
- package/service_locator.js.map +1 -0
- package/session_pool/session.d.ts +7 -20
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +16 -4
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +12 -7
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +22 -11
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +7 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +14 -4
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +13 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +16 -10
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +10 -6
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +74 -7
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +89 -52
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -1
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -2
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +6 -5
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +10 -8
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +12 -22
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -3
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAGlC,OAAO,GAAG,MAAM,YAAY,CAAC;AAG7B,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAgBxC,OAAO,EACH,qBAAqB,EACrB,6BAA6B,EAC7B,oCAAoC,EACpC,iCAAiC,EACjC,oBAAoB,EACpB,8BAA8B,GACjC,MAAM,aAAa,CAAC;AAwKrB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAN,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,KAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAC9B,OAOC;IAED,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,kHAAkH;SACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,EAAE,CACE,OAAc,EACd,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC;QACrD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC;QACtD,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QACtC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,EAC3B,aAAa,EACb,gBAAgB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,GAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAK,eAAe,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,KAAK,UAAU,qBAAqB,CAChC,eAAwE,EACxE,MAA4B;QAE5B,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,OAAO,CAAC,GAAG,CACb,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,gBAAgB,CAAC;oBACpB,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,MAAM;iBAC1C,CAAkB,CAAC;YACxB,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,eAAe,GAAqB,EAAE,CAAC;QAE7C,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,UAAU,sBAAsB;QACjC,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,4GAA4G;QAC5G,qFAAqF;QACrF,IAAI,eAAiC,CAAC;QACtC,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,eAAe,GAAG,8BAA8B,CAC5C,cAAc,EACd,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,EACxE,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,+CAA+C;YAC/C,MAAM,iBAAiB,GAAG,8BAA8B,CACpD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;YACF,0EAA0E;YAC1E,eAAe,GAAG,8BAA8B,CAC5C,iBAAiB,EACjB,uBAAuB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,EACxE,EAAE,EACF,OAAO,CAAC,QAAQ,EAChB,CAAC,GAAG,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CACrC,CAAC;QACN,CAAC;QAED,MAAM,qBAAqB,CACvB,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EACvC,SAAS,CACZ,CAAC;QAEF,0FAA0F;QAC1F,IAAI,wBAAwB,EAAE,CAAC;YAC3B,MAAM,kBAAkB,GAAqB,EAAE,CAAC;YAChD,eAAe,GAAG,qBAAqB,CAAC,eAAe,EAAE,wBAAwB,EAAE,CAAC,CAAC,EAAE,EAAE,CACrF,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAC7B,CAAC;YACF,MAAM,qBAAqB,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACjE,CAAC;QAED,kEAAkE;QAClE,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAE9C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,qBAAqB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAC;QACnE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import type { Awaitable } from '@crawlee/types';
|
|
1
2
|
import type { RequestOptions } from '../request.js';
|
|
2
|
-
import { Request } from '../request.js';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links.js';
|
|
4
4
|
export { tryAbsoluteURL } from '@crawlee/utils';
|
|
5
5
|
export type UrlPatternObject = {
|
|
@@ -18,6 +18,11 @@ export type RegExpObject = {
|
|
|
18
18
|
regexp: RegExp;
|
|
19
19
|
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
|
|
20
20
|
export type RegExpInput = RegExp | RegExpObject;
|
|
21
|
+
export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
|
|
22
|
+
export type SkippedRequestCallback = (args: {
|
|
23
|
+
url: string;
|
|
24
|
+
reason: SkippedRequestReason;
|
|
25
|
+
}) => Awaitable<void>;
|
|
21
26
|
/**
|
|
22
27
|
* @ignore
|
|
23
28
|
*/
|
|
@@ -45,23 +50,35 @@ export declare function validateGlobPattern(glob: string): string;
|
|
|
45
50
|
*/
|
|
46
51
|
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
|
|
47
52
|
/**
|
|
53
|
+
* Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
|
|
54
|
+
* from the first matching pattern into each RequestOptions entry.
|
|
55
|
+
*
|
|
56
|
+
* When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
|
|
48
57
|
* @ignore
|
|
49
58
|
*/
|
|
50
|
-
export declare function
|
|
51
|
-
export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
|
|
59
|
+
export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueLinksOptions['strategy'], onSkippedUrl?: (url: string) => void): RequestOptions[];
|
|
52
60
|
/**
|
|
53
61
|
* @ignore
|
|
54
62
|
*/
|
|
55
63
|
export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'>): RequestOptions[];
|
|
56
64
|
/**
|
|
57
|
-
* Takes
|
|
58
|
-
* {@link enqueueLinks} to modify
|
|
65
|
+
* Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
|
|
66
|
+
* by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
|
|
59
67
|
*/
|
|
60
68
|
export interface RequestTransform {
|
|
61
69
|
/**
|
|
62
70
|
* @param original Request options to be modified.
|
|
63
|
-
* @returns The modified request options to enqueue
|
|
71
|
+
* @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
|
|
72
|
+
* or a falsy value / `'skip'` to exclude the request from the queue.
|
|
64
73
|
*/
|
|
65
|
-
(original: RequestOptions): RequestOptions | false | undefined | null;
|
|
74
|
+
(original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
|
|
66
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* Applies a {@link RequestTransform} function to a list of request options.
|
|
78
|
+
* Options for which the transform returns a falsy value are removed from the list.
|
|
79
|
+
* @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
|
|
80
|
+
* @ignore
|
|
81
|
+
* @internal
|
|
82
|
+
*/
|
|
83
|
+
export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];
|
|
67
84
|
//# sourceMappingURL=shared.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAKhD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD,MAAM,MAAM,oBAAoB,GAC1B,WAAW,GACX,OAAO,GACP,cAAc,GACd,SAAS,GACT,WAAW,GACX,UAAU,GACV,OAAO,CAAC;AAEd,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAA;CAAE,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9G;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC1C,cAAc,EAAE,cAAc,EAAE,EAChC,eAAe,EAAE,gBAAgB,EAAE,GAAG,SAAS,EAC/C,eAAe,GAAE,gBAAgB,EAAO,EACxC,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,EAC1C,YAAY,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GACrC,cAAc,EAAE,CA4BlB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EACtD,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAmBD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;;OAIG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,GAAG,WAAW,CAAC;CAChG;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACjC,cAAc,EAAE,cAAc,EAAE,EAChC,WAAW,EAAE,gBAAgB,EAC7B,SAAS,CAAC,EAAE,CAAC,cAAc,EAAE,cAAc,KAAK,IAAI,GACrD,cAAc,EAAE,CAkBlB"}
|
package/enqueue_links/shared.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import {
|
|
2
|
+
import { Minimatch } from 'minimatch';
|
|
3
3
|
import { purlToRegExp } from '@apify/pseudo_url';
|
|
4
|
-
import { Request } from '../request.js';
|
|
5
4
|
export { tryAbsoluteURL } from '@crawlee/utils';
|
|
6
5
|
const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
|
|
7
6
|
/**
|
|
@@ -110,51 +109,37 @@ export function constructRegExpObjectsFromRegExps(regexps) {
|
|
|
110
109
|
});
|
|
111
110
|
}
|
|
112
111
|
/**
|
|
112
|
+
* Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
|
|
113
|
+
* from the first matching pattern into each RequestOptions entry.
|
|
114
|
+
*
|
|
115
|
+
* When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
|
|
113
116
|
* @ignore
|
|
114
117
|
*/
|
|
115
|
-
export function
|
|
118
|
+
export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
|
|
119
|
+
const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
|
|
120
|
+
const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
|
|
116
121
|
return requestOptions
|
|
117
|
-
.map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
|
|
118
122
|
.filter(({ url }) => {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
}
|
|
123
|
+
const matchesExclude = excludeMatchers.some(({ match }) => match(url));
|
|
124
|
+
if (matchesExclude) {
|
|
125
|
+
onSkippedUrl?.(url);
|
|
126
|
+
}
|
|
127
|
+
return !matchesExclude;
|
|
123
128
|
})
|
|
124
|
-
.map((
|
|
125
|
-
if (!
|
|
126
|
-
return
|
|
127
|
-
}
|
|
128
|
-
for (const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const request = typeof opts === 'string'
|
|
132
|
-
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
|
|
133
|
-
: { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
|
|
134
|
-
return new Request(request);
|
|
129
|
+
.map((opts) => {
|
|
130
|
+
if (!includeMatchers) {
|
|
131
|
+
return { ...opts, enqueueStrategy: strategy };
|
|
132
|
+
}
|
|
133
|
+
for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
|
|
134
|
+
if (match(opts.url)) {
|
|
135
|
+
return { ...opts, ...patternOptions, enqueueStrategy: strategy };
|
|
135
136
|
}
|
|
136
137
|
}
|
|
137
138
|
// didn't match any positive pattern
|
|
139
|
+
onSkippedUrl?.(opts.url);
|
|
138
140
|
return null;
|
|
139
141
|
})
|
|
140
|
-
.filter((
|
|
141
|
-
}
|
|
142
|
-
export function filterRequestsByPatterns(requests, patterns) {
|
|
143
|
-
if (!patterns?.length) {
|
|
144
|
-
return requests;
|
|
145
|
-
}
|
|
146
|
-
const filtered = [];
|
|
147
|
-
for (const request of requests) {
|
|
148
|
-
for (const urlPatternObject of patterns) {
|
|
149
|
-
const { regexp, glob } = urlPatternObject;
|
|
150
|
-
if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
|
|
151
|
-
filtered.push(request);
|
|
152
|
-
// Break the pattern loop, as we already matched this request once
|
|
153
|
-
break;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
return filtered;
|
|
142
|
+
.filter((opts) => opts !== null);
|
|
158
143
|
}
|
|
159
144
|
/**
|
|
160
145
|
* @ignore
|
|
@@ -187,4 +172,48 @@ export function createRequestOptions(sources, options = {}) {
|
|
|
187
172
|
return requestOptions;
|
|
188
173
|
});
|
|
189
174
|
}
|
|
175
|
+
/**
|
|
176
|
+
* @ignore
|
|
177
|
+
*/
|
|
178
|
+
function createPatternObjectMatcher(urlPatternObject) {
|
|
179
|
+
const { regexp, glob } = urlPatternObject;
|
|
180
|
+
let match;
|
|
181
|
+
if (regexp) {
|
|
182
|
+
match = (url) => regexp.test(url);
|
|
183
|
+
}
|
|
184
|
+
else if (glob) {
|
|
185
|
+
const m = new Minimatch(glob, { nocase: true });
|
|
186
|
+
match = (url) => m.match(url);
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
match = () => false;
|
|
190
|
+
}
|
|
191
|
+
return { ...urlPatternObject, match };
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Applies a {@link RequestTransform} function to a list of request options.
|
|
195
|
+
* Options for which the transform returns a falsy value are removed from the list.
|
|
196
|
+
* @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
|
|
197
|
+
* @ignore
|
|
198
|
+
* @internal
|
|
199
|
+
*/
|
|
200
|
+
export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
|
|
201
|
+
return requestOptions
|
|
202
|
+
.map((opts) => {
|
|
203
|
+
const transformed = transformFn(opts);
|
|
204
|
+
if (transformed === 'skip') {
|
|
205
|
+
onSkipped?.(opts);
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
if (transformed === 'unchanged') {
|
|
209
|
+
return opts;
|
|
210
|
+
}
|
|
211
|
+
if (!transformed) {
|
|
212
|
+
onSkipped?.(opts);
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
return transformed;
|
|
216
|
+
})
|
|
217
|
+
.filter((r) => r !== null);
|
|
218
|
+
}
|
|
190
219
|
//# sourceMappingURL=shared.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAuC3C;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,8BAA8B,CAC1C,cAAgC,EAChC,eAA+C,EAC/C,kBAAsC,EAAE,EACxC,QAA0C,EAC1C,YAAoC;IAEpC,MAAM,eAAe,GAAG,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9G,OAAO,cAAc;SAChB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,IAAI,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,CAAC,cAAc,CAAC;IAC3B,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,OAAO,EAAE,GAAG,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,cAAc,EAAE,IAAI,eAAe,EAAE,CAAC;YACvE,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;YACrE,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,YAAY,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAChC,OAAsD,EACtD,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,0BAA0B,CAAC,gBAAkC;IAClE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;IAC1C,IAAI,KAAK,CAAC;IACV,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;SAAM,CAAC;QACJ,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,GAAG,gBAAgB,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC;AAeD;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACjC,cAAgC,EAChC,WAA6B,EAC7B,SAAoD;IAEpD,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YACzB,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,SAAS,EAAE,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO,WAAW,CAAC;IACvB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAuB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACxD,CAAC"}
|
package/errors.d.ts
CHANGED
|
@@ -30,4 +30,22 @@ export declare class RetryRequestError extends Error {
|
|
|
30
30
|
export declare class SessionError extends RetryRequestError {
|
|
31
31
|
constructor(message?: string);
|
|
32
32
|
}
|
|
33
|
+
export declare class ContextPipelineInterruptedError extends Error {
|
|
34
|
+
constructor(message?: string);
|
|
35
|
+
}
|
|
36
|
+
export declare class ContextPipelineInitializationError extends Error {
|
|
37
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
38
|
+
}
|
|
39
|
+
export declare class ContextPipelineCleanupError extends CriticalError {
|
|
40
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
41
|
+
}
|
|
42
|
+
export declare class RequestHandlerError extends Error {
|
|
43
|
+
constructor(error: unknown, options?: ErrorOptions);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
47
|
+
*/
|
|
48
|
+
export declare class ServiceConflictError extends Error {
|
|
49
|
+
constructor(serviceName: string, newValue: unknown, existingValue: unknown);
|
|
50
|
+
}
|
|
33
51
|
//# sourceMappingURL=errors.d.ts.map
|
package/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC5B,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED;;;;GAIG;AACH,qBAAa,YAAa,SAAQ,iBAAiB;gBACnC,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,+BAAgC,SAAQ,KAAK;gBAC1C,OAAO,CAAC,EAAE,MAAM;CAG/B;AAED,qBAAa,kCAAmC,SAAQ,KAAK;gBAC7C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,2BAA4B,SAAQ,aAAa;gBAC9C,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED,qBAAa,mBAAoB,SAAQ,KAAK;gBAC9B,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,YAAY;CAGrD;AAED;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;gBAC/B,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO;CAM7E"}
|
package/errors.js
CHANGED
|
@@ -34,4 +34,33 @@ export class SessionError extends RetryRequestError {
|
|
|
34
34
|
super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
|
+
export class ContextPipelineInterruptedError extends Error {
|
|
38
|
+
constructor(message) {
|
|
39
|
+
super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class ContextPipelineInitializationError extends Error {
|
|
43
|
+
constructor(error, options) {
|
|
44
|
+
super(undefined, { cause: error, ...options });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export class ContextPipelineCleanupError extends CriticalError {
|
|
48
|
+
constructor(error, options) {
|
|
49
|
+
super(undefined, { cause: error, ...options });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export class RequestHandlerError extends Error {
|
|
53
|
+
constructor(error, options) {
|
|
54
|
+
super(undefined, { cause: error, ...options });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Thrown when attempting to set a different service instance after one has already been retrieved.
|
|
59
|
+
*/
|
|
60
|
+
export class ServiceConflictError extends Error {
|
|
61
|
+
constructor(serviceName, newValue, existingValue) {
|
|
62
|
+
super(`Service ${serviceName} is already in use. ` +
|
|
63
|
+
`Existing value: ${existingValue}, attempted new value: ${newValue}.`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
37
66
|
//# sourceMappingURL=errors.js.map
|
package/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;CAAG;AAE/C;;;GAGG;AACH,MAAM,OAAO,aAAc,SAAQ,iBAAiB;CAAG;AAEvD;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;CAAG;AAEvD;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACxC,YAAY,OAAgB;QACxB,KAAK,CAAC,OAAO,IAAI,gDAAgD,CAAC,CAAC;IACvE,CAAC;CACJ;AAED;;;;GAIG;AACH,MAAM,OAAO,YAAa,SAAQ,iBAAiB;IAC/C,YAAY,OAAgB;QACxB,KAAK,CAAC,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;CACJ;AAED,MAAM,OAAO,+BAAgC,SAAQ,KAAK;IACtD,YAAY,OAAgB;QACxB,KAAK,CAAC,kEAAkE,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9G,CAAC;CACJ;AAED,MAAM,OAAO,kCAAmC,SAAQ,KAAK;IACzD,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,2BAA4B,SAAQ,aAAa;IAC1D,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC1C,YAAY,KAAc,EAAE,OAAsB;QAC9C,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC3C,YAAY,WAAmB,EAAE,QAAiB,EAAE,aAAsB;QACtE,KAAK,CACD,WAAW,WAAW,sBAAsB;YACxC,mBAAmB,aAAa,0BAA0B,QAAQ,GAAG,CAC5E,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import type { BetterIntervalID } from '@apify/utilities';
|
|
3
|
-
|
|
3
|
+
export interface EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `persistState` events in milliseconds. */
|
|
5
|
+
persistStateIntervalMillis: number;
|
|
6
|
+
}
|
|
4
7
|
export declare const enum EventType {
|
|
5
8
|
PERSIST_STATE = "persistState",
|
|
6
9
|
SYSTEM_INFO = "systemInfo",
|
|
@@ -14,15 +17,15 @@ interface Intervals {
|
|
|
14
17
|
systemInfo?: BetterIntervalID;
|
|
15
18
|
}
|
|
16
19
|
export declare abstract class EventManager {
|
|
17
|
-
readonly config: Configuration;
|
|
18
20
|
protected events: AsyncEventEmitter<{}>;
|
|
19
21
|
protected initialized: boolean;
|
|
20
22
|
protected intervals: Intervals;
|
|
21
23
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
22
24
|
protected log: import("@apify/log").Log;
|
|
23
|
-
|
|
25
|
+
private persistStateIntervalMillis;
|
|
26
|
+
constructor(options: EventManagerOptions);
|
|
24
27
|
/**
|
|
25
|
-
* Initializes the event manager by
|
|
28
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
26
29
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
27
30
|
*/
|
|
28
31
|
init(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAGpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,WAAW,mBAAmB;IAChC,sEAAsE;IACtE,0BAA0B,EAAE,MAAM,CAAC;CACtC;AAED,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAC9B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;IAChD,OAAO,CAAC,0BAA0B,CAAS;gBAE/B,OAAO,EAAE,mBAAmB;IAKxC;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}
|
package/events/event_manager.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import log from '@apify/log';
|
|
3
3
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
|
-
import { Configuration } from '../configuration.js';
|
|
5
4
|
export var EventType;
|
|
6
5
|
(function (EventType) {
|
|
7
6
|
EventType["PERSIST_STATE"] = "persistState";
|
|
@@ -11,28 +10,27 @@ export var EventType;
|
|
|
11
10
|
EventType["EXIT"] = "exit";
|
|
12
11
|
})(EventType || (EventType = {}));
|
|
13
12
|
export class EventManager {
|
|
14
|
-
config;
|
|
15
13
|
events = new AsyncEventEmitter();
|
|
16
14
|
initialized = false;
|
|
17
15
|
intervals = {};
|
|
18
16
|
log = log.child({ prefix: 'Events' });
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
persistStateIntervalMillis;
|
|
18
|
+
constructor(options) {
|
|
19
|
+
this.persistStateIntervalMillis = options.persistStateIntervalMillis;
|
|
21
20
|
this.events.setMaxListeners(50);
|
|
22
21
|
}
|
|
23
22
|
/**
|
|
24
|
-
* Initializes the event manager by
|
|
23
|
+
* Initializes the event manager by starting the `persistState` event interval.
|
|
25
24
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
26
25
|
*/
|
|
27
26
|
async init() {
|
|
28
27
|
if (this.initialized) {
|
|
29
28
|
return;
|
|
30
29
|
}
|
|
31
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
32
30
|
this.intervals.persistState = betterSetInterval((intervalCallback) => {
|
|
33
31
|
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
34
32
|
intervalCallback();
|
|
35
|
-
}, persistStateIntervalMillis);
|
|
33
|
+
}, this.persistStateIntervalMillis);
|
|
36
34
|
this.initialized = true;
|
|
37
35
|
}
|
|
38
36
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAO1E,MAAM,CAAN,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,KAAT,SAAS,QAM1B;AASD,MAAM,OAAgB,YAAY;IACpB,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;IACjC,WAAW,GAAG,KAAK,CAAC;IACpB,SAAS,GAAc,EAAE,CAAC;IAC1B,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACxC,0BAA0B,CAAS;IAE3C,YAAY,OAA4B;QACpC,IAAI,CAAC,0BAA0B,GAAG,OAAO,CAAC,0BAA0B,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,iBAAiB,CAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAEpC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,oCAAoC;QACpC,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;IAC/C,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,6BAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,6BAA6B,EAAE,CAAC;IACvD,CAAC;CACJ"}
|
|
@@ -1,8 +1,19 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Configuration } from '../configuration.js';
|
|
2
|
+
import { EventManager, type EventManagerOptions } from './event_manager.js';
|
|
3
|
+
export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
4
|
+
/** Interval between emitted `systemInfo` events in milliseconds. */
|
|
5
|
+
systemInfoIntervalMillis: number;
|
|
6
|
+
}
|
|
2
7
|
export declare class LocalEventManager extends EventManager {
|
|
3
|
-
private
|
|
8
|
+
private systemInfoIntervalMillis;
|
|
9
|
+
constructor(options: LocalEventManagerOptions);
|
|
10
|
+
/**
|
|
11
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
13
|
+
*/
|
|
14
|
+
static fromConfig(config?: Configuration): LocalEventManager;
|
|
4
15
|
/**
|
|
5
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
16
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
6
17
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
7
18
|
*/
|
|
8
19
|
init(): Promise<void>;
|
|
@@ -18,7 +29,6 @@ export declare class LocalEventManager extends EventManager {
|
|
|
18
29
|
* @internal
|
|
19
30
|
*/
|
|
20
31
|
isContainerizedWrapper(): Promise<boolean>;
|
|
21
|
-
private getCurrentCpuTicks;
|
|
22
32
|
/**
|
|
23
33
|
* Creates a SystemInfo object based on local metrics.
|
|
24
34
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"local_event_manager.d.ts","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAa,MAAM,oBAAoB,CAAC;AAEvF,MAAM,WAAW,wBAAyB,SAAQ,mBAAmB;IACjE,oEAAoE;IACpE,wBAAwB,EAAE,MAAM,CAAC;CACpC;AAED,qBAAa,iBAAkB,SAAQ,YAAY;IAC/C,OAAO,CAAC,wBAAwB,CAAS;gBAE7B,OAAO,EAAE,wBAAwB;IAK7C;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,iBAAiB;IAS5D;;;OAGG;IACY,IAAI;IAcnB;;OAEG;IACY,KAAK;IASpB;;OAEG;IACG,mBAAmB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAQzD;;OAEG;IACG,sBAAsB;IAK5B;;OAEG;YACW,gBAAgB;YAQhB,aAAa;YASb,gBAAgB;CAYjC"}
|
|
@@ -1,12 +1,26 @@
|
|
|
1
|
-
import os from 'node:os';
|
|
2
|
-
import { getCurrentCpuTicksV2, getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
|
|
3
1
|
import log from '@apify/log';
|
|
4
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
+
import { serviceLocator } from '../service_locator.js';
|
|
5
4
|
import { EventManager } from './event_manager.js';
|
|
6
5
|
export class LocalEventManager extends EventManager {
|
|
7
|
-
|
|
6
|
+
systemInfoIntervalMillis;
|
|
7
|
+
constructor(options) {
|
|
8
|
+
super(options);
|
|
9
|
+
this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
13
|
+
* Uses the global configuration from the service locator if none is provided.
|
|
14
|
+
*/
|
|
15
|
+
static fromConfig(config) {
|
|
16
|
+
const resolvedConfig = config ?? serviceLocator.getConfiguration();
|
|
17
|
+
return new LocalEventManager({
|
|
18
|
+
persistStateIntervalMillis: resolvedConfig.get('persistStateIntervalMillis'),
|
|
19
|
+
systemInfoIntervalMillis: resolvedConfig.get('systemInfoIntervalMillis'),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
8
22
|
/**
|
|
9
|
-
* Initializes the EventManager and sets up periodic `systemInfo`
|
|
23
|
+
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
10
24
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
11
25
|
*/
|
|
12
26
|
async init() {
|
|
@@ -14,9 +28,8 @@ export class LocalEventManager extends EventManager {
|
|
|
14
28
|
return;
|
|
15
29
|
}
|
|
16
30
|
await super.init();
|
|
17
|
-
const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
|
|
18
31
|
this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
|
|
19
|
-
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
|
|
32
|
+
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
|
|
20
33
|
}
|
|
21
34
|
/**
|
|
22
35
|
* @inheritDoc
|
|
@@ -33,7 +46,7 @@ export class LocalEventManager extends EventManager {
|
|
|
33
46
|
*/
|
|
34
47
|
async emitSystemInfoEvent(intervalCallback) {
|
|
35
48
|
const info = await this.createSystemInfo({
|
|
36
|
-
maxUsedCpuRatio:
|
|
49
|
+
maxUsedCpuRatio: serviceLocator.getConfiguration().get('maxUsedCpuRatio'),
|
|
37
50
|
});
|
|
38
51
|
this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
|
|
39
52
|
intervalCallback();
|
|
@@ -42,17 +55,8 @@ export class LocalEventManager extends EventManager {
|
|
|
42
55
|
* @internal
|
|
43
56
|
*/
|
|
44
57
|
async isContainerizedWrapper() {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
getCurrentCpuTicks() {
|
|
48
|
-
const cpus = os.cpus();
|
|
49
|
-
return cpus.reduce((acc, cpu) => {
|
|
50
|
-
const cpuTimes = Object.values(cpu.times);
|
|
51
|
-
return {
|
|
52
|
-
idle: acc.idle + cpu.times.idle,
|
|
53
|
-
total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
|
|
54
|
-
};
|
|
55
|
-
}, { idle: 0, total: 0 });
|
|
58
|
+
const { isContainerized } = await import('@crawlee/utils');
|
|
59
|
+
return serviceLocator.getConfiguration().get('containerized', await isContainerized());
|
|
56
60
|
}
|
|
57
61
|
/**
|
|
58
62
|
* Creates a SystemInfo object based on local metrics.
|
|
@@ -65,18 +69,8 @@ export class LocalEventManager extends EventManager {
|
|
|
65
69
|
};
|
|
66
70
|
}
|
|
67
71
|
async createCpuInfo(options) {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return {
|
|
71
|
-
cpuCurrentUsage: usedCpuRatio * 100,
|
|
72
|
-
isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
const ticks = this.getCurrentCpuTicks();
|
|
76
|
-
const idleTicksDelta = ticks.idle - this.previousTicks.idle;
|
|
77
|
-
const totalTicksDelta = ticks.total - this.previousTicks.total;
|
|
78
|
-
const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
|
|
79
|
-
Object.assign(this.previousTicks, ticks);
|
|
72
|
+
const { getCurrentCpuTicksV2 } = await import('@crawlee/utils');
|
|
73
|
+
const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
|
|
80
74
|
return {
|
|
81
75
|
cpuCurrentUsage: usedCpuRatio * 100,
|
|
82
76
|
isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
|
|
@@ -84,13 +78,8 @@ export class LocalEventManager extends EventManager {
|
|
|
84
78
|
}
|
|
85
79
|
async createMemoryInfo() {
|
|
86
80
|
try {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
return {
|
|
90
|
-
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
const memInfo = await getMemoryInfo();
|
|
81
|
+
const { getMemoryInfo } = await import('@crawlee/utils');
|
|
82
|
+
const memInfo = await getMemoryInfo(await this.isContainerizedWrapper());
|
|
94
83
|
return {
|
|
95
84
|
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
|
|
96
85
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"local_event_manager.js","sourceRoot":"","sources":["../../src/events/local_event_manager.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,YAAY,CAAC;AAC7B,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAI1E,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAuC,MAAM,oBAAoB,CAAC;AAOvF,MAAM,OAAO,iBAAkB,SAAQ,YAAY;IACvC,wBAAwB,CAAS;IAEzC,YAAY,OAAiC;QACzC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,UAAU,CAAC,MAAsB;QACpC,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAEnE,OAAO,IAAI,iBAAiB,CAAC;YACzB,0BAA0B,EAAE,cAAc,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAC5E,wBAAwB,EAAE,cAAc,CAAC,GAAG,CAAC,0BAA0B,CAAC;SAC3E,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,iBAAiB,CACzC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EACnC,IAAI,CAAC,wBAAwB,CAChC,CAAC;IACN,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAW,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,gBAA+B;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC;YACrC,eAAe,EAAE,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,iBAAiB,CAAC;SAC5E,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,IAAI,2CAAwB,IAAI,CAAC,CAAC;QAC9C,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QACxB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC3D,OAAO,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAoC;QAC/D,OAAO;YACH,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YACtC,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACvB,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAoC;QAC5D,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QACrF,OAAO;YACH,eAAe,EAAE,YAAY,GAAG,GAAG;YACnC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe;SAC1D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,IAAI,CAAC;YACD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YACzD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;YACzE,OAAO;gBACH,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,mBAAmB;aAC1E,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,yBAAyB,CAAC,CAAC;YACvD,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}
|