@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +85 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +159 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +19 -28
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +15 -15
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +21 -24
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +32 -18
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +45 -24
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +8 -5
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +7 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +3 -2
- package/index.d.ts.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/package.json +9 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +74 -10
- package/request.d.ts.map +1 -1
- package/request.js +85 -23
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +234 -0
- package/service_locator.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +26 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +36 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +65 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +101 -100
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +90 -46
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +149 -121
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +3 -1
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +3 -1
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +104 -22
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +166 -51
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +9 -9
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +13 -8
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +87 -22
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +127 -77
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +1 -3
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +2 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +3 -3
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +4 -5
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +5 -5
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +10 -7
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +91 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +236 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/utils.d.ts +47 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +57 -5
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +4 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +4 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGrD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEtE,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAI,IAAO,EAAE,KAAc;IAC7D,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;AACL,CAAC;AA6FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,CAAgB;IAEnB;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,OAA+B;QAC1D,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,MAAM;gBACF,GAAG,IAAI;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAmB,CAAC;aACrE,CAAC;QACN,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,UAAU,CACrB,OAA+B,EAC/B,QAAQ,GAAG,+BAA+B;QAE1C,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,UAAU,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClG,IAAI,UAAU,IAAI,CAAC;gBAAE,MAAM;YAE3B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YAClF,MAAM,IAAI,CAAC;YAEX,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM;YACtF,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,UAAkC,EAAE;QACvC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC3C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,UAAkC,EAAE;QACxC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;YAChD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACzB,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAE1E,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/E,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC;YACxD,cAAc,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI;SACjF,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
package/storages/index.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
7
8
|
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
-
export * from './
|
|
9
|
+
export * from './storage_instance_manager.js';
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.d.ts.map
|
package/storages/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
|
package/storages/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
7
8
|
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
-
export * from './
|
|
9
|
+
export * from './storage_instance_manager.js';
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.js.map
|
package/storages/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { Dictionary,
|
|
1
|
+
import type { Dictionary, KeyValueStoreClient } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { Awaitable } from '../typedefs.js';
|
|
4
|
-
import type {
|
|
4
|
+
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
5
|
+
import type { StorageOpenOptions } from './utils.js';
|
|
5
6
|
/**
|
|
6
7
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
7
8
|
*
|
|
@@ -70,7 +71,6 @@ export declare class KeyValueStore {
|
|
|
70
71
|
readonly config: Configuration;
|
|
71
72
|
readonly id: string;
|
|
72
73
|
readonly name?: string;
|
|
73
|
-
readonly storageObject?: Record<string, unknown>;
|
|
74
74
|
private readonly client;
|
|
75
75
|
private persistStateEventStarted;
|
|
76
76
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
@@ -152,6 +152,8 @@ export declare class KeyValueStore {
|
|
|
152
152
|
recordExists(key: string): Promise<boolean>;
|
|
153
153
|
getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
|
|
154
154
|
private ensurePersistStateEvent;
|
|
155
|
+
private fetchKeyValuePages;
|
|
156
|
+
private fetchKeyPages;
|
|
155
157
|
/**
|
|
156
158
|
* Saves or deletes a record in the key-value store.
|
|
157
159
|
* The function returns a promise that resolves once the record has been saved or deleted.
|
|
@@ -207,7 +209,7 @@ export declare class KeyValueStore {
|
|
|
207
209
|
* Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
|
|
208
210
|
* Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
|
|
209
211
|
* is the record key, `index` is a zero-based index of the key in the current iteration
|
|
210
|
-
*
|
|
212
|
+
* and `info` is an object that contains a single property `size`
|
|
211
213
|
* indicating size of the record in bytes.
|
|
212
214
|
*
|
|
213
215
|
* If the `iteratee` function returns a Promise then it is awaited before the next call.
|
|
@@ -225,11 +227,99 @@ export declare class KeyValueStore {
|
|
|
225
227
|
* @param [options] All `forEachKey()` parameters.
|
|
226
228
|
*/
|
|
227
229
|
forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
|
|
228
|
-
|
|
230
|
+
/**
|
|
231
|
+
* Returns key-value store keys.
|
|
232
|
+
*
|
|
233
|
+
* When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
|
|
234
|
+
* When used as an async iterable (`for await...of`), iterates over all keys across pages
|
|
235
|
+
* without loading everything into memory at once.
|
|
236
|
+
*
|
|
237
|
+
* **Example usage:**
|
|
238
|
+
* ```javascript
|
|
239
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
240
|
+
*
|
|
241
|
+
* // Iterate over all keys (memory-efficient for large stores)
|
|
242
|
+
* for await (const key of keyValueStore.keys()) {
|
|
243
|
+
* console.log(key);
|
|
244
|
+
* }
|
|
245
|
+
*
|
|
246
|
+
* // Or fetch all keys at once
|
|
247
|
+
* const allKeys = await keyValueStore.keys();
|
|
248
|
+
* console.log(allKeys);
|
|
249
|
+
* ```
|
|
250
|
+
*
|
|
251
|
+
* @param options Options for the iteration.
|
|
252
|
+
*/
|
|
253
|
+
keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
|
|
254
|
+
/**
|
|
255
|
+
* Returns key-value store values.
|
|
256
|
+
*
|
|
257
|
+
* When awaited (`await store.values()`), returns all values as a flat `T[]` array.
|
|
258
|
+
* When used as an async iterable (`for await...of`), iterates over all values across pages
|
|
259
|
+
* without loading everything into memory at once.
|
|
260
|
+
*
|
|
261
|
+
* **Example usage:**
|
|
262
|
+
* ```javascript
|
|
263
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
264
|
+
*
|
|
265
|
+
* // Iterate over all values (memory-efficient for large stores)
|
|
266
|
+
* for await (const value of keyValueStore.values()) {
|
|
267
|
+
* console.log(value);
|
|
268
|
+
* }
|
|
269
|
+
*
|
|
270
|
+
* // Or fetch all values at once
|
|
271
|
+
* const allValues = await keyValueStore.values();
|
|
272
|
+
* console.log(allValues);
|
|
273
|
+
* ```
|
|
274
|
+
*
|
|
275
|
+
* @param options Options for the iteration.
|
|
276
|
+
*/
|
|
277
|
+
values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
|
|
278
|
+
/**
|
|
279
|
+
* Returns key-value store entries (key-value pairs).
|
|
280
|
+
*
|
|
281
|
+
* When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
|
|
282
|
+
* When used as an async iterable (`for await...of`), iterates over all entries across pages
|
|
283
|
+
* without loading everything into memory at once.
|
|
284
|
+
*
|
|
285
|
+
* **Example usage:**
|
|
286
|
+
* ```javascript
|
|
287
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
288
|
+
*
|
|
289
|
+
* // Iterate over all entries (memory-efficient for large stores)
|
|
290
|
+
* for await (const [key, value] of keyValueStore.entries()) {
|
|
291
|
+
* console.log(`${key}: ${value}`);
|
|
292
|
+
* }
|
|
293
|
+
*
|
|
294
|
+
* // Or fetch all entries at once
|
|
295
|
+
* const allEntries = await keyValueStore.entries();
|
|
296
|
+
* console.log(allEntries);
|
|
297
|
+
* ```
|
|
298
|
+
*
|
|
299
|
+
* @param options Options for the iteration.
|
|
300
|
+
*/
|
|
301
|
+
entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
|
|
302
|
+
/**
|
|
303
|
+
* Default async iterator for the key-value store, iterating over entries (key-value pairs).
|
|
304
|
+
* Allows using the store directly in a `for await...of` loop.
|
|
305
|
+
*
|
|
306
|
+
* **Example usage:**
|
|
307
|
+
* ```javascript
|
|
308
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
309
|
+
* for await (const [key, value] of keyValueStore) {
|
|
310
|
+
* console.log(`${key}: ${value}`);
|
|
311
|
+
* }
|
|
312
|
+
* ```
|
|
313
|
+
*/
|
|
314
|
+
[Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
|
|
229
315
|
/**
|
|
230
316
|
* Returns a file URL for the given key.
|
|
317
|
+
*
|
|
318
|
+
* If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
|
|
319
|
+
*
|
|
320
|
+
* @param key The key of the record to generate the public URL for.
|
|
231
321
|
*/
|
|
232
|
-
getPublicUrl(key: string): string
|
|
322
|
+
getPublicUrl(key: string): Promise<string | undefined>;
|
|
233
323
|
/**
|
|
234
324
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
235
325
|
*
|
|
@@ -239,12 +329,13 @@ export declare class KeyValueStore {
|
|
|
239
329
|
*
|
|
240
330
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
241
331
|
*
|
|
242
|
-
* @param [
|
|
243
|
-
* ID or name of the key-value store to be opened. If
|
|
244
|
-
*
|
|
332
|
+
* @param [identifier]
|
|
333
|
+
* ID or name of the key-value store to be opened. If a string is provided, it will first be
|
|
334
|
+
* looked up as an ID; if no such storage exists, it will be treated as a name.
|
|
335
|
+
* If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
|
|
245
336
|
* @param [options] Storage manager options.
|
|
246
337
|
*/
|
|
247
|
-
static open(
|
|
338
|
+
static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
|
|
248
339
|
/**
|
|
249
340
|
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
250
341
|
*
|
|
@@ -380,27 +471,18 @@ export interface KeyConsumer {
|
|
|
380
471
|
export interface KeyValueStoreOptions {
|
|
381
472
|
id: string;
|
|
382
473
|
name?: string;
|
|
383
|
-
client:
|
|
384
|
-
storageObject?: Record<string, unknown>;
|
|
474
|
+
client: KeyValueStoreClient;
|
|
385
475
|
}
|
|
386
476
|
export interface RecordOptions {
|
|
387
477
|
/**
|
|
388
478
|
* Specifies a custom MIME content type of the record.
|
|
389
479
|
*/
|
|
390
480
|
contentType?: string;
|
|
391
|
-
/**
|
|
392
|
-
* Specifies a custom timeout for the `set-record` API call, in seconds.
|
|
393
|
-
*/
|
|
394
|
-
timeoutSecs?: number;
|
|
395
|
-
/**
|
|
396
|
-
* If set to `true`, the `set-record` API call will not be retried if it times out.
|
|
397
|
-
*/
|
|
398
|
-
doNotRetryTimeouts?: boolean;
|
|
399
481
|
}
|
|
400
482
|
export interface KeyValueStoreIteratorOptions {
|
|
401
483
|
/**
|
|
402
|
-
*
|
|
484
|
+
* If set, only keys that start with this prefix are returned.
|
|
403
485
|
*/
|
|
404
|
-
|
|
486
|
+
prefix?: string;
|
|
405
487
|
}
|
|
406
488
|
//# sourceMappingURL=key_value_store.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAyB,MAAM,gBAAgB,CAAC;AAO7F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAOrD;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAclB,QAAQ,CAAC,MAAM;IAbnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;YAsBhB,kBAAkB;YAclB,aAAa;IAc5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8D3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAS3F;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAShG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,CAAC,GAAG,OAAO,EACf,OAAO,GAAE,4BAAiC,GAC3C,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;IAStD;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAI1F;;;;;;OAMG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAI5D;;;;;;;;;;;;;;OAcG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,aAAa,CAAC;IAyBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,mBAAmB,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -3,12 +3,14 @@ import { join } from 'node:path';
|
|
|
3
3
|
import JSON5 from 'json5';
|
|
4
4
|
import ow, { ArgumentError } from 'ow';
|
|
5
5
|
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
|
|
6
|
-
import log from '@apify/log';
|
|
7
6
|
import { jsonStringifyExtended } from '@apify/utilities';
|
|
8
7
|
import { Configuration } from '../configuration.js';
|
|
8
|
+
import { serviceLocator } from '../service_locator.js';
|
|
9
9
|
import { checkStorageAccess } from './access_checking.js';
|
|
10
|
-
import {
|
|
11
|
-
import { purgeDefaultStorages } from './utils.js';
|
|
10
|
+
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
11
|
+
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
12
|
+
/** @internal */
|
|
13
|
+
const KVS_KEYS_DEFAULT_LIMIT = 1000;
|
|
12
14
|
/**
|
|
13
15
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
14
16
|
*
|
|
@@ -97,7 +99,6 @@ export class KeyValueStore {
|
|
|
97
99
|
config;
|
|
98
100
|
id;
|
|
99
101
|
name;
|
|
100
|
-
storageObject;
|
|
101
102
|
client;
|
|
102
103
|
persistStateEventStarted = false;
|
|
103
104
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
@@ -109,8 +110,7 @@ export class KeyValueStore {
|
|
|
109
110
|
this.config = config;
|
|
110
111
|
this.id = options.id;
|
|
111
112
|
this.name = options.name;
|
|
112
|
-
this.
|
|
113
|
-
this.client = options.client.keyValueStore(this.id);
|
|
113
|
+
this.client = options.client;
|
|
114
114
|
}
|
|
115
115
|
/**
|
|
116
116
|
* Gets a value from the key-value store.
|
|
@@ -147,7 +147,7 @@ export class KeyValueStore {
|
|
|
147
147
|
async getValue(key, defaultValue) {
|
|
148
148
|
checkStorageAccess();
|
|
149
149
|
ow(key, ow.string.nonEmpty);
|
|
150
|
-
const record = await this.client.
|
|
150
|
+
const record = await this.client.getValue(key);
|
|
151
151
|
return record?.value ?? defaultValue ?? null;
|
|
152
152
|
}
|
|
153
153
|
/**
|
|
@@ -182,21 +182,36 @@ export class KeyValueStore {
|
|
|
182
182
|
if (this.persistStateEventStarted) {
|
|
183
183
|
return;
|
|
184
184
|
}
|
|
185
|
-
|
|
186
|
-
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
187
|
-
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
188
|
-
this.config.getEventManager().on('persistState', async () => {
|
|
185
|
+
serviceLocator.getEventManager().on('persistState', async () => {
|
|
189
186
|
const promises = [];
|
|
190
187
|
for (const [key, value] of this.cache) {
|
|
191
|
-
promises.push(this.setValue(key, value
|
|
192
|
-
timeoutSecs,
|
|
193
|
-
doNotRetryTimeouts: true,
|
|
194
|
-
}).catch((error) => log.warning(`Failed to persist the state value to ${key}`, { error })));
|
|
188
|
+
promises.push(this.setValue(key, value).catch((error) => serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error })));
|
|
195
189
|
}
|
|
196
190
|
await Promise.all(promises);
|
|
197
191
|
});
|
|
198
192
|
this.persistStateEventStarted = true;
|
|
199
193
|
}
|
|
194
|
+
async *fetchKeyValuePages(options, mapRecord) {
|
|
195
|
+
for await (const page of this.fetchKeyPages(options)) {
|
|
196
|
+
const results = [];
|
|
197
|
+
for (const item of page) {
|
|
198
|
+
const record = await this.client.getValue(item.key);
|
|
199
|
+
if (record)
|
|
200
|
+
results.push(mapRecord(item.key, record.value));
|
|
201
|
+
}
|
|
202
|
+
yield results;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
async *fetchKeyPages(options, limit = KVS_KEYS_DEFAULT_LIMIT) {
|
|
206
|
+
let exclusiveStartKey;
|
|
207
|
+
while (true) {
|
|
208
|
+
const items = await this.client.listKeys({ ...options, exclusiveStartKey, limit });
|
|
209
|
+
yield items;
|
|
210
|
+
if (items.length < limit)
|
|
211
|
+
break;
|
|
212
|
+
exclusiveStartKey = items[items.length - 1].key;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
200
215
|
/**
|
|
201
216
|
* Saves or deletes a record in the key-value store.
|
|
202
217
|
* The function returns a promise that resolves once the record has been saved or deleted.
|
|
@@ -254,8 +269,6 @@ export class KeyValueStore {
|
|
|
254
269
|
}
|
|
255
270
|
ow(options, ow.object.exactShape({
|
|
256
271
|
contentType: ow.optional.string.nonEmpty,
|
|
257
|
-
timeoutSecs: ow.optional.number,
|
|
258
|
-
doNotRetryTimeouts: ow.optional.boolean,
|
|
259
272
|
}));
|
|
260
273
|
// Make copy of options, don't update what user passed.
|
|
261
274
|
const optionsCopy = { ...options };
|
|
@@ -277,15 +290,12 @@ export class KeyValueStore {
|
|
|
277
290
|
}
|
|
278
291
|
// In this case delete the record.
|
|
279
292
|
if (value === null)
|
|
280
|
-
return this.client.
|
|
293
|
+
return this.client.deleteValue(key);
|
|
281
294
|
value = maybeStringify(value, optionsCopy);
|
|
282
|
-
return this.client.
|
|
295
|
+
return this.client.setValue({
|
|
283
296
|
key,
|
|
284
297
|
value,
|
|
285
298
|
contentType: optionsCopy.contentType,
|
|
286
|
-
}, {
|
|
287
|
-
timeoutSecs: optionsCopy.timeoutSecs,
|
|
288
|
-
doNotRetryTimeouts: optionsCopy.doNotRetryTimeouts,
|
|
289
299
|
});
|
|
290
300
|
}
|
|
291
301
|
/**
|
|
@@ -294,9 +304,8 @@ export class KeyValueStore {
|
|
|
294
304
|
*/
|
|
295
305
|
async drop() {
|
|
296
306
|
checkStorageAccess();
|
|
297
|
-
await this.client.
|
|
298
|
-
|
|
299
|
-
manager.closeStorage(this);
|
|
307
|
+
await this.client.drop();
|
|
308
|
+
serviceLocator.getStorageInstanceManager().removeFromCache(this);
|
|
300
309
|
}
|
|
301
310
|
/** @internal */
|
|
302
311
|
clearCache() {
|
|
@@ -307,7 +316,7 @@ export class KeyValueStore {
|
|
|
307
316
|
* Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
|
|
308
317
|
* Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
|
|
309
318
|
* is the record key, `index` is a zero-based index of the key in the current iteration
|
|
310
|
-
*
|
|
319
|
+
* and `info` is an object that contains a single property `size`
|
|
311
320
|
* indicating size of the record in bytes.
|
|
312
321
|
*
|
|
313
322
|
* If the `iteratee` function returns a Promise then it is awaited before the next call.
|
|
@@ -326,29 +335,131 @@ export class KeyValueStore {
|
|
|
326
335
|
*/
|
|
327
336
|
async forEachKey(iteratee, options = {}) {
|
|
328
337
|
checkStorageAccess();
|
|
329
|
-
return this._forEachKey(iteratee, options);
|
|
330
|
-
}
|
|
331
|
-
async _forEachKey(iteratee, options = {}, index = 0) {
|
|
332
|
-
const { exclusiveStartKey } = options;
|
|
333
338
|
ow(iteratee, ow.function);
|
|
334
339
|
ow(options, ow.object.exactShape({
|
|
335
|
-
|
|
340
|
+
prefix: ow.optional.string,
|
|
336
341
|
}));
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
342
|
+
let index = 0;
|
|
343
|
+
for await (const page of this.fetchKeyPages(options)) {
|
|
344
|
+
for (const item of page) {
|
|
345
|
+
await iteratee(item.key, index++, { size: item.size });
|
|
346
|
+
}
|
|
341
347
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Returns key-value store keys.
|
|
351
|
+
*
|
|
352
|
+
* When awaited (`await store.keys()`), returns all keys as a flat `string[]` array.
|
|
353
|
+
* When used as an async iterable (`for await...of`), iterates over all keys across pages
|
|
354
|
+
* without loading everything into memory at once.
|
|
355
|
+
*
|
|
356
|
+
* **Example usage:**
|
|
357
|
+
* ```javascript
|
|
358
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
359
|
+
*
|
|
360
|
+
* // Iterate over all keys (memory-efficient for large stores)
|
|
361
|
+
* for await (const key of keyValueStore.keys()) {
|
|
362
|
+
* console.log(key);
|
|
363
|
+
* }
|
|
364
|
+
*
|
|
365
|
+
* // Or fetch all keys at once
|
|
366
|
+
* const allKeys = await keyValueStore.keys();
|
|
367
|
+
* console.log(allKeys);
|
|
368
|
+
* ```
|
|
369
|
+
*
|
|
370
|
+
* @param options Options for the iteration.
|
|
371
|
+
*/
|
|
372
|
+
keys(options = {}) {
|
|
373
|
+
checkStorageAccess();
|
|
374
|
+
return createDualIterable({
|
|
375
|
+
createPages: () => this.fetchKeyPages(options),
|
|
376
|
+
extractItems: (page) => page.map((item) => item.key),
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Returns key-value store values.
|
|
381
|
+
*
|
|
382
|
+
* When awaited (`await store.values()`), returns all values as a flat `T[]` array.
|
|
383
|
+
* When used as an async iterable (`for await...of`), iterates over all values across pages
|
|
384
|
+
* without loading everything into memory at once.
|
|
385
|
+
*
|
|
386
|
+
* **Example usage:**
|
|
387
|
+
* ```javascript
|
|
388
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
389
|
+
*
|
|
390
|
+
* // Iterate over all values (memory-efficient for large stores)
|
|
391
|
+
* for await (const value of keyValueStore.values()) {
|
|
392
|
+
* console.log(value);
|
|
393
|
+
* }
|
|
394
|
+
*
|
|
395
|
+
* // Or fetch all values at once
|
|
396
|
+
* const allValues = await keyValueStore.values();
|
|
397
|
+
* console.log(allValues);
|
|
398
|
+
* ```
|
|
399
|
+
*
|
|
400
|
+
* @param options Options for the iteration.
|
|
401
|
+
*/
|
|
402
|
+
values(options = {}) {
|
|
403
|
+
checkStorageAccess();
|
|
404
|
+
return createDualIterable({
|
|
405
|
+
createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
|
|
406
|
+
extractItems: (page) => page,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Returns key-value store entries (key-value pairs).
|
|
411
|
+
*
|
|
412
|
+
* When awaited (`await store.entries()`), returns all entries as a flat `[key, value][]` array.
|
|
413
|
+
* When used as an async iterable (`for await...of`), iterates over all entries across pages
|
|
414
|
+
* without loading everything into memory at once.
|
|
415
|
+
*
|
|
416
|
+
* **Example usage:**
|
|
417
|
+
* ```javascript
|
|
418
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
419
|
+
*
|
|
420
|
+
* // Iterate over all entries (memory-efficient for large stores)
|
|
421
|
+
* for await (const [key, value] of keyValueStore.entries()) {
|
|
422
|
+
* console.log(`${key}: ${value}`);
|
|
423
|
+
* }
|
|
424
|
+
*
|
|
425
|
+
* // Or fetch all entries at once
|
|
426
|
+
* const allEntries = await keyValueStore.entries();
|
|
427
|
+
* console.log(allEntries);
|
|
428
|
+
* ```
|
|
429
|
+
*
|
|
430
|
+
* @param options Options for the iteration.
|
|
431
|
+
*/
|
|
432
|
+
entries(options = {}) {
|
|
433
|
+
checkStorageAccess();
|
|
434
|
+
return createDualIterable({
|
|
435
|
+
createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
|
|
436
|
+
extractItems: (page) => page,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Default async iterator for the key-value store, iterating over entries (key-value pairs).
|
|
441
|
+
* Allows using the store directly in a `for await...of` loop.
|
|
442
|
+
*
|
|
443
|
+
* **Example usage:**
|
|
444
|
+
* ```javascript
|
|
445
|
+
* const keyValueStore = await KeyValueStore.open();
|
|
446
|
+
* for await (const [key, value] of keyValueStore) {
|
|
447
|
+
* console.log(`${key}: ${value}`);
|
|
448
|
+
* }
|
|
449
|
+
* ```
|
|
450
|
+
*/
|
|
451
|
+
async *[Symbol.asyncIterator]() {
|
|
452
|
+
yield* this.entries();
|
|
345
453
|
}
|
|
346
454
|
/**
|
|
347
455
|
* Returns a file URL for the given key.
|
|
456
|
+
*
|
|
457
|
+
* If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
|
|
458
|
+
*
|
|
459
|
+
* @param key The key of the record to generate the public URL for.
|
|
348
460
|
*/
|
|
349
|
-
getPublicUrl(key) {
|
|
350
|
-
|
|
351
|
-
return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`;
|
|
461
|
+
async getPublicUrl(key) {
|
|
462
|
+
return this.client.getPublicUrl(key);
|
|
352
463
|
}
|
|
353
464
|
/**
|
|
354
465
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
@@ -359,23 +470,27 @@ export class KeyValueStore {
|
|
|
359
470
|
*
|
|
360
471
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
361
472
|
*
|
|
362
|
-
* @param [
|
|
363
|
-
* ID or name of the key-value store to be opened. If
|
|
364
|
-
*
|
|
473
|
+
* @param [identifier]
|
|
474
|
+
* ID or name of the key-value store to be opened. If a string is provided, it will first be
|
|
475
|
+
* looked up as an ID; if no such storage exists, it will be treated as a name.
|
|
476
|
+
* If `null` or `undefined`, the function returns the default key-value store associated with the crawler run.
|
|
365
477
|
* @param [options] Storage manager options.
|
|
366
478
|
*/
|
|
367
|
-
static async open(
|
|
479
|
+
static async open(identifier, options = {}) {
|
|
368
480
|
checkStorageAccess();
|
|
369
|
-
ow(storeIdOrName, ow.optional.any(ow.string, ow.null));
|
|
370
481
|
ow(options, ow.object.exactShape({
|
|
371
482
|
config: ow.optional.object.instanceOf(Configuration),
|
|
372
483
|
storageClient: ow.optional.object,
|
|
373
484
|
}));
|
|
374
485
|
options.config ??= Configuration.getGlobalConfig();
|
|
375
|
-
options.storageClient
|
|
376
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, client
|
|
377
|
-
const
|
|
378
|
-
return
|
|
486
|
+
const client = options.storageClient ?? serviceLocator.getStorageClient();
|
|
487
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
|
|
488
|
+
const resolved = await resolveStorageIdentifier(identifier, client, 'KeyValueStore');
|
|
489
|
+
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
490
|
+
...resolved,
|
|
491
|
+
clientOpener: () => client.createKeyValueStoreClient(resolved),
|
|
492
|
+
clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
|
|
493
|
+
});
|
|
379
494
|
}
|
|
380
495
|
/**
|
|
381
496
|
* Gets a value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
@@ -479,7 +594,7 @@ export class KeyValueStore {
|
|
|
479
594
|
*/
|
|
480
595
|
static async getInput() {
|
|
481
596
|
const store = await this.open();
|
|
482
|
-
const inputKey = store.config.
|
|
597
|
+
const inputKey = store.config.inputKey;
|
|
483
598
|
const cwd = process.cwd();
|
|
484
599
|
const possibleExtensions = ['', '.json', '.txt'];
|
|
485
600
|
// Attempt to read input from root file instead of key-value store
|