@crawlee/core 3.13.6-beta.0 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -3
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +76 -225
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/index.d.ts +3 -3
- package/autoscaling/index.d.ts.map +1 -1
- package/autoscaling/index.js +3 -6
- package/autoscaling/index.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -3
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +42 -137
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +2 -2
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +20 -55
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +4 -4
- package/configuration.d.ts.map +1 -1
- package/configuration.js +79 -126
- package/configuration.js.map +1 -1
- package/cookie_utils.js +17 -25
- package/cookie_utils.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +9 -8
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +47 -121
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +1 -1
- package/crawlers/crawler_extension.d.ts.map +1 -1
- package/crawlers/crawler_extension.js +4 -20
- package/crawlers/crawler_extension.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +3 -6
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +8 -38
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js +18 -40
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +6 -6
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +6 -9
- package/crawlers/index.js.map +1 -1
- package/crawlers/statistics.d.ts +3 -3
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +59 -167
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +2 -2
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +41 -47
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/index.d.ts +2 -2
- package/enqueue_links/index.d.ts.map +1 -1
- package/enqueue_links/index.js +2 -5
- package/enqueue_links/index.js.map +1 -1
- package/enqueue_links/shared.d.ts +4 -4
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +23 -35
- package/enqueue_links/shared.js.map +1 -1
- package/errors.js +5 -13
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +16 -45
- package/events/event_manager.js.map +1 -1
- package/events/index.d.ts +2 -2
- package/events/index.d.ts.map +1 -1
- package/events/index.js +2 -5
- package/events/index.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +15 -28
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/base-http-client.d.ts +1 -1
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +5 -8
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/form-data-like.js +1 -2
- package/http_clients/form-data-like.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +1 -1
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +6 -11
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/http_clients/index.d.ts +2 -2
- package/http_clients/index.d.ts.map +1 -1
- package/http_clients/index.js +2 -5
- package/http_clients/index.js.map +1 -1
- package/index.d.ts +16 -16
- package/index.d.ts.map +1 -1
- package/index.js +17 -22
- package/index.js.map +1 -1
- package/log.js +2 -11
- package/log.js.map +1 -1
- package/package.json +27 -33
- package/proxy_configuration.d.ts +1 -1
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +22 -78
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +2 -2
- package/request.d.ts.map +1 -1
- package/request.js +76 -147
- package/request.js.map +1 -1
- package/router.d.ts +3 -3
- package/router.d.ts.map +1 -1
- package/router.js +6 -21
- package/router.js.map +1 -1
- package/serialization.d.ts.map +1 -1
- package/serialization.js +22 -44
- package/serialization.js.map +1 -1
- package/session_pool/consts.js +3 -6
- package/session_pool/consts.js.map +1 -1
- package/session_pool/errors.js +3 -11
- package/session_pool/errors.js.map +1 -1
- package/session_pool/events.js +1 -4
- package/session_pool/events.js.map +1 -1
- package/session_pool/index.d.ts +5 -5
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +5 -8
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +3 -17
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +52 -116
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +7 -7
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +42 -126
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +4 -9
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +4 -4
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +45 -76
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +11 -11
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +11 -18
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +3 -3
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +62 -102
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +3 -3
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +83 -183
- package/storages/request_list.js.map +1 -1
- package/storages/request_provider.d.ts +5 -5
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +97 -201
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +4 -4
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +27 -40
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +4 -4
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +18 -42
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +4 -4
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +96 -181
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +3 -3
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +14 -42
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +16 -23
- package/storages/utils.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.js +2 -6
- package/typedefs.js.map +1 -1
- package/validators.js +6 -10
- package/validators.js.map +1 -1
- package/index.mjs +0 -88
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIpD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAuDvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IAuB5B;IAtBH,GAAG,CAAM;IACT,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACZ,kBAAkB,CAAW;IACtC,kBAAkB,CAAqB;IACvC,aAAa,GAAG,KAAK,CAAC;IAExB,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAEjC;;OAEG;IACH,YACI,UAA8B,EAAE,EACvB,SAAS,aAAa,CAAC,eAAe,EAAE;QAEjD,KAAK,EAAE,CAAC;QAFC,WAAM,GAAN,MAAM,CAAkC;QAIjD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvD,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,GAAG,iBAAiB,EACnC,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,kBAAkB,GAAG,oBAAoB,EACzC,GAAG,GAAG,UAAU,EAChB,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,GACJ,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GACZ,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAExB,IAAI,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAE9B,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBAClD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC;YACzB,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;YAC7C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,+CAA+C,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CACrG,CAAC;IACV,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,6BAA6B,CACnC,WAAwB,EACxB,UAA+C,EAAE;QAEjD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACxC,OAAO,IAAI,OAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;YAEnG,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B,EAAE,MAAsB;QAClE,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access_checking.d.ts","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"access_checking.d.ts","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAIhD;;GAEG;AACH,eAAO,MAAM,kBAAkB,wBAA4C,CAAC;AAE5E;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,GAAU,CAAC,EAAE,eAAe,MAAM,IAAI,EAAE,UAAU,MAAM,SAAS,CAAC,CAAC,CAAC,eAC7D,CAAC"}
|
|
@@ -1,19 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.withCheckedStorageAccess = exports.checkStorageAccess = void 0;
|
|
4
|
-
const node_async_hooks_1 = require("node:async_hooks");
|
|
5
|
-
const storage = new node_async_hooks_1.AsyncLocalStorage();
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
const storage = new AsyncLocalStorage();
|
|
6
3
|
/**
|
|
7
4
|
* Invoke a storage access checker function defined using {@link withCheckedStorageAccess} higher up in the call stack.
|
|
8
5
|
*/
|
|
9
|
-
const checkStorageAccess = () => storage.getStore()?.checkFunction();
|
|
10
|
-
exports.checkStorageAccess = checkStorageAccess;
|
|
6
|
+
export const checkStorageAccess = () => storage.getStore()?.checkFunction();
|
|
11
7
|
/**
|
|
12
8
|
* Define a storage access checker function that should be used by calls to {@link checkStorageAccess} in the callbacks.
|
|
13
9
|
*
|
|
14
10
|
* @param checkFunction The check function that should be invoked by {@link checkStorageAccess} calls
|
|
15
11
|
* @param callback The code that should be invoked with the `checkFunction` setting
|
|
16
12
|
*/
|
|
17
|
-
const withCheckedStorageAccess = async (checkFunction, callback) => storage.run({ checkFunction }, callback);
|
|
18
|
-
exports.withCheckedStorageAccess = withCheckedStorageAccess;
|
|
13
|
+
export const withCheckedStorageAccess = async (checkFunction, callback) => storage.run({ checkFunction }, callback);
|
|
19
14
|
//# sourceMappingURL=access_checking.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"access_checking.js","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"access_checking.js","sourceRoot":"","sources":["../../src/storages/access_checking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAIrD,MAAM,OAAO,GAAG,IAAI,iBAAiB,EAAiC,CAAC;AAEvE;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC;AAE5E;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAK,EAAK,aAAyB,EAAE,QAA4B,EAAE,EAAE,CACzG,OAAO,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,EAAE,QAAQ,CAAC,CAAC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
|
|
2
|
-
import { Configuration } from '../configuration';
|
|
3
|
-
import { type Log } from '../log';
|
|
4
|
-
import type { Awaitable } from '../typedefs';
|
|
5
|
-
import type { StorageManagerOptions } from './storage_manager';
|
|
2
|
+
import { Configuration } from '../configuration.js';
|
|
3
|
+
import { type Log } from '../log.js';
|
|
4
|
+
import type { Awaitable } from '../typedefs.js';
|
|
5
|
+
import type { StorageManagerOptions } from './storage_manager.js';
|
|
6
6
|
/** @internal */
|
|
7
7
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
8
8
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,KAAK,GAAG,EAAO,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;CAAG;AAE7F,MAAM,WAAW,sBACb,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC3F,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAWjD,QAAQ,CAAC,MAAM;IAVnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,EAAE,GAAG,CAAoC;IAE5C;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAOrD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAyBjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IA0BpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAMjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBzG;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAC/B,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAsBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
package/storages/dataset.js
CHANGED
|
@@ -1,20 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const configuration_1 = require("../configuration");
|
|
11
|
-
const log_1 = require("../log");
|
|
12
|
-
const access_checking_1 = require("./access_checking");
|
|
13
|
-
const key_value_store_1 = require("./key_value_store");
|
|
14
|
-
const storage_manager_1 = require("./storage_manager");
|
|
15
|
-
const utils_1 = require("./utils");
|
|
1
|
+
import { stringify } from 'csv-stringify/sync';
|
|
2
|
+
import ow from 'ow';
|
|
3
|
+
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
|
|
4
|
+
import { Configuration } from '../configuration.js';
|
|
5
|
+
import { log } from '../log.js';
|
|
6
|
+
import { checkStorageAccess } from './access_checking.js';
|
|
7
|
+
import { KeyValueStore } from './key_value_store.js';
|
|
8
|
+
import { StorageManager } from './storage_manager.js';
|
|
9
|
+
import { purgeDefaultStorages } from './utils.js';
|
|
16
10
|
/** @internal */
|
|
17
|
-
|
|
11
|
+
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
18
12
|
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
19
13
|
/**
|
|
20
14
|
* Accepts a JSON serializable object as an input, validates its serializability,
|
|
@@ -22,7 +16,7 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
|
|
|
22
16
|
* in an array to provide better error messages. Returns serialized object.
|
|
23
17
|
* @ignore
|
|
24
18
|
*/
|
|
25
|
-
function checkAndSerialize(item, limitBytes, index) {
|
|
19
|
+
export function checkAndSerialize(item, limitBytes, index) {
|
|
26
20
|
const s = typeof index === 'number' ? ` at index ${index} ` : ' ';
|
|
27
21
|
const isItemObject = item && typeof item === 'object' && !Array.isArray(item);
|
|
28
22
|
if (!isItemObject) {
|
|
@@ -51,7 +45,7 @@ function checkAndSerialize(item, limitBytes, index) {
|
|
|
51
45
|
* The function assumes that none of the items is larger than limitBytes and does not validate.
|
|
52
46
|
* @ignore
|
|
53
47
|
*/
|
|
54
|
-
function chunkBySize(items, limitBytes) {
|
|
48
|
+
export function chunkBySize(items, limitBytes) {
|
|
55
49
|
if (!items.length)
|
|
56
50
|
return [];
|
|
57
51
|
if (items.length === 1)
|
|
@@ -133,41 +127,17 @@ function chunkBySize(items, limitBytes) {
|
|
|
133
127
|
* ```
|
|
134
128
|
* @category Result Stores
|
|
135
129
|
*/
|
|
136
|
-
class Dataset {
|
|
130
|
+
export class Dataset {
|
|
131
|
+
config;
|
|
132
|
+
id;
|
|
133
|
+
name;
|
|
134
|
+
client;
|
|
135
|
+
log = log.child({ prefix: 'Dataset' });
|
|
137
136
|
/**
|
|
138
137
|
* @internal
|
|
139
138
|
*/
|
|
140
|
-
constructor(options, config =
|
|
141
|
-
|
|
142
|
-
enumerable: true,
|
|
143
|
-
configurable: true,
|
|
144
|
-
writable: true,
|
|
145
|
-
value: config
|
|
146
|
-
});
|
|
147
|
-
Object.defineProperty(this, "id", {
|
|
148
|
-
enumerable: true,
|
|
149
|
-
configurable: true,
|
|
150
|
-
writable: true,
|
|
151
|
-
value: void 0
|
|
152
|
-
});
|
|
153
|
-
Object.defineProperty(this, "name", {
|
|
154
|
-
enumerable: true,
|
|
155
|
-
configurable: true,
|
|
156
|
-
writable: true,
|
|
157
|
-
value: void 0
|
|
158
|
-
});
|
|
159
|
-
Object.defineProperty(this, "client", {
|
|
160
|
-
enumerable: true,
|
|
161
|
-
configurable: true,
|
|
162
|
-
writable: true,
|
|
163
|
-
value: void 0
|
|
164
|
-
});
|
|
165
|
-
Object.defineProperty(this, "log", {
|
|
166
|
-
enumerable: true,
|
|
167
|
-
configurable: true,
|
|
168
|
-
writable: true,
|
|
169
|
-
value: log_1.log.child({ prefix: 'Dataset' })
|
|
170
|
-
});
|
|
139
|
+
constructor(options, config = Configuration.getGlobalConfig()) {
|
|
140
|
+
this.config = config;
|
|
171
141
|
this.id = options.id;
|
|
172
142
|
this.name = options.name;
|
|
173
143
|
this.client = options.client.dataset(this.id);
|
|
@@ -197,10 +167,10 @@ class Dataset {
|
|
|
197
167
|
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
198
168
|
*/
|
|
199
169
|
async pushData(data) {
|
|
200
|
-
|
|
201
|
-
(
|
|
170
|
+
checkStorageAccess();
|
|
171
|
+
ow(data, 'data', ow.object);
|
|
202
172
|
const dispatch = async (payload) => this.client.pushItems(payload);
|
|
203
|
-
const limit =
|
|
173
|
+
const limit = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
|
|
204
174
|
// Handle singular Objects
|
|
205
175
|
if (!Array.isArray(data)) {
|
|
206
176
|
const payload = checkAndSerialize(data, limit);
|
|
@@ -219,7 +189,7 @@ class Dataset {
|
|
|
219
189
|
* Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
|
|
220
190
|
*/
|
|
221
191
|
async getData(options = {}) {
|
|
222
|
-
|
|
192
|
+
checkStorageAccess();
|
|
223
193
|
try {
|
|
224
194
|
return await this.client.listItems(options);
|
|
225
195
|
}
|
|
@@ -236,7 +206,7 @@ class Dataset {
|
|
|
236
206
|
* via the `listItems()` client method, which gives you only paginated results.
|
|
237
207
|
*/
|
|
238
208
|
async export(options = {}) {
|
|
239
|
-
|
|
209
|
+
checkStorageAccess();
|
|
240
210
|
const items = [];
|
|
241
211
|
const fetchNextChunk = async (offset = 0) => {
|
|
242
212
|
const limit = 1000;
|
|
@@ -260,11 +230,11 @@ class Dataset {
|
|
|
260
230
|
* @param [contentType] Only JSON and CSV are supported currently, defaults to JSON.
|
|
261
231
|
*/
|
|
262
232
|
async exportTo(key, options, contentType) {
|
|
263
|
-
const kvStore = await
|
|
233
|
+
const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });
|
|
264
234
|
const items = await this.export(options);
|
|
265
235
|
if (contentType === 'text/csv') {
|
|
266
236
|
const keys = Object.keys(items[0]);
|
|
267
|
-
const value =
|
|
237
|
+
const value = stringify([
|
|
268
238
|
keys,
|
|
269
239
|
...items.map((item) => {
|
|
270
240
|
return keys.map((k) => item[k]);
|
|
@@ -305,7 +275,7 @@ class Dataset {
|
|
|
305
275
|
* @param [options] An optional options object where you can provide the dataset and target KVS name.
|
|
306
276
|
*/
|
|
307
277
|
static async exportToJSON(key, options) {
|
|
308
|
-
|
|
278
|
+
checkStorageAccess();
|
|
309
279
|
const dataset = await this.open(options?.fromDataset);
|
|
310
280
|
await dataset.exportToJSON(key, options);
|
|
311
281
|
}
|
|
@@ -316,7 +286,7 @@ class Dataset {
|
|
|
316
286
|
* @param [options] An optional options object where you can provide the dataset and target KVS name.
|
|
317
287
|
*/
|
|
318
288
|
static async exportToCSV(key, options) {
|
|
319
|
-
|
|
289
|
+
checkStorageAccess();
|
|
320
290
|
const dataset = await this.open(options?.fromDataset);
|
|
321
291
|
await dataset.exportToCSV(key, options);
|
|
322
292
|
}
|
|
@@ -343,7 +313,7 @@ class Dataset {
|
|
|
343
313
|
* ```
|
|
344
314
|
*/
|
|
345
315
|
async getInfo() {
|
|
346
|
-
|
|
316
|
+
checkStorageAccess();
|
|
347
317
|
return this.client.get();
|
|
348
318
|
}
|
|
349
319
|
/**
|
|
@@ -367,13 +337,13 @@ class Dataset {
|
|
|
367
337
|
* @default 0
|
|
368
338
|
*/
|
|
369
339
|
async forEach(iteratee, options = {}, index = 0) {
|
|
370
|
-
|
|
340
|
+
checkStorageAccess();
|
|
371
341
|
if (!options.offset)
|
|
372
342
|
options.offset = 0;
|
|
373
343
|
if (options.format && options.format !== 'json')
|
|
374
344
|
throw new Error('Dataset.forEach/map/reduce() support only a "json" format.');
|
|
375
345
|
if (!options.limit)
|
|
376
|
-
options.limit =
|
|
346
|
+
options.limit = DATASET_ITERATORS_DEFAULT_LIMIT;
|
|
377
347
|
const { items, total, limit, offset } = await this.getData(options);
|
|
378
348
|
for (const item of items) {
|
|
379
349
|
await iteratee(item, index++);
|
|
@@ -394,7 +364,7 @@ class Dataset {
|
|
|
394
364
|
* @param [options] All `map()` parameters.
|
|
395
365
|
*/
|
|
396
366
|
async map(iteratee, options = {}) {
|
|
397
|
-
|
|
367
|
+
checkStorageAccess();
|
|
398
368
|
const result = [];
|
|
399
369
|
await this.forEach(async (item, index) => {
|
|
400
370
|
const res = await iteratee(item, index);
|
|
@@ -403,7 +373,7 @@ class Dataset {
|
|
|
403
373
|
return result;
|
|
404
374
|
}
|
|
405
375
|
async reduce(iteratee, memo, options = {}) {
|
|
406
|
-
|
|
376
|
+
checkStorageAccess();
|
|
407
377
|
let currentMemo = memo;
|
|
408
378
|
const wrappedFunc = async (item, index) => {
|
|
409
379
|
if (index === 0 && currentMemo === undefined) {
|
|
@@ -423,9 +393,9 @@ class Dataset {
|
|
|
423
393
|
* depending on the mode of operation.
|
|
424
394
|
*/
|
|
425
395
|
async drop() {
|
|
426
|
-
|
|
396
|
+
checkStorageAccess();
|
|
427
397
|
await this.client.delete();
|
|
428
|
-
const manager =
|
|
398
|
+
const manager = StorageManager.getManager(Dataset, this.config);
|
|
429
399
|
manager.closeStorage(this);
|
|
430
400
|
}
|
|
431
401
|
/**
|
|
@@ -443,16 +413,16 @@ class Dataset {
|
|
|
443
413
|
* @param [options] Storage manager options.
|
|
444
414
|
*/
|
|
445
415
|
static async open(datasetIdOrName, options = {}) {
|
|
446
|
-
|
|
447
|
-
(
|
|
448
|
-
(
|
|
449
|
-
config:
|
|
450
|
-
storageClient:
|
|
416
|
+
checkStorageAccess();
|
|
417
|
+
ow(datasetIdOrName, ow.optional.string);
|
|
418
|
+
ow(options, ow.object.exactShape({
|
|
419
|
+
config: ow.optional.object.instanceOf(Configuration),
|
|
420
|
+
storageClient: ow.optional.object,
|
|
451
421
|
}));
|
|
452
|
-
options.config
|
|
453
|
-
options.storageClient
|
|
454
|
-
await
|
|
455
|
-
const manager =
|
|
422
|
+
options.config ??= Configuration.getGlobalConfig();
|
|
423
|
+
options.storageClient ??= options.config.getStorageClient();
|
|
424
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
|
|
425
|
+
const manager = StorageManager.getManager(this, options.config);
|
|
456
426
|
return manager.openStorage(datasetIdOrName, options.storageClient);
|
|
457
427
|
}
|
|
458
428
|
/**
|
|
@@ -491,5 +461,4 @@ class Dataset {
|
|
|
491
461
|
return dataset.getData(options);
|
|
492
462
|
}
|
|
493
463
|
}
|
|
494
|
-
exports.Dataset = Dataset;
|
|
495
464
|
//# sourceMappingURL=dataset.js.map
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAWH;IAVb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IAC5B,GAAG,GAAQ,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAE5C;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
package/storages/index.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export * from './dataset';
|
|
2
|
-
export * from './key_value_store';
|
|
3
|
-
export * from './request_list';
|
|
4
|
-
export * from './request_provider';
|
|
5
|
-
export { RequestQueueV1 } from './request_queue';
|
|
6
|
-
export { RequestQueue } from './request_queue_v2';
|
|
7
|
-
export { RequestQueue as RequestQueueV2 } from './request_queue_v2';
|
|
8
|
-
export * from './storage_manager';
|
|
9
|
-
export * from './utils';
|
|
10
|
-
export * from './access_checking';
|
|
11
|
-
export * from './sitemap_request_list';
|
|
1
|
+
export * from './dataset.js';
|
|
2
|
+
export * from './key_value_store.js';
|
|
3
|
+
export * from './request_list.js';
|
|
4
|
+
export * from './request_provider.js';
|
|
5
|
+
export { RequestQueueV1 } from './request_queue.js';
|
|
6
|
+
export { RequestQueue } from './request_queue_v2.js';
|
|
7
|
+
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
+
export * from './storage_manager.js';
|
|
9
|
+
export * from './utils.js';
|
|
10
|
+
export * from './access_checking.js';
|
|
11
|
+
export * from './sitemap_request_list.js';
|
|
12
12
|
//# sourceMappingURL=index.d.ts.map
|
package/storages/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
|
package/storages/index.js
CHANGED
|
@@ -1,19 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
Object.defineProperty(exports, "RequestQueue", { enumerable: true, get: function () { return request_queue_v2_1.RequestQueue; } });
|
|
13
|
-
var request_queue_v2_2 = require("./request_queue_v2");
|
|
14
|
-
Object.defineProperty(exports, "RequestQueueV2", { enumerable: true, get: function () { return request_queue_v2_2.RequestQueue; } });
|
|
15
|
-
tslib_1.__exportStar(require("./storage_manager"), exports);
|
|
16
|
-
tslib_1.__exportStar(require("./utils"), exports);
|
|
17
|
-
tslib_1.__exportStar(require("./access_checking"), exports);
|
|
18
|
-
tslib_1.__exportStar(require("./sitemap_request_list"), exports);
|
|
1
|
+
export * from './dataset.js';
|
|
2
|
+
export * from './key_value_store.js';
|
|
3
|
+
export * from './request_list.js';
|
|
4
|
+
export * from './request_provider.js';
|
|
5
|
+
export { RequestQueueV1 } from './request_queue.js';
|
|
6
|
+
export { RequestQueue } from './request_queue_v2.js';
|
|
7
|
+
export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
|
|
8
|
+
export * from './storage_manager.js';
|
|
9
|
+
export * from './utils.js';
|
|
10
|
+
export * from './access_checking.js';
|
|
11
|
+
export * from './sitemap_request_list.js';
|
|
19
12
|
//# sourceMappingURL=index.js.map
|
package/storages/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Dictionary, StorageClient } from '@crawlee/types';
|
|
2
|
-
import { Configuration } from '../configuration';
|
|
3
|
-
import type { Awaitable } from '../typedefs';
|
|
4
|
-
import type { StorageManagerOptions } from './storage_manager';
|
|
2
|
+
import { Configuration } from '../configuration.js';
|
|
3
|
+
import type { Awaitable } from '../typedefs.js';
|
|
4
|
+
import type { StorageManagerOptions } from './storage_manager.js';
|
|
5
5
|
/**
|
|
6
6
|
* Helper function to possibly stringify value if options.contentType is not set.
|
|
7
7
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQrF,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQrF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MA0B5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAelB,QAAQ,CAAC,MAAM;IAdnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IA0CrE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;IA2B/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsE3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAMpF,WAAW;IAwBzB;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAsB7G;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;OAkBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA6B/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|