@crawlee/core 3.0.0-alpha.1 → 3.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +286 -0
- package/autoscaling/autoscaled_pool.d.ts.map +1 -0
- package/autoscaling/autoscaled_pool.js +624 -0
- package/autoscaling/autoscaled_pool.js.map +1 -0
- package/autoscaling/index.d.ts +4 -0
- package/autoscaling/index.d.ts.map +1 -0
- package/autoscaling/index.js +7 -0
- package/autoscaling/index.js.map +1 -0
- package/autoscaling/snapshotter.d.ts +193 -0
- package/autoscaling/snapshotter.d.ts.map +1 -0
- package/autoscaling/snapshotter.js +377 -0
- package/autoscaling/snapshotter.js.map +1 -0
- package/autoscaling/system_status.d.ts +181 -0
- package/autoscaling/system_status.d.ts.map +1 -0
- package/autoscaling/system_status.js +194 -0
- package/autoscaling/system_status.js.map +1 -0
- package/cache_container.d.ts +18 -0
- package/cache_container.d.ts.map +1 -0
- package/cache_container.js +42 -0
- package/cache_container.js.map +1 -0
- package/configuration.d.ts +150 -0
- package/configuration.d.ts.map +1 -0
- package/configuration.js +284 -0
- package/configuration.js.map +1 -0
- package/constants.d.ts +3 -0
- package/constants.d.ts.map +1 -0
- package/constants.js +6 -0
- package/constants.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +23 -0
- package/crawlers/crawler_commons.d.ts.map +1 -0
- package/crawlers/crawler_commons.js +3 -0
- package/crawlers/crawler_commons.js.map +1 -0
- package/crawlers/crawler_extension.d.ts +11 -0
- package/crawlers/crawler_extension.d.ts.map +1 -0
- package/crawlers/crawler_extension.js +30 -0
- package/crawlers/crawler_extension.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +25 -0
- package/crawlers/crawler_utils.d.ts.map +1 -0
- package/crawlers/crawler_utils.js +72 -0
- package/crawlers/crawler_utils.js.map +1 -0
- package/crawlers/index.d.ts +5 -0
- package/crawlers/index.d.ts.map +1 -0
- package/crawlers/index.js +8 -0
- package/crawlers/index.js.map +1 -0
- package/crawlers/statistics.d.ts +147 -0
- package/crawlers/statistics.d.ts.map +1 -0
- package/crawlers/statistics.js +360 -0
- package/crawlers/statistics.js.map +1 -0
- package/enqueue_links/enqueue_links.d.ts +138 -0
- package/enqueue_links/enqueue_links.d.ts.map +1 -0
- package/enqueue_links/enqueue_links.js +123 -0
- package/enqueue_links/enqueue_links.js.map +1 -0
- package/enqueue_links/index.d.ts +3 -0
- package/enqueue_links/index.d.ts.map +1 -0
- package/enqueue_links/index.js +6 -0
- package/enqueue_links/index.js.map +1 -0
- package/enqueue_links/shared.d.ts +60 -0
- package/enqueue_links/shared.d.ts.map +1 -0
- package/enqueue_links/shared.js +148 -0
- package/enqueue_links/shared.js.map +1 -0
- package/errors.d.ts +8 -0
- package/errors.d.ts.map +1 -0
- package/errors.js +12 -0
- package/errors.js.map +1 -0
- package/events/event_manager.d.ts +41 -0
- package/events/event_manager.d.ts.map +1 -0
- package/events/event_manager.js +103 -0
- package/events/event_manager.js.map +1 -0
- package/events/index.d.ts +3 -0
- package/events/index.d.ts.map +1 -0
- package/events/index.js +6 -0
- package/events/index.js.map +1 -0
- package/events/local_event_manager.d.ts +32 -0
- package/events/local_event_manager.d.ts.map +1 -0
- package/events/local_event_manager.js +114 -0
- package/events/local_event_manager.js.map +1 -0
- package/index.d.ts +14 -0
- package/index.d.ts.map +1 -0
- package/index.js +17 -0
- package/index.js.map +1 -0
- package/index.mjs +63 -0
- package/log.d.ts +3 -0
- package/log.d.ts.map +1 -0
- package/log.js +12 -0
- package/log.js.map +1 -0
- package/package.json +4 -7
- package/proxy_configuration.d.ts +181 -0
- package/proxy_configuration.d.ts.map +1 -0
- package/proxy_configuration.js +209 -0
- package/proxy_configuration.js.map +1 -0
- package/request.d.ts +174 -0
- package/request.d.ts.map +1 -0
- package/request.js +270 -0
- package/request.js.map +1 -0
- package/serialization.d.ts +33 -0
- package/serialization.d.ts.map +1 -0
- package/serialization.js +146 -0
- package/serialization.js.map +1 -0
- package/session_pool/errors.d.ts +8 -0
- package/session_pool/errors.d.ts.map +1 -0
- package/session_pool/errors.js +20 -0
- package/session_pool/errors.js.map +1 -0
- package/session_pool/events.d.ts +3 -0
- package/session_pool/events.d.ts.map +1 -0
- package/session_pool/events.js +6 -0
- package/session_pool/events.js.map +1 -0
- package/session_pool/index.d.ts +6 -0
- package/session_pool/index.d.ts.map +1 -0
- package/session_pool/index.js +9 -0
- package/session_pool/index.js.map +1 -0
- package/session_pool/session.d.ts +212 -0
- package/session_pool/session.d.ts.map +1 -0
- package/session_pool/session.js +360 -0
- package/session_pool/session.js.map +1 -0
- package/session_pool/session_pool.d.ts +222 -0
- package/session_pool/session_pool.d.ts.map +1 -0
- package/session_pool/session_pool.js +389 -0
- package/session_pool/session_pool.js.map +1 -0
- package/session_pool/session_utils.d.ts +9 -0
- package/session_pool/session_utils.d.ts.map +1 -0
- package/session_pool/session_utils.js +19 -0
- package/session_pool/session_utils.js.map +1 -0
- package/storages/dataset.d.ts +358 -0
- package/storages/dataset.d.ts.map +1 -0
- package/storages/dataset.js +415 -0
- package/storages/dataset.js.map +1 -0
- package/storages/index.d.ts +7 -0
- package/storages/index.d.ts.map +1 -0
- package/storages/index.js +11 -0
- package/storages/index.js.map +1 -0
- package/storages/key_value_store.d.ts +242 -0
- package/storages/key_value_store.d.ts.map +1 -0
- package/storages/key_value_store.js +319 -0
- package/storages/key_value_store.js.map +1 -0
- package/storages/request_list.d.ts +461 -0
- package/storages/request_list.d.ts.map +1 -0
- package/storages/request_list.js +715 -0
- package/storages/request_list.js.map +1 -0
- package/storages/request_queue.d.ts +312 -0
- package/storages/request_queue.d.ts.map +1 -0
- package/storages/request_queue.js +708 -0
- package/storages/request_queue.js.map +1 -0
- package/storages/storage.d.ts +260 -0
- package/storages/storage.d.ts.map +1 -0
- package/storages/storage.js +3 -0
- package/storages/storage.js.map +1 -0
- package/storages/storage_manager.d.ts +40 -0
- package/storages/storage_manager.d.ts.map +1 -0
- package/storages/storage_manager.js +119 -0
- package/storages/storage_manager.js.map +1 -0
- package/tsconfig.build.tsbuildinfo +1 -0
- package/typedefs.d.ts +12 -0
- package/typedefs.d.ts.map +1 -0
- package/typedefs.js +15 -0
- package/typedefs.js.map +1 -0
- package/validators.d.ts +21 -0
- package/validators.d.ts.map +1 -0
- package/validators.js +26 -0
- package/validators.js.map +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;;AAAA,0CAA6D;AAC7D,0DAAiE;AACjE,gDAAwD;AACxD,4DAA4B;AAC5B,mDAA2D;AAC3D,oDAAoB;AACpB,0CAAkD;AAClD,uDAAmD;AACnD,gCAA6B;AAC7B,wCAAqD;AAGrD,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;GAGG;AACU,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAEzC,gBAAgB;AACH,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC;;;;GAIG;AACU,QAAA,mCAAmC,GAAG,KAAM,CAAC;AAE1D;;;GAGG;AACU,QAAA,2BAA2B,GAAG,CAAC,CAAC;AAE7C;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;GAIG;AACU,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,SAAiB;IAC1C,MAAM,GAAG,GAAG,gBAAM;SACb,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAE3B,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7B,CAAC;AARD,oCAQC;AAiCD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAa,YAAY;IA4CrB;;OAEG;IACH,YAAY,OAA4B;QA9CxC;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;WAAC;QAC5C;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;mBAAc,EAAE;WAAC;QACjB;;;;mBAAY,IAAA,gCAAoB,GAAE;WAAC;QACnC;;;;;WAA2B;QAE3B;;;;WAIG;QACH;;;;mBAAwB,IAAI,+BAAc,EAAU;WAAC;QACrD;;;;mBAMY,IAAI;WAAC;QAEjB,6DAA6D;QAC7D,8EAA8E;QAC9E;;;;mBAAa,IAAI,GAAG,EAAE;WAAC;QAEvB,oFAAoF;QACpF,uDAAuD;QACvD,yCAAyC;QACzC;;;;mBAAkB,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC;WAAC;QAE3E,mFAAmF;QACnF,mFAAmF;QACnF;;;;mBAAoB,CAAC;WAAC;QACtB;;;;mBAAsB,CAAC;WAAC;QAExB,0DAA0D;QAC1D,uEAAuE;QACvE,2DAA2D;QAC3D,wDAAwD;QACxD;;;;mBAAgB,IAAI,yBAAQ,CAE1B,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;WAAC;QAMlC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAuB,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CAAC,WAAqC,EAAE,UAAwC,EAAE;QAC9F,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnC,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,MAAM,OAAO,GAAG,WAAW,YAAY,iBAAO;YAC1C,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,IAAI,iBAAO,CAAC,WAAW,CAAC,CAAC;QAE/B,mDAAmD;QACnD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,sCAAsC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEpD,IAAI,UAAU,EAAE;YACZ,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;aAClC,CAAC;SACL;QAED,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAuB,CAAC;QACtG,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEjD,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/F,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;SAC1D;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,WAAW,CACb,YAA0C,EAC1C,UAAwC,EAAE;QAE1C,IAAA,YAAE,EAAC,YAAY,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACpD,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,CAAC;QACL,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YAC9C,OAAO,WAAW,YAAY,iBAAO;gBACjC,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC,IAAI,iBAAO,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,MAAM,aAAa,GAAc,EAAE,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC5B,mDAAmD;YACnD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,sCAAsC;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEpD,IAAI,UAAU,EAAE;gBACZ,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;aACN;iBAAM;gBACH,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aAC/B;SACJ;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YACvB,OAAO,OAAO,CAAC;SAClB;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAEpF,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,yCAAyC;QACzC,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE;YACnD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YACpD,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEzC,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBAC/F,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACvB,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAElB,uFAAuF;QACvF,iEAAiE;QACjE,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,IAAA,kBAAO,EAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC9E,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;aACrB;YAED,OAAO,IAAI,CAAC;QAChB,CAAC,EAAE,EAAgB,CAAC,CAAC;QAErB,OAAO,IAAI,iBAAO,CAAC,mBAAgD,CAAC,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAEvD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;YAC/E,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC;aAC7D,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;SACf;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEnC,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;SACX;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9G,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;SACf;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC;SACf;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;YACpB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uBAAuB,OAAO,CAAC,EAAE,6CAA6C,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,OAAO,IAAI,CAAC;SACf;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAuB,CAAC;QAC5G,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAC9B,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAEjD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE;YACvC,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC9B;QAED,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,UAAwC,EAAE;QAC7E,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;SACvB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,EAAE,kCAAkC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClH,OAAO,IAAI,CAAC;SACf;QAED,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAuB,CAAC;QACzG,kBAAkB,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACjD,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;gBAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3G,OAAO;aACV;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAEnC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC5D,CAAC,EAAE,wCAAgC,CAAC,CAAC;QAErC,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAEhF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACjG,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAAsC;QAC5E,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC7B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,iBAAiB,EAAE,kBAAkB,CAAC,iBAAiB;SAC1D,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,yBAAiB,EAAE,6BAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,sCAAsC;QACtC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAU,CAAC;wBAAE,OAAO;oBAE/G,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACpD,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;SACV;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAE7H,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,qCAA4B,EAAE;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,qCAA4B,GAAG,CAAC,CAAC;SAC5G;QACD,MAAM,2BAA2B,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC;eAC9D,eAAe;eACf,SAAS,GAAG,qCAA4B,CAAC;QAEhD,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,2CAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,mCAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B;YACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEhB,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE;YAC5B,MAAM,WAAW,GAAG,2CAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,WAAW,oFAAoF,CAAC,CAAC;YAC9H,MAAM,IAAA,qBAAK,EAAC,WAAW,CAAC,CAAC;SAC5B;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACrE,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;SACtD;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,6BAAqB,EAAE;YACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACvD;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;QACjD,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B;QAC3C,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;QACjD,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;CACJ;AAnnBD,oCAmnBC"}
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { RequestOptions as RequestModel } from '../request';
|
|
3
|
+
import { AllowedHttpMethods, Dictionary } from '../typedefs';
|
|
4
|
+
/**
|
|
5
|
+
* A helper class that is used to report results from various
|
|
6
|
+
* {@link RequestQueue} functions as well as {@link enqueueLinks}.
|
|
7
|
+
*/
|
|
8
|
+
export interface QueueOperationInfo {
|
|
9
|
+
/** Indicates if request was already present in the queue. */
|
|
10
|
+
wasAlreadyPresent: boolean;
|
|
11
|
+
/** Indicates if request was already marked as handled. */
|
|
12
|
+
wasAlreadyHandled: boolean;
|
|
13
|
+
/** The ID of the added request */
|
|
14
|
+
requestId: string;
|
|
15
|
+
}
|
|
16
|
+
export interface DatasetCollectionClientOptions {
|
|
17
|
+
storageDir: string;
|
|
18
|
+
}
|
|
19
|
+
export interface DatasetCollectionData {
|
|
20
|
+
id: string;
|
|
21
|
+
name?: string;
|
|
22
|
+
createdAt: Date;
|
|
23
|
+
modifiedAt: Date;
|
|
24
|
+
accessedAt: Date;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Dataset collection client.
|
|
28
|
+
*/
|
|
29
|
+
export interface DatasetCollectionClient {
|
|
30
|
+
list(): Promise<PaginatedList<Dataset>>;
|
|
31
|
+
getOrCreate(name: string): Promise<DatasetCollectionData>;
|
|
32
|
+
}
|
|
33
|
+
export interface Dataset extends DatasetCollectionData {
|
|
34
|
+
itemCount: number;
|
|
35
|
+
}
|
|
36
|
+
export interface DatasetClientUpdateOptions {
|
|
37
|
+
name?: string;
|
|
38
|
+
}
|
|
39
|
+
export interface DatasetClientListOptions {
|
|
40
|
+
desc?: boolean;
|
|
41
|
+
limit?: number;
|
|
42
|
+
offset?: number;
|
|
43
|
+
}
|
|
44
|
+
export interface PaginatedList<Data> {
|
|
45
|
+
/** Total count of entries in the dataset. */
|
|
46
|
+
total: number;
|
|
47
|
+
/** Count of dataset entries returned in this set. */
|
|
48
|
+
count: number;
|
|
49
|
+
/** Position of the first returned entry in the dataset. */
|
|
50
|
+
offset: number;
|
|
51
|
+
/** Maximum number of dataset entries requested. */
|
|
52
|
+
limit: number;
|
|
53
|
+
/** Should the results be in descending order. */
|
|
54
|
+
desc: boolean;
|
|
55
|
+
/** Dataset entries based on chosen format parameter. */
|
|
56
|
+
items: Data[];
|
|
57
|
+
}
|
|
58
|
+
export interface DatasetInfo {
|
|
59
|
+
id: string;
|
|
60
|
+
name?: string;
|
|
61
|
+
userId?: string;
|
|
62
|
+
createdAt: Date;
|
|
63
|
+
modifiedAt: Date;
|
|
64
|
+
accessedAt: Date;
|
|
65
|
+
itemCount: number;
|
|
66
|
+
cleanItemCount?: number;
|
|
67
|
+
actId?: string;
|
|
68
|
+
actRunId?: string;
|
|
69
|
+
stats?: DatasetStats;
|
|
70
|
+
fields?: string[];
|
|
71
|
+
}
|
|
72
|
+
export interface DatasetStats {
|
|
73
|
+
readCount?: number;
|
|
74
|
+
writeCount?: number;
|
|
75
|
+
deleteCount?: number;
|
|
76
|
+
storageBytes?: number;
|
|
77
|
+
}
|
|
78
|
+
export interface DatasetClient<Data extends Dictionary = Dictionary> {
|
|
79
|
+
get(): Promise<DatasetInfo | undefined>;
|
|
80
|
+
update(newFields: DatasetClientUpdateOptions): Promise<DatasetInfo>;
|
|
81
|
+
delete(): Promise<void>;
|
|
82
|
+
downloadItems(...args: unknown[]): Promise<Buffer>;
|
|
83
|
+
listItems(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
|
|
84
|
+
pushItems(items: Data | Data[] | string | string[]): Promise<void>;
|
|
85
|
+
}
|
|
86
|
+
export interface KeyValueStoreInfo {
|
|
87
|
+
id: string;
|
|
88
|
+
name?: string;
|
|
89
|
+
userId: string;
|
|
90
|
+
createdAt: Date;
|
|
91
|
+
modifiedAt: Date;
|
|
92
|
+
accessedAt: Date;
|
|
93
|
+
actId?: string;
|
|
94
|
+
actRunId?: string;
|
|
95
|
+
stats?: KeyValueStoreStats;
|
|
96
|
+
}
|
|
97
|
+
export interface KeyValueStoreStats {
|
|
98
|
+
readCount?: number;
|
|
99
|
+
writeCount?: number;
|
|
100
|
+
deleteCount?: number;
|
|
101
|
+
listCount?: number;
|
|
102
|
+
storageBytes?: number;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Key-value store collection client.
|
|
106
|
+
*/
|
|
107
|
+
export interface KeyValueStoreCollectionClient {
|
|
108
|
+
list(): Promise<PaginatedList<KeyValueStoreInfo>>;
|
|
109
|
+
getOrCreate(name: string): Promise<KeyValueStoreInfo>;
|
|
110
|
+
}
|
|
111
|
+
export interface KeyValueStoreRecord {
|
|
112
|
+
key: string;
|
|
113
|
+
value: any;
|
|
114
|
+
contentType?: string;
|
|
115
|
+
}
|
|
116
|
+
export interface KeyValueStoreClientUpdateOptions {
|
|
117
|
+
name?: string;
|
|
118
|
+
}
|
|
119
|
+
export interface KeyValueStoreClientListOptions {
|
|
120
|
+
limit?: number;
|
|
121
|
+
exclusiveStartKey?: string;
|
|
122
|
+
}
|
|
123
|
+
export interface KeyValueStoreItemData {
|
|
124
|
+
key: string;
|
|
125
|
+
size: number;
|
|
126
|
+
}
|
|
127
|
+
export interface KeyValueStoreClientListData {
|
|
128
|
+
count: number;
|
|
129
|
+
limit: number;
|
|
130
|
+
exclusiveStartKey?: string;
|
|
131
|
+
isTruncated: boolean;
|
|
132
|
+
nextExclusiveStartKey?: string;
|
|
133
|
+
items: KeyValueStoreItemData[];
|
|
134
|
+
}
|
|
135
|
+
export interface KeyValueStoreClientGetRecordOptions {
|
|
136
|
+
buffer?: boolean;
|
|
137
|
+
stream?: boolean;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Key-value Store client.
|
|
141
|
+
*/
|
|
142
|
+
export interface KeyValueStoreClient {
|
|
143
|
+
get(): Promise<KeyValueStoreInfo | undefined>;
|
|
144
|
+
update(newFields: KeyValueStoreClientUpdateOptions): Promise<KeyValueStoreInfo>;
|
|
145
|
+
delete(): Promise<void>;
|
|
146
|
+
listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
|
|
147
|
+
getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
|
|
148
|
+
setRecord(record: KeyValueStoreRecord): Promise<void>;
|
|
149
|
+
deleteRecord(key: string): Promise<void>;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Request queue collection client.
|
|
153
|
+
*/
|
|
154
|
+
export interface RequestQueueCollectionClient {
|
|
155
|
+
list(): Promise<PaginatedList<RequestQueueInfo>>;
|
|
156
|
+
getOrCreate(name: string): Promise<RequestQueueInfo>;
|
|
157
|
+
}
|
|
158
|
+
export interface QueueHead {
|
|
159
|
+
limit: number;
|
|
160
|
+
queueModifiedAt: Date;
|
|
161
|
+
hadMultipleClients: boolean;
|
|
162
|
+
items: RequestModel[];
|
|
163
|
+
}
|
|
164
|
+
export interface RequestQueueInfo {
|
|
165
|
+
id: string;
|
|
166
|
+
name?: string;
|
|
167
|
+
userId: string;
|
|
168
|
+
createdAt: Date;
|
|
169
|
+
modifiedAt: Date;
|
|
170
|
+
accessedAt: Date;
|
|
171
|
+
expireAt?: string;
|
|
172
|
+
totalRequestCount: number;
|
|
173
|
+
handledRequestCount: number;
|
|
174
|
+
pendingRequestCount: number;
|
|
175
|
+
actId?: string;
|
|
176
|
+
actRunId?: string;
|
|
177
|
+
hadMultipleClients: boolean;
|
|
178
|
+
stats: RequestQueueStats;
|
|
179
|
+
}
|
|
180
|
+
export interface RequestQueueStats {
|
|
181
|
+
readCount?: number;
|
|
182
|
+
writeCount?: number;
|
|
183
|
+
deleteCount?: number;
|
|
184
|
+
headItemReadCount?: number;
|
|
185
|
+
storageBytes?: number;
|
|
186
|
+
}
|
|
187
|
+
export interface RequestQueueClientOptions {
|
|
188
|
+
name: string;
|
|
189
|
+
storageDir: string;
|
|
190
|
+
}
|
|
191
|
+
export interface ListOptions {
|
|
192
|
+
/**
|
|
193
|
+
* @default 100
|
|
194
|
+
*/
|
|
195
|
+
limit?: number;
|
|
196
|
+
}
|
|
197
|
+
export interface RequestOptions {
|
|
198
|
+
forefront?: boolean;
|
|
199
|
+
[k: string]: unknown;
|
|
200
|
+
}
|
|
201
|
+
export interface RequestSchema {
|
|
202
|
+
uniqueKey: string;
|
|
203
|
+
url: string;
|
|
204
|
+
method?: AllowedHttpMethods;
|
|
205
|
+
payload?: string;
|
|
206
|
+
retryCount?: number;
|
|
207
|
+
errorMessages?: string[];
|
|
208
|
+
headers?: Dictionary<string>;
|
|
209
|
+
userData?: Dictionary;
|
|
210
|
+
handledAt?: string;
|
|
211
|
+
noRetry?: boolean;
|
|
212
|
+
loadedUrl?: string;
|
|
213
|
+
}
|
|
214
|
+
export interface ProcessedRequest {
|
|
215
|
+
uniqueKey: string;
|
|
216
|
+
requestId: string;
|
|
217
|
+
wasAlreadyPresent: boolean;
|
|
218
|
+
wasAlreadyHandled: boolean;
|
|
219
|
+
}
|
|
220
|
+
export interface UnprocessedRequest {
|
|
221
|
+
uniqueKey: string;
|
|
222
|
+
url: string;
|
|
223
|
+
method?: AllowedHttpMethods;
|
|
224
|
+
}
|
|
225
|
+
export interface BatchAddRequestsResult {
|
|
226
|
+
processedRequests: ProcessedRequest[];
|
|
227
|
+
unprocessedRequests: UnprocessedRequest[];
|
|
228
|
+
}
|
|
229
|
+
export interface RequestQueueClient {
|
|
230
|
+
get(): Promise<RequestQueueInfo | undefined>;
|
|
231
|
+
update(newFields: {
|
|
232
|
+
name?: string;
|
|
233
|
+
}): Promise<RequestQueueInfo | undefined>;
|
|
234
|
+
delete(): Promise<void>;
|
|
235
|
+
listHead(options?: ListOptions): Promise<QueueHead>;
|
|
236
|
+
addRequest(request: RequestModel, options?: RequestOptions): Promise<QueueOperationInfo>;
|
|
237
|
+
batchAddRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
|
|
238
|
+
getRequest(id: string): Promise<RequestOptions | undefined>;
|
|
239
|
+
updateRequest(request: RequestModel, options?: RequestOptions): Promise<QueueOperationInfo>;
|
|
240
|
+
deleteRequest(_id: string): Promise<unknown>;
|
|
241
|
+
}
|
|
242
|
+
export interface RequestQueueOptions {
|
|
243
|
+
clientKey?: string;
|
|
244
|
+
timeoutSecs?: number;
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Represents a storage capable of working with datasets, KV stores and request queues.
|
|
248
|
+
*/
|
|
249
|
+
export interface StorageClient {
|
|
250
|
+
datasets(): DatasetCollectionClient;
|
|
251
|
+
dataset(id: string): DatasetClient;
|
|
252
|
+
keyValueStores(): KeyValueStoreCollectionClient;
|
|
253
|
+
keyValueStore(id: string): KeyValueStoreClient;
|
|
254
|
+
requestQueues(): RequestQueueCollectionClient;
|
|
255
|
+
requestQueue(id: string, options?: RequestQueueOptions): RequestQueueClient;
|
|
256
|
+
stats?: {
|
|
257
|
+
rateLimitErrors: number[];
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
//# sourceMappingURL=storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../src/storages/storage.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,cAAc,IAAI,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE7D;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAE/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,8BAA8B;IAC3C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC7D;AAED,MAAM,WAAW,OAAQ,SAAQ,qBAAqB;IAClD,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACrC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI;IAC/B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AACD,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAC/D,GAAG,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,CAAC,SAAS,EAAE,0BAA0B,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACpE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,aAAa,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,SAAS,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5E,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC1C,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAClD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gCAAgC;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,2BAA2B;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC;IACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,KAAK,EAAE,qBAAqB,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,mCAAmC;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,GAAG,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,CAAC,SAAS,EAAE,gCAAgC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAChF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,8BAA8B,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACzF,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mCAAmC,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAChH,SAAS,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IACzC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,IAAI,CAAC;IACtB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IACxB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC3B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAsB;IACnC,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IAC/B,GAAG,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IAC7C,MAAM,CAAC,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IAC5E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACzF,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACvG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAC5D,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5F,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAmB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,QAAQ,IAAI,uBAAuB,CAAC;IACpC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,CAAC;IACnC,cAAc,IAAI,6BAA6B,CAAC;IAChD,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC/C,aAAa,IAAI,4BAA4B,CAAC;IAC9C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CAAC;IAC5E,KAAK,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACzC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/storages/storage.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Configuration } from '../configuration';
|
|
2
|
+
import { Constructor } from '../typedefs';
|
|
3
|
+
import { StorageClient } from './storage';
|
|
4
|
+
export interface IStorage {
|
|
5
|
+
id: string;
|
|
6
|
+
name?: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* StorageManager takes care of opening remote or local storages.
|
|
10
|
+
* @ignore
|
|
11
|
+
*/
|
|
12
|
+
export declare class StorageManager<T extends IStorage = IStorage> {
|
|
13
|
+
private readonly config;
|
|
14
|
+
private static readonly MAX_OPENED_STORAGES;
|
|
15
|
+
private readonly name;
|
|
16
|
+
private readonly StorageConstructor;
|
|
17
|
+
private readonly cache;
|
|
18
|
+
constructor(StorageConstructor: Constructor<T>, config?: Configuration);
|
|
19
|
+
openStorage(idOrName?: string, client?: StorageClient): Promise<T>;
|
|
20
|
+
closeStorage(storage: {
|
|
21
|
+
id: string;
|
|
22
|
+
name?: string;
|
|
23
|
+
}): void;
|
|
24
|
+
/**
|
|
25
|
+
* Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
|
|
26
|
+
*/
|
|
27
|
+
protected _getOrCreateStorage(storageIdOrName: string, storageConstructorName: string, apiClient: StorageClient): Promise<import("./storage").DatasetCollectionData>;
|
|
28
|
+
protected _getStorageClientFactories(client: StorageClient, storageConstructorName: string): {
|
|
29
|
+
createStorageClient: ((id: string) => import("./storage").DatasetClient<import("../typedefs").Dictionary<unknown>>) | ((id: string) => import("./storage").KeyValueStoreClient) | ((id: string, options?: import("./storage").RequestQueueOptions | undefined) => import("./storage").RequestQueueClient);
|
|
30
|
+
createStorageCollectionClient: (() => import("./storage").DatasetCollectionClient) | (() => import("./storage").KeyValueStoreCollectionClient) | (() => import("./storage").RequestQueueCollectionClient);
|
|
31
|
+
};
|
|
32
|
+
protected _addStorageToCache(storage: T): void;
|
|
33
|
+
}
|
|
34
|
+
export interface StorageManagerOptions {
|
|
35
|
+
/**
|
|
36
|
+
* SDK configuration instance, defaults to the static register.
|
|
37
|
+
*/
|
|
38
|
+
config?: Configuration;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=storage_manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAc1C,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IACnD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAGhC,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;IAOvD,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBxE,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAAC,eAAe,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa;IAcrH,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.StorageManager = void 0;
|
|
4
|
+
const consts_1 = require("@apify/consts");
|
|
5
|
+
const datastructures_1 = require("@apify/datastructures");
|
|
6
|
+
const configuration_1 = require("../configuration");
|
|
7
|
+
const DEFAULT_ID_ENV_VAR_NAMES = {
|
|
8
|
+
Dataset: consts_1.ENV_VARS.DEFAULT_DATASET_ID,
|
|
9
|
+
KeyValueStore: consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
|
|
10
|
+
RequestQueue: consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
|
|
11
|
+
};
|
|
12
|
+
const DEFAULT_ID_CONFIG_KEYS = {
|
|
13
|
+
Dataset: 'defaultDatasetId',
|
|
14
|
+
KeyValueStore: 'defaultKeyValueStoreId',
|
|
15
|
+
RequestQueue: 'defaultRequestQueueId',
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* StorageManager takes care of opening remote or local storages.
|
|
19
|
+
* @ignore
|
|
20
|
+
*/
|
|
21
|
+
class StorageManager {
|
|
22
|
+
constructor(StorageConstructor, config = configuration_1.Configuration.getGlobalConfig()) {
|
|
23
|
+
Object.defineProperty(this, "config", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: config
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(this, "name", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: void 0
|
|
34
|
+
});
|
|
35
|
+
Object.defineProperty(this, "StorageConstructor", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: void 0
|
|
40
|
+
});
|
|
41
|
+
Object.defineProperty(this, "cache", {
|
|
42
|
+
enumerable: true,
|
|
43
|
+
configurable: true,
|
|
44
|
+
writable: true,
|
|
45
|
+
value: void 0
|
|
46
|
+
});
|
|
47
|
+
this.StorageConstructor = StorageConstructor;
|
|
48
|
+
this.name = this.StorageConstructor.name;
|
|
49
|
+
this.cache = new datastructures_1.LruCache({ maxLength: StorageManager.MAX_OPENED_STORAGES });
|
|
50
|
+
}
|
|
51
|
+
async openStorage(idOrName, client) {
|
|
52
|
+
if (!idOrName) {
|
|
53
|
+
const defaultIdEnvVarName = DEFAULT_ID_ENV_VAR_NAMES[this.name];
|
|
54
|
+
const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
|
|
55
|
+
idOrName = this.config.get(defaultIdConfigKey);
|
|
56
|
+
if (!idOrName)
|
|
57
|
+
throw new Error(`The '${defaultIdEnvVarName}' environment variable is not defined.`);
|
|
58
|
+
}
|
|
59
|
+
const cacheKey = idOrName;
|
|
60
|
+
let storage = this.cache.get(cacheKey);
|
|
61
|
+
if (!storage) {
|
|
62
|
+
client ?? (client = this.config.getStorageClient());
|
|
63
|
+
const storageObject = await this._getOrCreateStorage(idOrName, this.name, client);
|
|
64
|
+
storage = new this.StorageConstructor({
|
|
65
|
+
id: storageObject.id,
|
|
66
|
+
name: storageObject.name,
|
|
67
|
+
client,
|
|
68
|
+
});
|
|
69
|
+
this._addStorageToCache(storage);
|
|
70
|
+
}
|
|
71
|
+
return storage;
|
|
72
|
+
}
|
|
73
|
+
closeStorage(storage) {
|
|
74
|
+
const idKey = storage.id;
|
|
75
|
+
this.cache.remove(idKey);
|
|
76
|
+
if (storage.name) {
|
|
77
|
+
const nameKey = storage.name;
|
|
78
|
+
this.cache.remove(nameKey);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
|
|
83
|
+
*/
|
|
84
|
+
async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
|
|
85
|
+
const { createStorageClient, createStorageCollectionClient, } = this._getStorageClientFactories(apiClient, storageConstructorName);
|
|
86
|
+
const storageClient = createStorageClient(storageIdOrName);
|
|
87
|
+
const existingStorage = await storageClient.get();
|
|
88
|
+
if (existingStorage)
|
|
89
|
+
return existingStorage;
|
|
90
|
+
const storageCollectionClient = createStorageCollectionClient();
|
|
91
|
+
return storageCollectionClient.getOrCreate(storageIdOrName);
|
|
92
|
+
}
|
|
93
|
+
_getStorageClientFactories(client, storageConstructorName) {
|
|
94
|
+
// Dataset => dataset
|
|
95
|
+
const clientName = storageConstructorName[0].toLowerCase() + storageConstructorName.slice(1);
|
|
96
|
+
// dataset => datasets
|
|
97
|
+
const collectionClientName = `${clientName}s`;
|
|
98
|
+
return {
|
|
99
|
+
createStorageClient: client[clientName].bind(client),
|
|
100
|
+
createStorageCollectionClient: client[collectionClientName].bind(client),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
_addStorageToCache(storage) {
|
|
104
|
+
const idKey = storage.id;
|
|
105
|
+
this.cache.add(idKey, storage);
|
|
106
|
+
if (storage.name) {
|
|
107
|
+
const nameKey = storage.name;
|
|
108
|
+
this.cache.add(nameKey, storage);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.StorageManager = StorageManager;
|
|
113
|
+
Object.defineProperty(StorageManager, "MAX_OPENED_STORAGES", {
|
|
114
|
+
enumerable: true,
|
|
115
|
+
configurable: true,
|
|
116
|
+
writable: true,
|
|
117
|
+
value: 1000
|
|
118
|
+
});
|
|
119
|
+
//# sourceMappingURL=storage_manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":";;;AAAA,0CAAyC;AACzC,0DAAiD;AACjD,oDAAiD;AAIjD,MAAM,wBAAwB,GAAG;IAC7B,OAAO,EAAE,iBAAQ,CAAC,kBAAkB;IACpC,aAAa,EAAE,iBAAQ,CAAC,0BAA0B;IAClD,YAAY,EAAE,iBAAQ,CAAC,wBAAwB;CACzC,CAAC;AAEX,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAa,cAAc;IAMvB,YACI,kBAAkC,EACjB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QANrB;;;;;WAAoE;QACpE;;;;;WAAuE;QACvE;;;;;WAAoC;QAMhC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;QACzF,IAAI,CAAC,KAAK,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAiB,EAAE,MAAsB;QACvD,IAAI,CAAC,QAAQ,EAAE;YACX,MAAM,mBAAmB,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;YACzD,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,mBAAmB,wCAAwC,CAAC,CAAC;SACvG;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE;YACV,MAAM,KAAN,MAAM,GAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,MAAM;aACT,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;SACpC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC9B;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,eAAuB,EAAE,sBAA8B,EAAE,SAAwB;QACjH,MAAM,EACF,mBAAmB,EACnB,6BAA6B,GAChC,GAAG,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAgB,CAAC;QAC5G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;SACpC;IACL,CAAC;;AAvFL,wCAwFC;AAvFG;;;;WAA8C,IAAI;GAAC"}
|