@crawlee/core 4.0.0-beta.65 → 4.0.0-beta.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +3 -0
- package/configuration.d.ts.map +1 -1
- package/configuration.js +3 -0
- package/configuration.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +1 -1
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +2 -2
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +1 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/events/event_manager.d.ts +26 -2
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +1 -0
- package/events/event_manager.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +8 -5
- package/service_locator.d.ts +17 -17
- package/service_locator.d.ts.map +1 -1
- package/service_locator.js +31 -26
- package/service_locator.js.map +1 -1
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/session_pool.d.ts +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +10 -3
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +28 -13
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +3 -4
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +3 -4
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +61 -13
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +119 -86
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.d.ts +1 -1
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +4 -4
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +5 -5
- package/storages/request_loader.d.ts.map +1 -1
- package/storages/request_manager.d.ts +10 -1
- package/storages/request_manager.d.ts.map +1 -1
- package/storages/request_manager_tandem.d.ts +13 -2
- package/storages/request_manager_tandem.d.ts.map +1 -1
- package/storages/request_manager_tandem.js +23 -4
- package/storages/request_manager_tandem.js.map +1 -1
- package/storages/request_queue.d.ts +283 -44
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +593 -212
- package/storages/request_queue.js.map +1 -1
- package/storages/sitemap_request_loader.d.ts +1 -1
- package/storages/sitemap_request_loader.d.ts.map +1 -1
- package/storages/sitemap_request_loader.js +2 -2
- package/storages/sitemap_request_loader.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +10 -10
- package/storages/storage_instance_manager.d.ts.map +1 -1
- package/storages/storage_instance_manager.js +33 -33
- package/storages/storage_instance_manager.js.map +1 -1
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +7 -7
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -4
- package/storages/utils.js.map +1 -1
- package/validators.d.ts +4 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +4 -0
- package/validators.js.map +1 -1
- package/storages/request_provider.d.ts +0 -325
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -619
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -88
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -437
- package/storages/request_queue_v2.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAI7D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EACH,mCAAmC,EACnC,YAAY,EACZ,2BAA2B,EAC3B,iBAAiB,EACjB,qBAAqB,EACrB,gCAAgC,GACnC,MAAM,YAAY,CAAC;AAEpB,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAM,YAAa,SAAQ,eAAe;IAC9B,qBAAqB,GAMjB,IAAI,CAAC;IAET,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC;;OAEG;IACH,YAAY,OAA+B,EAAE,SAAwB,cAAc,CAAC,gBAAgB,EAAE;QAClG,KAAK,CACD;YACI,GAAG,OAAO;YACV,SAAS,EAAE,cAAc;YACzB,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EACD,MAAM,CACT,CAAC;IACN,CAAC;IAED;;OAEG;IACI,eAAe;QAClB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,gBAAgB;QAC3B,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC;aAC1E,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;QACZ,CAAC;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE;gBACvF,aAAa;aAChB,CAAC,CAAC;YACH,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,gCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,oCAAoC;QACpC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACvC,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,iBAAiB,EAAE,qBAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,uDAAuD;QACvD,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IACI,CAAC,SAAS;wBACV,CAAC,SAAS;wBACV,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;wBAC9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAU,CAAC;wBAEjD,OAAO;oBAEX,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACnD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;oBACrF,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;wBACT,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;QACX,CAAC;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GACrF,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAErC,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,4BAA4B,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,4BAA4B,GAAG,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,2BAA2B,GAC7B,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,eAAe,IAAI,SAAS,GAAG,4BAA4B,CAAC;QAEpG,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,mCAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,2BAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAExF,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,mCAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,eAAe,WAAW,oFAAoF,CACjH,CAAC;YACF,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,2CAA2C;IAClC,KAAK,CAAC,UAAU;QACrB,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/D,MAAM,OAAO,GAAG,2CACZ,IAAI,CAAC,qBAAqB,GAAG,GACjC,8BAA8B,CAAC;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAChG,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAmD;QAChF,kBAAkB,EAAE,CAAC;QAErB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAChC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEnD,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAG,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE;oBAC5E,SAAS,EAAE,OAAO,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;YAEpC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAG,EAAE,SAAS,CAAC,CAAC;QAC7D,CAAC,EAAE,gCAAgC,CAAC,CAAC;QAErC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QAC9C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;QAEpC,OAAO,GAAG,CAAC;IACf,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QAEf,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ;AAED,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,OAAO,EACH,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,UAAU,EACV,qBAAqB,EACrB,KAAK,GACR,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAOpD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEhE;;;GAGG;AACH,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,OAAO,YAAY;IA+CE;IA9CvB,EAAE,CAAS;IACX,IAAI,CAAU;IACd,WAAW,GAAG,EAAE,CAAC;IACjB,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACnC,OAAO,CAAsB;IACnB,kBAAkB,CAAsB;IAElD,GAAG,CAAgB;IAEX,aAAa,GAAG,KAAK,CAAC;IAEpB,YAAY,CAA2B;IAEvC,uBAAuB,GAAG,KAAK,CAAC;IAEhC,2BAA2B,GAAG,CAAC,CAAC;IAE1C;;;;OAIG;IACO,6BAA6B,GAAG,CAAC,CAAC;IAElC,UAAU,CAAkB;IAEnB,MAAM,CAAe;IAEvB,YAAY,GAAG,IAAI,mBAAmB,CAAoB;QACvE,UAAU,EAAE,CAAC;QACb,iBAAiB,EAAE,CAAC;KACvB,CAAC,CAAC;IAEH;;;OAGG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,YACI,OAA4B,EACT,SAAwB,aAAa,CAAC,eAAe,EAAE;QAAvD,WAAM,GAAN,MAAM,CAAiD;QAE1E,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAE/B,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,SAAS,GAAG,EAAE,CAAC,CAAC;QAE/G,IAAI,CAAC,MAAM,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC3C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACnD,OAAO,iBAAiB,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe;QACjB,MAAM,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACxE,OAAO,iBAAiB,GAAG,mBAAmB,CAAC;IACnD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CACZ,WAAmB,EACnB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE3G,OAAO,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAClD,CAAC;QAED,EAAE,CACE,WAAW,EACX,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,GAAG,EAAE,EAAE,CAAC,MAAM;YACd,EAAE,EAAE,EAAE,CAAC,SAAS;SACnB,CAAC,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAExF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS;aACZ,CAAC;QACN,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9F,MAAM,kBAAkB,GAAG;YACvB,GAAG,iBAAiB,CAAC,CAAC,CAAC;YACvB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,WAAW,CACb,YAA0B,EAC1B,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,YAAY,EACZ,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAE3C,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YAC3C,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;gBAC1C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;gBACxF,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC5F,CAAC;iBAAM,CAAC;gBACJ,QAAQ,CAAC,IAAI,CACT,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAA6B,CAAC,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;YACP,CAAC;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;QACL,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAErG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,2CAA2C;QAC3C,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;YACpD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAoC,EACpC,UAAqC,EAAE;QAEvC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QAEF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAChD,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC/C,CAAC,CACL,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,KAAK,SAAS,CAAC,CAAC,gBAAgB;YAC5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAChC,qBAAqB;gBACrB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACzD,MAAM,IAAI,KAAK,CACX,6EAA6E,OAAO,CAAC,IAAI,CAAC,EAAE,CAC/F,CAAC;oBACN,CAAC;oBAED,IAAI,IAAI,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBACxB,MAAM,IAAI,KAAK,CACX,gFAAgF,OAAO,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;oBACN,CAAC;oBAED,IACK,IAAY,CAAC,eAAe,KAAK,SAAS;wBAC3C,OAAQ,IAAY,CAAC,eAAe,KAAK,QAAQ,EACnD,CAAC;wBACC,MAAM,IAAI,KAAK,CACX,yFAAyF,OAAO,CAAC,IAAI,CAAC,EAAE,CAC3G,CAAC;oBACN,CAAC;gBACL,CAAC;gBAED,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBAChE,8BAA8B;oBAC9B,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACJ,uBAAuB;oBACvB,MAAM,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,IAAuB,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,wBAAwB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEtE,MAAM,MAAM,GAAG,qBAAqB,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1F,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAEtD,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA0B,EAAE,KAAK,GAAG,IAAI,EAAE,EAAE;YAC/F,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAEtC,eAAe,CAAC,IAAI,CAChB,GAAG,CAAC,MAAM,uCAAuC,CAC7C,gBAAgB,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CACjF,EACD,KAAK,CACR,CAAC,CACL,CAAC;YACN,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,8DAA8D;QAC9D,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QACnF,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,yDAAyD;QACzD,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;gBACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,uCAAuC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjG,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QACtC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACtB,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAoC,SAAiB;QACjE,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAChE,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,gBAAgB;QAClB,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAC3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7D,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAgB;QACvC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CACL,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;QAE7F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC;YAC7D,GAAG,OAAO;YACV,SAAS;SACZ,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,kBAAkB,GAAG;YACvB,GAAG,gBAAgB;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CACL,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAEnF,wDAAwD;QACxD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,kBAAkB,GAAG;YACvB,GAAG,gBAAgB;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,kBAAkB,EAAE,CAAC;QAErB,4EAA4E;QAC5E,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;IACrC,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,oCAAoC,CAAC,IAAY;QACnD,IAAI,IAAI,IAAI,IAAI,CAAC,6BAA6B,EAAE,CAAC;YAC7C,OAAO;QACX,CAAC;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC;QAC1C,MAAM,IAAI,CAAC,OAAO,CAAC,oCAAoC,EAAE,CAAC,IAAI,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QACnF,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,kBAAkB,CAAC,SAAS;SAC1C,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAE3B,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,2BAA2B,GAAG,CAAC,CAAC;QAErC,8FAA8F;QAC9F,mGAAmG;QACnG,+FAA+F;QAC/F,IAAI,CAAC,6BAA6B,GAAG,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,eAAe;QACjB,mFAAmF;QACnF,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrD,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBACrC,GAAG,EAAE,eAAe;gBACpB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE;aACpD,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0CAA0C,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,MAAsB,EACtB,eAAiC,EACjC,OAAqC;QAErC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QAElF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,cAAc,EAAE,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC7D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAIjC;QACG,OAAO,kBAAkB,CAAC;YACtB,GAAG,OAAO;YACV,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;QACpF,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAEnE,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;QAE5E,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;QAE5F,MAAM,KAAK,GAAG,MAAM,cAAc;aAC7B,yBAAyB,EAAE;aAC3B,WAAW,CAAe,IAA4C,EAAE;YACrE,GAAG,QAAQ;YACX,aAAa,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,yBAAyB,CAAC,QAAQ,CAAC;YACvE,eAAe,EAAE,cAAc,CAAC,yBAAyB,EAAE,EAAE,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI;SACnG,CAAC,CAAC;QACP,KAAK,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QACtD,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACvB,6EAA6E;YAC7E,kEAAkE;YAClE,0FAA0F;YAC1F,2FAA2F;YAC3F,oFAAoF;YACpF,KAAK,CAAC,OAAO,GAAG,MAAM,cAAc,CAAC,yBAAyB,CAAC;gBAC3D,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;aACjC,CAAC,CAAC;YAEH,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
@@ -243,7 +243,7 @@ export declare class SitemapRequestLoader implements IRequestLoader {
|
|
|
243
243
|
/**
|
|
244
244
|
* @inheritDoc
|
|
245
245
|
*/
|
|
246
|
-
|
|
246
|
+
markRequestAsHandled(request: Request): Promise<void>;
|
|
247
247
|
private ensureInProgress;
|
|
248
248
|
}
|
|
249
249
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap_request_loader.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAKxE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAI3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAM5D,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,2BAA4B,SAAQ,cAAc;IAC/D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAgBD;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,cAAc;IACvD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B;;OAEG;IACH,OAAO,CAAC,GAAG,CAAgB;IAE3B,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAkE;IAE5F,gBAAgB;IAChB,OAAO;IAwDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA6BtF;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAMtC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAKxC;;;;OAIG;IACG,QAAQ,CAAC,cAAc,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAQ1E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxC;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA8B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAejD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,
|
|
1
|
+
{"version":3,"file":"sitemap_request_loader.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAKxE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAI3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAM5D,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,2BAA4B,SAAQ,cAAc;IAC/D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAgBD;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,cAAc;IACvD;;;OAGG;IACH,UAAU,cAAqB;IAE/B;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B;;OAEG;IACH,OAAO,CAAC,GAAG,CAAgB;IAE3B,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAkE;IAE5F,gBAAgB;IAChB,OAAO;IAwDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA6BtF;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAMtC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAKxC;;;;OAIG;IACG,QAAQ,CAAC,cAAc,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IAQ1E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxC;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA8B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAejD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAO3D,OAAO,CAAC,gBAAgB;CAK3B"}
|
|
@@ -296,7 +296,7 @@ export class SitemapRequestLoader {
|
|
|
296
296
|
async toTandem(requestManager) {
|
|
297
297
|
// Import here to avoid circular imports.
|
|
298
298
|
const { RequestManagerTandem } = await import('./request_manager_tandem.js');
|
|
299
|
-
const { RequestQueue } = await import('./
|
|
299
|
+
const { RequestQueue } = await import('./request_queue.js');
|
|
300
300
|
return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
|
|
301
301
|
}
|
|
302
302
|
/**
|
|
@@ -419,7 +419,7 @@ export class SitemapRequestLoader {
|
|
|
419
419
|
/**
|
|
420
420
|
* @inheritDoc
|
|
421
421
|
*/
|
|
422
|
-
async
|
|
422
|
+
async markRequestAsHandled(request) {
|
|
423
423
|
this.handledUrlCount += 1;
|
|
424
424
|
this.ensureInProgress(request.url);
|
|
425
425
|
this.inProgress.delete(request.url);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sitemap_request_loader.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAG9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,8BAA8B,CAAC;AAsG7D;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAC7B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAU;IAE1B;;OAEG;IACK,GAAG,CAAgB;IAEnB,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAkE;IAE5F,gBAAgB;IAChB,YAAoB,OAAoC;QACpD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5C,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAEhF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAE/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GAGtB;QACG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAoC;QAClD,MAAM,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,oBAAoB,CAAC;YACzC,GAAG,WAAW;YACd,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC;YAClB,mBAAmB,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,UAAU,EAAE;SAC1G,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,qGAAqG;QACrG,kCAAkC;QAClC,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC;IAC5F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,wEAAwE;QACxE,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACrE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,cAAgC;QAC3C,yCAAyC;QACzC,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAC7E,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;QAE/D,OAAO,IAAI,oBAAoB,CAAC,IAAI,EAAE,cAAc,IAAI,CAAC,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,2FAA2F;YAC3F,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,QAAQ,CAAC;YAC3C,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACc,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA4B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEzF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;IACL,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"sitemap_request_loader.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAG9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,8BAA8B,CAAC;AAsG7D;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAC7B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAU;IAE1B;;OAEG;IACK,GAAG,CAAgB;IAEnB,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAkE;IAE5F,gBAAgB;IAChB,YAAoB,OAAoC;QACpD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5C,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAEhF,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAE/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GAGtB;QACG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAoC;QAClD,MAAM,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,oBAAoB,CAAC;YACzC,GAAG,WAAW;YACd,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC;YAClB,mBAAmB,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,UAAU,EAAE;SAC1G,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,qGAAqG;QACrG,kCAAkC;QAClC,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC;IAC5F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,wEAAwE;QACxE,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACrE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,cAAgC;QAC3C,yCAAyC;QACzC,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;QAC7E,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAE5D,OAAO,IAAI,oBAAoB,CAAC,IAAI,EAAE,cAAc,IAAI,CAAC,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,2FAA2F;YAC3F,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,QAAQ,CAAC;YAC3C,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACc,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA4B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEzF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,mFAAmF;QACnF,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAgB;QACvC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
2
2
|
import type { Constructor } from '../typedefs.js';
|
|
3
3
|
export type { StorageIdentifier } from '@crawlee/types';
|
|
4
4
|
/**
|
|
@@ -13,10 +13,10 @@ type Hashable = string;
|
|
|
13
13
|
* Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
|
|
14
14
|
*
|
|
15
15
|
* A single instance manages all storage types. Instances are cached by
|
|
16
|
-
* `(storageClass, id/name/alias,
|
|
16
|
+
* `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
|
|
17
17
|
*
|
|
18
18
|
* The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
|
|
19
|
-
* and a pre-bound `
|
|
19
|
+
* and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
|
|
20
20
|
* assigns a reserved default alias.
|
|
21
21
|
*
|
|
22
22
|
* @ignore
|
|
@@ -32,14 +32,14 @@ export declare class StorageInstanceManager {
|
|
|
32
32
|
* @param name Storage name (mutually exclusive with `id` and `alias`).
|
|
33
33
|
* @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
|
|
34
34
|
* Automatically assigned when no identifier is provided.
|
|
35
|
-
* @param
|
|
35
|
+
* @param backendOpener A **lazy** factory that creates the sub-backend.
|
|
36
36
|
* Only called on a cache miss.
|
|
37
|
-
* @param
|
|
37
|
+
* @param backendCacheKey Opaque key identifying the storage backend, so that the same logical
|
|
38
38
|
* storage opened through different clients is cached separately.
|
|
39
39
|
*/
|
|
40
|
-
openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias,
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, { id, name, alias, backendOpener, backendCacheKey, }: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
|
|
41
|
+
backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
|
|
42
|
+
backendCacheKey: Hashable;
|
|
43
43
|
}): Promise<TStorage>;
|
|
44
44
|
/**
|
|
45
45
|
* Remove a storage instance from the cache (called from `storage.drop()`).
|
|
@@ -80,7 +80,7 @@ export interface DefaultStorageIdentifier {
|
|
|
80
80
|
/**
|
|
81
81
|
* Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
|
|
82
82
|
* `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
|
|
83
|
-
* the `StorageInstanceManager` and `
|
|
83
|
+
* the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
|
|
84
84
|
*
|
|
85
85
|
* - `null` / `undefined` / `{}` → default storage alias
|
|
86
86
|
* - `string` → resolved via `storageExists` (ID-first, then name)
|
|
@@ -88,5 +88,5 @@ export interface DefaultStorageIdentifier {
|
|
|
88
88
|
* - `{ name }` → `{ name }`
|
|
89
89
|
* - `{ alias }` → `{ alias }`
|
|
90
90
|
*/
|
|
91
|
-
export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined,
|
|
91
|
+
export declare function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, storageBackend: StorageBackend, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;
|
|
92
92
|
//# sourceMappingURL=storage_instance_manager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage_instance_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,
|
|
1
|
+
{"version":3,"file":"storage_instance_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,cAAc,EACd,oBAAoB,EACpB,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACpB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GAAG,MAAM,CAAC;AAqJvB;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiC;IAE7D;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,QAAQ,SAAS,QAAQ,EACvC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,aAAa,EACb,eAAe,GAClB,EAAE,CAAC,yBAAyB,GAAG,wBAAwB,CAAC,GAAG;QACxD,aAAa,EAAE,MAAM,OAAO,CAAC,cAAc,GAAG,oBAAoB,GAAG,mBAAmB,CAAC,CAAC;QAC1F,eAAe,EAAE,QAAQ,CAAC;KAC7B,GACF,OAAO,CAAC,QAAQ,CAAC;IAsEpB;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI;IAIzC;;;;OAIG;IACH,UAAU,IAAI,IAAI;CASrB;AAED;;;GAGG;AACH,MAAM,MAAM,yBAAyB,GAC/B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACrC,EAAE,CAAC,EAAE,KAAK,CAAC;IACX,IAAI,CAAC,EAAE,KAAK,CAAC;IACb,KAAK,CAAC,EAAE,KAAK,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,wBAAwB,CAC1C,UAAU,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,GAAG,SAAS,EACzD,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAC1D,OAAO,CAAC,yBAAyB,CAAC,CA0BpC"}
|
|
@@ -4,7 +4,7 @@ const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
|
4
4
|
/**
|
|
5
5
|
* Three-tier cache for storage instances, modelled after crawlee-python's `_StorageCache`.
|
|
6
6
|
*
|
|
7
|
-
* Each tier maps `[storageClass][key][
|
|
7
|
+
* Each tier maps `[storageClass][key][backendCacheKey] → instance`:
|
|
8
8
|
* - `byId` — keyed by the backend-assigned storage id
|
|
9
9
|
* - `byName` — keyed by the persistent storage name
|
|
10
10
|
* - `byAlias` — keyed by a run-scoped alias (e.g. `'__default__'` for unnamed storages)
|
|
@@ -13,7 +13,7 @@ class StorageCache {
|
|
|
13
13
|
byId = new Map();
|
|
14
14
|
byName = new Map();
|
|
15
15
|
byAlias = new Map();
|
|
16
|
-
get(cls, { id, name, alias,
|
|
16
|
+
get(cls, { id, name, alias, backendCacheKey, }) {
|
|
17
17
|
for (const [tier, key] of [
|
|
18
18
|
[this.byId, id],
|
|
19
19
|
[this.byName, name],
|
|
@@ -21,7 +21,7 @@ class StorageCache {
|
|
|
21
21
|
]) {
|
|
22
22
|
if (key === undefined)
|
|
23
23
|
continue;
|
|
24
|
-
const cached = tier.get(cls)?.get(key)?.get(
|
|
24
|
+
const cached = tier.get(cls)?.get(key)?.get(backendCacheKey);
|
|
25
25
|
if (cached) {
|
|
26
26
|
if (cached instanceof cls) {
|
|
27
27
|
return cached;
|
|
@@ -32,27 +32,27 @@ class StorageCache {
|
|
|
32
32
|
return undefined;
|
|
33
33
|
}
|
|
34
34
|
/** Write a single entry into a given tier. */
|
|
35
|
-
setInMap(tier, cls, key, instance,
|
|
35
|
+
setInMap(tier, cls, key, instance, backendCacheKey) {
|
|
36
36
|
if (!tier.has(cls))
|
|
37
37
|
tier.set(cls, new Map());
|
|
38
38
|
const keyMap = tier.get(cls);
|
|
39
39
|
if (!keyMap.has(key))
|
|
40
40
|
keyMap.set(key, new Map());
|
|
41
|
-
keyMap.get(key).set(
|
|
41
|
+
keyMap.get(key).set(backendCacheKey, instance);
|
|
42
42
|
}
|
|
43
43
|
/**
|
|
44
44
|
* Cache an instance under its actual id, name, and an optional alias.
|
|
45
45
|
*/
|
|
46
|
-
set(cls, instance,
|
|
46
|
+
set(cls, instance, backendCacheKey, alias) {
|
|
47
47
|
// Always cache by id.
|
|
48
|
-
this.setInMap(this.byId, cls, instance.id, instance,
|
|
48
|
+
this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
|
|
49
49
|
// Cache by name — only for named storages.
|
|
50
50
|
if (instance.name) {
|
|
51
|
-
this.setInMap(this.byName, cls, instance.name, instance,
|
|
51
|
+
this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
|
|
52
52
|
}
|
|
53
53
|
// Cache by alias — only for unnamed storages opened via alias.
|
|
54
54
|
if (alias !== undefined) {
|
|
55
|
-
this.setInMap(this.byAlias, cls, alias, instance,
|
|
55
|
+
this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
removeFromCache(instance) {
|
|
@@ -74,15 +74,15 @@ class StorageCache {
|
|
|
74
74
|
* Ensure that the same string is not used as both a name and an alias for the same
|
|
75
75
|
* storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
|
|
76
76
|
*/
|
|
77
|
-
checkNameAliasConflict(cls, { name, alias,
|
|
77
|
+
checkNameAliasConflict(cls, { name, alias, backendCacheKey }) {
|
|
78
78
|
if (alias) {
|
|
79
|
-
const existingByName = this.byName.get(cls)?.get(alias)?.get(
|
|
79
|
+
const existingByName = this.byName.get(cls)?.get(alias)?.get(backendCacheKey);
|
|
80
80
|
if (existingByName) {
|
|
81
81
|
throw new Error(`Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`);
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
84
|
if (name) {
|
|
85
|
-
const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(
|
|
85
|
+
const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(backendCacheKey);
|
|
86
86
|
if (existingByAlias) {
|
|
87
87
|
throw new Error(`Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
|
|
88
88
|
` If you meant to open the alias storage, use { alias: "${name}" } instead.`);
|
|
@@ -113,10 +113,10 @@ class StorageCache {
|
|
|
113
113
|
* Unified manager for opening and caching storage instances (Dataset, KeyValueStore, RequestQueue).
|
|
114
114
|
*
|
|
115
115
|
* A single instance manages all storage types. Instances are cached by
|
|
116
|
-
* `(storageClass, id/name/alias,
|
|
116
|
+
* `(storageClass, id/name/alias, backendCacheKey)` so the same storage is never opened twice.
|
|
117
117
|
*
|
|
118
118
|
* The manager itself does not resolve identifiers — callers pass explicit `id`, `name`, or `alias` (at most one),
|
|
119
|
-
* and a pre-bound `
|
|
119
|
+
* and a pre-bound `backendOpener` promise. When none of `id`, `name`, `alias` are provided, the manager automatically
|
|
120
120
|
* assigns a reserved default alias.
|
|
121
121
|
*
|
|
122
122
|
* @ignore
|
|
@@ -132,34 +132,34 @@ export class StorageInstanceManager {
|
|
|
132
132
|
* @param name Storage name (mutually exclusive with `id` and `alias`).
|
|
133
133
|
* @param alias Run-scoped alias (mutually exclusive with `id` and `name`).
|
|
134
134
|
* Automatically assigned when no identifier is provided.
|
|
135
|
-
* @param
|
|
135
|
+
* @param backendOpener A **lazy** factory that creates the sub-backend.
|
|
136
136
|
* Only called on a cache miss.
|
|
137
|
-
* @param
|
|
137
|
+
* @param backendCacheKey Opaque key identifying the storage backend, so that the same logical
|
|
138
138
|
* storage opened through different clients is cached separately.
|
|
139
139
|
*/
|
|
140
|
-
async openStorage(cls, { id, name, alias,
|
|
140
|
+
async openStorage(cls, { id, name, alias, backendOpener, backendCacheKey, }) {
|
|
141
141
|
// Auto-set alias='__default__' when no parameters are specified (mirrors crawlee-python).
|
|
142
142
|
if (!id && !name && !alias) {
|
|
143
143
|
alias = DEFAULT_STORAGE_ALIAS;
|
|
144
144
|
}
|
|
145
145
|
// Fast-path cache check (no lock).
|
|
146
146
|
if (alias !== undefined) {
|
|
147
|
-
const cached = this.cache.get(cls, { alias,
|
|
147
|
+
const cached = this.cache.get(cls, { alias, backendCacheKey });
|
|
148
148
|
if (cached)
|
|
149
149
|
return cached;
|
|
150
150
|
}
|
|
151
151
|
else if (id) {
|
|
152
|
-
const cached = this.cache.get(cls, { id,
|
|
152
|
+
const cached = this.cache.get(cls, { id, backendCacheKey });
|
|
153
153
|
if (cached)
|
|
154
154
|
return cached;
|
|
155
155
|
}
|
|
156
156
|
else if (name) {
|
|
157
|
-
const cached = this.cache.get(cls, { name,
|
|
157
|
+
const cached = this.cache.get(cls, { name, backendCacheKey });
|
|
158
158
|
if (cached)
|
|
159
159
|
return cached;
|
|
160
160
|
}
|
|
161
161
|
const identifierKey = id ?? name ?? alias ?? DEFAULT_STORAGE_ALIAS;
|
|
162
|
-
const lockKey = `${cls.name}:${identifierKey}:${
|
|
162
|
+
const lockKey = `${cls.name}:${identifierKey}:${backendCacheKey}`;
|
|
163
163
|
if (!this.openerLocks.has(lockKey)) {
|
|
164
164
|
this.openerLocks.set(lockKey, new AsyncQueue());
|
|
165
165
|
}
|
|
@@ -168,32 +168,32 @@ export class StorageInstanceManager {
|
|
|
168
168
|
try {
|
|
169
169
|
// Double-check cache under lock (another caller may have filled it while we waited).
|
|
170
170
|
if (alias !== undefined) {
|
|
171
|
-
const cached = this.cache.get(cls, { alias,
|
|
171
|
+
const cached = this.cache.get(cls, { alias, backendCacheKey });
|
|
172
172
|
if (cached)
|
|
173
173
|
return cached;
|
|
174
174
|
}
|
|
175
175
|
else if (id) {
|
|
176
|
-
const cached = this.cache.get(cls, { id,
|
|
176
|
+
const cached = this.cache.get(cls, { id, backendCacheKey });
|
|
177
177
|
if (cached)
|
|
178
178
|
return cached;
|
|
179
179
|
}
|
|
180
180
|
else if (name) {
|
|
181
|
-
const cached = this.cache.get(cls, { name,
|
|
181
|
+
const cached = this.cache.get(cls, { name, backendCacheKey });
|
|
182
182
|
if (cached)
|
|
183
183
|
return cached;
|
|
184
184
|
}
|
|
185
185
|
// Prevent the same string from being used as both a name and an alias.
|
|
186
|
-
this.cache.checkNameAliasConflict(cls, { name, alias,
|
|
187
|
-
// Cache miss — create the sub-
|
|
188
|
-
const
|
|
189
|
-
const storageInfo = await
|
|
186
|
+
this.cache.checkNameAliasConflict(cls, { name, alias, backendCacheKey });
|
|
187
|
+
// Cache miss — create the sub-backend and storage instance.
|
|
188
|
+
const subBackend = await backendOpener();
|
|
189
|
+
const storageInfo = await subBackend.getMetadata();
|
|
190
190
|
const instance = new cls({
|
|
191
191
|
id: storageInfo.id,
|
|
192
192
|
name: storageInfo.name,
|
|
193
|
-
|
|
193
|
+
backend: subBackend,
|
|
194
194
|
});
|
|
195
195
|
// Atomic cache writes (no awaits between these).
|
|
196
|
-
this.cache.set(cls, instance,
|
|
196
|
+
this.cache.set(cls, instance, backendCacheKey, alias);
|
|
197
197
|
return instance;
|
|
198
198
|
}
|
|
199
199
|
finally {
|
|
@@ -228,7 +228,7 @@ export class StorageInstanceManager {
|
|
|
228
228
|
/**
|
|
229
229
|
* Decompose a user-provided `identifier` (the `Dataset.open()` / `KeyValueStore.open()` /
|
|
230
230
|
* `RequestQueue.open()` argument) into separate `id`, `name`, and `alias` fields that
|
|
231
|
-
* the `StorageInstanceManager` and `
|
|
231
|
+
* the `StorageInstanceManager` and `StorageBackend.create*Client` expect.
|
|
232
232
|
*
|
|
233
233
|
* - `null` / `undefined` / `{}` → default storage alias
|
|
234
234
|
* - `string` → resolved via `storageExists` (ID-first, then name)
|
|
@@ -236,12 +236,12 @@ export class StorageInstanceManager {
|
|
|
236
236
|
* - `{ name }` → `{ name }`
|
|
237
237
|
* - `{ alias }` → `{ alias }`
|
|
238
238
|
*/
|
|
239
|
-
export async function resolveStorageIdentifier(identifier,
|
|
239
|
+
export async function resolveStorageIdentifier(identifier, storageBackend, storageType) {
|
|
240
240
|
if (identifier === null || identifier === undefined) {
|
|
241
241
|
return { alias: DEFAULT_STORAGE_ALIAS };
|
|
242
242
|
}
|
|
243
243
|
if (typeof identifier === 'string') {
|
|
244
|
-
if (
|
|
244
|
+
if (storageBackend.storageExists && (await storageBackend.storageExists(identifier, storageType))) {
|
|
245
245
|
return { id: identifier };
|
|
246
246
|
}
|
|
247
247
|
return { name: identifier };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,
|
|
1
|
+
{"version":3,"file":"storage_instance_manager.js","sourceRoot":"","sources":["../../src/storages/storage_instance_manager.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAgBnD,wDAAwD;AACxD,MAAM,qBAAqB,GAAG,aAAa,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,YAAY;IACL,IAAI,GAAc,IAAI,GAAG,EAAE,CAAC;IAC5B,MAAM,GAAc,IAAI,GAAG,EAAE,CAAC;IAC9B,OAAO,GAAc,IAAI,GAAG,EAAE,CAAC;IAExC,GAAG,CACC,GAAmB,EACnB,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,eAAe,GAKc;QAEjC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI;YACtB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACf,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;YACnB,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACa,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC7D,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,MAAM,YAAa,GAAuD,EAAE,CAAC;oBAC7E,OAAO,MAAW,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,8CAA8C;IACtC,QAAQ,CACZ,IAAe,EACf,GAAmB,EACnB,GAAW,EACX,QAAW,EACX,eAAyB;QAEzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,GAAG,CAAqB,GAAmB,EAAE,QAAW,EAAE,eAAyB,EAAE,KAAc;QAC/F,sBAAsB;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAEtE,2CAA2C;QAC3C,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAED,eAAe,CAAC,QAAkB;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAoC,CAAC;QAElE,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAExB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;oBACtC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,sBAAsB,CAClB,GAAmB,EACnB,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAgE;QAE9F,IAAI,KAAK,EAAE,CAAC;YACR,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC9E,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CACX,mCAAmC,KAAK,oEAAoE,CAC/G,CAAC;YACN,CAAC;QACL,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC/E,IAAI,eAAe,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACX,kCAAkC,IAAI,qEAAqE;oBACvG,0DAA0D,IAAI,cAAc,CACnF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,CAAC,SAAS;QACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAY,CAAC;QACjC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACrC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACtB,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACnB,MAAM,QAAQ,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACJ;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,sBAAsB;IACd,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;IAC3B,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,WAAW,CACb,GAA0B,EAC1B,EACI,EAAE,EACF,IAAI,EACJ,KAAK,EACL,aAAa,EACb,eAAe,GAIlB;QAED,0FAA0F;QAC1F,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACzB,KAAK,GAAG,qBAAqB,CAAC;QAClC,CAAC;QAED,mCAAmC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAC/D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;YAC5D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;YAC9D,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC9B,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,IAAI,IAAI,IAAI,KAAK,IAAI,qBAAqB,CAAC;QACnE,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;QAElE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;QAE7C,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACD,qFAAqF;YACrF,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC/D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,EAAE,EAAE,CAAC;gBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC5D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBAC9D,IAAI,MAAM;oBAAE,OAAO,MAAM,CAAC;YAC9B,CAAC;YAED,uEAAuE;YACvE,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;YAEzE,4DAA4D;YAC5D,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,MAChB,UACH,CAAC,WAAW,EAAE,CAAC;YAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;gBACrB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,OAAO,EAAE,UAAU;aACtB,CAAa,CAAC;YAEf,iDAAiD;YACjD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC;YAEtD,OAAO,QAAQ,CAAC;QACpB,CAAC;gBAAS,CAAC;YACP,KAAK,CAAC,KAAK,EAAE,CAAC;YAEd,0DAA0D;YAC1D,4DAA4D;YAC5D,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,QAAkB;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,UAAU;QACN,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YAC5C,IAAI,YAAY,IAAI,QAAQ,IAAI,OAAQ,QAAgB,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAChF,QAAgB,CAAC,UAAU,EAAE,CAAC;YACnC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACJ;AAoBD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAyD,EACzD,cAA8B,EAC9B,WAAyD;IAEzD,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACjC,IAAI,cAAc,CAAC,aAAa,IAAI,CAAC,MAAM,cAAc,CAAC,aAAa,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;YAChG,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC;IACrC,CAAC;IAED,IAAI,OAAO,IAAI,UAAU,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,6CAA6C;IAC7C,OAAO,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backend-independent usage counters tracked by the storage frontend classes
|
|
3
|
+
* ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
|
|
4
|
+
*
|
|
5
|
+
* These count the operations the frontend issues against its underlying storage backend, so they are
|
|
6
|
+
* meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
|
|
7
|
+
* — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
|
|
8
|
+
* once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
|
|
9
|
+
* of bytes stored) are intentionally not included here; read those from the backend's own API instead.
|
|
10
|
+
*/
|
|
11
|
+
/** Usage counters for a {@link Dataset}. */
|
|
12
|
+
export interface DatasetStats {
|
|
13
|
+
/** Number of read operations issued to the dataset client (e.g. `getData`). */
|
|
14
|
+
readCount: number;
|
|
15
|
+
/** Number of write operations issued to the dataset client (e.g. `pushData`). */
|
|
16
|
+
writeCount: number;
|
|
17
|
+
}
|
|
18
|
+
/** Usage counters for a {@link KeyValueStore}. */
|
|
19
|
+
export interface KeyValueStoreStats {
|
|
20
|
+
/** Number of read operations issued to the key-value store client (e.g. `getValue`). */
|
|
21
|
+
readCount: number;
|
|
22
|
+
/** Number of write operations issued to the key-value store client (e.g. `setValue`). */
|
|
23
|
+
writeCount: number;
|
|
24
|
+
/** Number of delete operations issued to the key-value store client (e.g. `deleteValue`). */
|
|
25
|
+
deleteCount: number;
|
|
26
|
+
/** Number of listing operations issued to the key-value store client (e.g. `listKeys`). */
|
|
27
|
+
listCount: number;
|
|
28
|
+
}
|
|
29
|
+
/** Usage counters for a {@link RequestQueue}. */
|
|
30
|
+
export interface RequestQueueStats {
|
|
31
|
+
/** Number of write operations issued to the request queue client (add / handle / reclaim). */
|
|
32
|
+
writeCount: number;
|
|
33
|
+
/** Number of queue-head reads issued to the request queue client (`fetchNextRequest`). */
|
|
34
|
+
headItemReadCount: number;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* A tiny mutable counter that the storage frontends increment on each client call and expose through
|
|
38
|
+
* a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
|
|
39
|
+
* the buckets that make sense for it.
|
|
40
|
+
*/
|
|
41
|
+
export declare class StorageStatsTracker<T extends Record<keyof T, number>> {
|
|
42
|
+
private readonly counters;
|
|
43
|
+
constructor(initial: T);
|
|
44
|
+
/** Increment a counter bucket by `by` (default `1`). */
|
|
45
|
+
add(key: keyof T, by?: number): void;
|
|
46
|
+
/** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
|
|
47
|
+
get current(): T;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=storage_stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_stats.d.ts","sourceRoot":"","sources":["../../src/storages/storage_stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,+CAA+C;AAC/C,MAAM,WAAW,YAAY;IACzB,+EAA+E;IAC/E,SAAS,EAAE,MAAM,CAAC;IAClB,iFAAiF;IACjF,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qDAAqD;AACrD,MAAM,WAAW,kBAAkB;IAC/B,wFAAwF;IACxF,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,UAAU,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB,2FAA2F;IAC3F,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAC9B,8FAA8F;IAC9F,UAAU,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,qBAAa,mBAAmB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAC9D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAI;gBAEjB,OAAO,EAAE,CAAC;IAItB,wDAAwD;IACxD,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,EAAE,SAAI,GAAG,IAAI;IAI/B,iGAAiG;IACjG,IAAI,OAAO,IAAI,CAAC,CAEf;CACJ"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backend-independent usage counters tracked by the storage frontend classes
|
|
3
|
+
* ({@link Dataset}, {@link KeyValueStore}, {@link RequestQueue}).
|
|
4
|
+
*
|
|
5
|
+
* These count the operations the frontend issues against its underlying storage backend, so they are
|
|
6
|
+
* meaningful for any storage backend (memory, file system, cloud). They are tallied per client call
|
|
7
|
+
* — e.g. iterating a key-value store increments `readCount` once per record fetched and `listCount`
|
|
8
|
+
* once per listed page. Backend-specific figures that the frontend cannot compute (such as the number
|
|
9
|
+
* of bytes stored) are intentionally not included here; read those from the backend's own API instead.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* A tiny mutable counter that the storage frontends increment on each client call and expose through
|
|
13
|
+
* a read-only `stats` snapshot. Generic over the concrete counter shape so each storage type gets only
|
|
14
|
+
* the buckets that make sense for it.
|
|
15
|
+
*/
|
|
16
|
+
export class StorageStatsTracker {
|
|
17
|
+
counters;
|
|
18
|
+
constructor(initial) {
|
|
19
|
+
this.counters = { ...initial };
|
|
20
|
+
}
|
|
21
|
+
/** Increment a counter bucket by `by` (default `1`). */
|
|
22
|
+
add(key, by = 1) {
|
|
23
|
+
this.counters[key] += by;
|
|
24
|
+
}
|
|
25
|
+
/** Return a snapshot of the current counters. The returned object is a copy and safe to keep. */
|
|
26
|
+
get current() {
|
|
27
|
+
return { ...this.counters };
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=storage_stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage_stats.js","sourceRoot":"","sources":["../../src/storages/storage_stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA8BH;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACX,QAAQ,CAAI;IAE7B,YAAY,OAAU;QAClB,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,wDAAwD;IACxD,GAAG,CAAC,GAAY,EAAE,EAAE,GAAG,CAAC;QACnB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAY,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,iGAAiG;IACjG,IAAI,OAAO;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IAChC,CAAC;CACJ"}
|
package/storages/utils.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BaseHttpClient, Dictionary,
|
|
1
|
+
import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { ProxyConfiguration } from '../proxy_configuration.js';
|
|
4
4
|
/**
|
|
@@ -10,7 +10,7 @@ interface PurgeDefaultStorageOptions {
|
|
|
10
10
|
*/
|
|
11
11
|
onlyPurgeOnce?: boolean;
|
|
12
12
|
config?: Configuration;
|
|
13
|
-
|
|
13
|
+
storageBackend?: StorageBackend;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
|
|
@@ -20,7 +20,7 @@ interface PurgeDefaultStorageOptions {
|
|
|
20
20
|
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
|
|
21
21
|
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
|
|
22
22
|
*
|
|
23
|
-
* This is a shortcut for running (optional) `purge` method on the
|
|
23
|
+
* This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
|
|
24
24
|
* it will call the `purge` method of the underlying storage implementation we are currently using. You can
|
|
25
25
|
* make sure the storage is purged only once for a given execution context if you set `onlyPurgeOnce` to `true` in
|
|
26
26
|
* the `options` object
|
|
@@ -34,10 +34,10 @@ export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOption
|
|
|
34
34
|
* explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@link Configuration}
|
|
35
35
|
* option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
|
|
36
36
|
*
|
|
37
|
-
* This is a shortcut for running (optional) `purge` method on the
|
|
37
|
+
* This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
|
|
38
38
|
* it will call the `purge` method of the underlying storage implementation we are currently using.
|
|
39
39
|
*/
|
|
40
|
-
export declare function purgeDefaultStorages(config?: Configuration,
|
|
40
|
+
export declare function purgeDefaultStorages(config?: Configuration, storageBackend?: StorageBackend): Promise<void>;
|
|
41
41
|
export interface UseStateOptions {
|
|
42
42
|
config?: Configuration;
|
|
43
43
|
/**
|
|
@@ -121,9 +121,9 @@ export interface StorageOpenOptions {
|
|
|
121
121
|
*/
|
|
122
122
|
config?: Configuration;
|
|
123
123
|
/**
|
|
124
|
-
* Optional storage
|
|
124
|
+
* Optional storage backend that should be used to open storages.
|
|
125
125
|
*/
|
|
126
|
-
|
|
126
|
+
storageBackend?: StorageBackend;
|
|
127
127
|
/**
|
|
128
128
|
* Used to pass the proxy configuration for the `requestsFromUrl` objects.
|
|
129
129
|
* Takes advantage of the internal address rotation and authentication process.
|
package/storages/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/storages/utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEjF,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;GAEG;AACH,UAAU,0BAA0B;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,cAAc,CAAC,EAAE,cAAc,CAAC;CACnC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAChG;;;;;;;;;;GAUG;AACH,wBAAsB,oBAAoB,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAwBnH,MAAM,WAAW,eAAe;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAChE,IAAI,CAAC,EAAE,MAAM,EACb,YAAY,GAAS,KAAK,EAC1B,OAAO,CAAC,EAAE,eAAe,kBAM5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAI7C;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAE7C,gBAAgB;AAChB,MAAM,WAAW,mBAAmB,CAAC,KAAK,EAAE,QAAQ;IAChD,8DAA8D;IAC9D,WAAW,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,6DAA6D;IAC7D,YAAY,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,KAAK,EAAE,CAAC;CAC7C;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAC9C,OAAO,EAAE,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,GAC9C,aAAa,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA6CzC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB;;OAEG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B"}
|