@crawlee/core 4.0.0-beta.66 → 4.0.0-beta.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/autoscaling/snapshotter.js +1 -1
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +3 -0
- package/configuration.d.ts.map +1 -1
- package/configuration.js +3 -0
- package/configuration.js.map +1 -1
- package/crawlers/statistics.d.ts +1 -1
- package/crawlers/statistics.js +2 -2
- package/crawlers/statistics.js.map +1 -1
- package/events/event_manager.d.ts +26 -2
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +1 -0
- package/events/event_manager.js.map +1 -1
- package/index.d.ts +2 -1
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +8 -5
- package/request.d.ts +10 -2
- package/request.d.ts.map +1 -1
- package/request.js +25 -7
- package/request.js.map +1 -1
- package/service_locator.d.ts +17 -17
- package/service_locator.d.ts.map +1 -1
- package/service_locator.js +31 -26
- package/service_locator.js.map +1 -1
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/session_pool.d.ts +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +10 -3
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +28 -13
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +61 -13
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +119 -86
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.js +2 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_manager.d.ts +2 -2
- package/storages/request_manager.d.ts.map +1 -1
- package/storages/request_manager_tandem.d.ts +1 -1
- package/storages/request_manager_tandem.d.ts.map +1 -1
- package/storages/request_manager_tandem.js +5 -4
- package/storages/request_manager_tandem.js.map +1 -1
- package/storages/request_queue.d.ts +13 -6
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +44 -27
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_instance_manager.d.ts +10 -10
- package/storages/storage_instance_manager.d.ts.map +1 -1
- package/storages/storage_instance_manager.js +33 -33
- package/storages/storage_instance_manager.js.map +1 -1
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +7 -7
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -4
- package/storages/utils.js.map +1 -1
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEtE,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAI,IAAO,EAAE,KAAc;IAC7D,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;AACL,CAAC;AA6FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAgBH;IAfb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,OAAO,CAAuB;IAC9B,GAAG,CAAgB;IAEF,YAAY,GAAG,IAAI,mBAAmB,CAAe;QAClE,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;KAChB,CAAC,CAAC;IAEH;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IAChE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,OAA+B;QAC1D,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,MAAM;gBACF,GAAG,IAAI;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,IAAI,CAAmB,CAAC;aACrE,CAAC;QACN,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,UAAU,CACrB,OAA+B,EAC/B,QAAQ,GAAG,+BAA+B;QAE1C,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,UAAU,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClG,IAAI,UAAU,IAAI,CAAC;gBAAE,MAAM;YAE3B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;YACnF,MAAM,IAAI,CAAC;YAEX,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC7B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM;YACtF,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAChC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,UAAkC,EAAE;QACvC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAC3C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,UAAkC,EAAE;QACxC,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;YAChD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK;SACrC,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACrC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QAEnD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;QAEpF,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;QAEvF,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,aAAa,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,oBAAoB,CAAC,QAAQ,CAAC;YAClE,eAAe,EAAE,cAAc,CAAC,yBAAyB,EAAE,EAAE,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI;SACnG,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
package/storages/index.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
|
+
export * from './key_value_store_codec.js';
|
|
3
4
|
export * from './request_list.js';
|
|
4
5
|
export type * from './request_loader.js';
|
|
5
6
|
export type * from './request_manager.js';
|
|
6
7
|
export * from './request_queue.js';
|
|
7
8
|
export * from './storage_instance_manager.js';
|
|
9
|
+
export * from './storage_stats.js';
|
|
8
10
|
export * from './utils.js';
|
|
9
11
|
export * from './access_checking.js';
|
|
10
12
|
export * from './sitemap_request_loader.js';
|
package/storages/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,sBAAsB,CAAC;AAC1C,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAClC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,sBAAsB,CAAC;AAC1C,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
|
package/storages/index.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
|
+
export * from './key_value_store_codec.js';
|
|
3
4
|
export * from './request_list.js';
|
|
4
5
|
export * from './request_queue.js';
|
|
5
6
|
export * from './storage_instance_manager.js';
|
|
7
|
+
export * from './storage_stats.js';
|
|
6
8
|
export * from './utils.js';
|
|
7
9
|
export * from './access_checking.js';
|
|
8
10
|
export * from './sitemap_request_loader.js';
|
package/storages/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAGlC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC"}
|
|
@@ -1,16 +1,9 @@
|
|
|
1
|
-
import type { Dictionary,
|
|
1
|
+
import type { Dictionary, KeyValueStoreBackend } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { Awaitable } from '../typedefs.js';
|
|
4
|
+
import type { KeyValueStoreStats } from './storage_stats.js';
|
|
4
5
|
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
5
6
|
import type { StorageOpenOptions } from './utils.js';
|
|
6
|
-
/**
|
|
7
|
-
* Helper function to possibly stringify value if options.contentType is not set.
|
|
8
|
-
*
|
|
9
|
-
* @ignore
|
|
10
|
-
*/
|
|
11
|
-
export declare const maybeStringify: <T>(value: T, options: {
|
|
12
|
-
contentType?: string;
|
|
13
|
-
}) => T;
|
|
14
7
|
/**
|
|
15
8
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
16
9
|
* for saving and reading data records or files. Each data record is
|
|
@@ -71,14 +64,20 @@ export declare class KeyValueStore {
|
|
|
71
64
|
readonly config: Configuration;
|
|
72
65
|
readonly id: string;
|
|
73
66
|
readonly name?: string;
|
|
74
|
-
private readonly
|
|
67
|
+
private readonly backend;
|
|
75
68
|
private persistStateEventStarted;
|
|
76
69
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
77
70
|
private readonly cache;
|
|
71
|
+
private readonly statsTracker;
|
|
78
72
|
/**
|
|
79
73
|
* @internal
|
|
80
74
|
*/
|
|
81
75
|
constructor(options: KeyValueStoreOptions, config?: Configuration);
|
|
76
|
+
/**
|
|
77
|
+
* Backend-independent usage counters tracked for this key-value store (read / write / delete /
|
|
78
|
+
* list operations issued to the underlying storage backend). Counted per backend call.
|
|
79
|
+
*/
|
|
80
|
+
get stats(): KeyValueStoreStats;
|
|
82
81
|
/**
|
|
83
82
|
* Gets a value from the key-value store.
|
|
84
83
|
*
|
|
@@ -143,6 +142,33 @@ export declare class KeyValueStore {
|
|
|
143
142
|
* on the MIME content type of the record, or the default value if the key is missing from the store.
|
|
144
143
|
*/
|
|
145
144
|
getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
|
|
145
|
+
/**
|
|
146
|
+
* Reads a record from the key-value store without parsing the value.
|
|
147
|
+
*
|
|
148
|
+
* Use this when you need the raw bytes and the content type — for example, to run your own
|
|
149
|
+
* parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
|
|
150
|
+
*
|
|
151
|
+
* There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
|
|
152
|
+
* passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
|
|
153
|
+
* is provided. To write pre-serialized bytes, call
|
|
154
|
+
* `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
|
|
155
|
+
*
|
|
156
|
+
* Returns `null` if the record does not exist.
|
|
157
|
+
*
|
|
158
|
+
* **Example usage:**
|
|
159
|
+
* ```javascript
|
|
160
|
+
* const store = await KeyValueStore.open();
|
|
161
|
+
* const record = await store.getRecord('huge.json');
|
|
162
|
+
* if (record) {
|
|
163
|
+
* const data = simdjson.parse(record.value);
|
|
164
|
+
* }
|
|
165
|
+
* ```
|
|
166
|
+
*
|
|
167
|
+
* @param key
|
|
168
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
169
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
170
|
+
*/
|
|
171
|
+
getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
|
|
146
172
|
/**
|
|
147
173
|
* Tests whether a record with the given key exists in the key-value store without retrieving its value.
|
|
148
174
|
*
|
|
@@ -394,6 +420,19 @@ export declare class KeyValueStore {
|
|
|
394
420
|
* @ignore
|
|
395
421
|
*/
|
|
396
422
|
static getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
|
|
423
|
+
/**
|
|
424
|
+
* Reads a record from the default {@link KeyValueStore} associated with the current crawler run
|
|
425
|
+
* without parsing the value.
|
|
426
|
+
*
|
|
427
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
|
|
428
|
+
* record does not exist.
|
|
429
|
+
*
|
|
430
|
+
* @param key
|
|
431
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
432
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
433
|
+
* @ignore
|
|
434
|
+
*/
|
|
435
|
+
static getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
|
|
397
436
|
/**
|
|
398
437
|
* Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
|
|
399
438
|
* @param key The queried record key.
|
|
@@ -435,8 +474,9 @@ export declare class KeyValueStore {
|
|
|
435
474
|
static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
|
|
436
475
|
/**
|
|
437
476
|
* Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
438
|
-
*
|
|
439
|
-
*
|
|
477
|
+
*
|
|
478
|
+
* The input is read from the default {@link KeyValueStore} under the configured input key
|
|
479
|
+
* (`CRAWLEE_INPUT_KEY`, default `INPUT`).
|
|
440
480
|
*
|
|
441
481
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
442
482
|
* If you need to use the input multiple times in your crawler,
|
|
@@ -471,7 +511,15 @@ export interface KeyConsumer {
|
|
|
471
511
|
export interface KeyValueStoreOptions {
|
|
472
512
|
id: string;
|
|
473
513
|
name?: string;
|
|
474
|
-
|
|
514
|
+
backend: KeyValueStoreBackend;
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* A raw, unparsed key-value store record as returned by {@link KeyValueStore.getRecord}: the
|
|
518
|
+
* verbatim bytes plus the content type, with parsing left to the caller.
|
|
519
|
+
*/
|
|
520
|
+
export interface KeyValueStoreRawRecord {
|
|
521
|
+
value: Buffer | ArrayBuffer;
|
|
522
|
+
contentType: string | null;
|
|
475
523
|
}
|
|
476
524
|
export interface RecordOptions {
|
|
477
525
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,oBAAoB,EAAyB,MAAM,gBAAgB,CAAC;AAK9F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAQrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,qBAAa,aAAa;IAqBlB,QAAQ,CAAC,MAAM;IApBnB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IAEvD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAK1B;IAEH;;OAEG;gBAEC,OAAO,EAAE,oBAAoB,EACpB,MAAM,gBAAkC;IAOrD;;;OAGG;IACH,IAAI,KAAK,IAAI,kBAAkB,CAE9B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAkDrE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAcpE;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB3G,OAAO,CAAC,uBAAuB;YAsBhB,kBAAkB;YAkBlB,aAAa;IAmB5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA4D3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAMlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAS3F;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAShG;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CAAC,CAAC,GAAG,OAAO,EACf,OAAO,GAAE,4BAAiC,GAC3C,aAAa,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;IAStD;;;;;;;;;;;OAWG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC;IAI1F;;;;;;OAMG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAI5D;;;;;;;;;;;;;;OAcG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,aAAa,CAAC;IAyBzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAClE;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAmC5E;;;;;;;;;;;OAWG;WACU,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAK3E;;;;OAIG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAK3C,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,GAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;OAmBG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,oBAAoB,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -1,44 +1,15 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import JSON5 from 'json5';
|
|
4
1
|
import ow, { ArgumentError } from 'ow';
|
|
5
2
|
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
|
|
6
|
-
import { jsonStringifyExtended } from '@apify/utilities';
|
|
7
3
|
import { Configuration } from '../configuration.js';
|
|
8
4
|
import { serviceLocator } from '../service_locator.js';
|
|
9
5
|
import { checkStorageAccess } from './access_checking.js';
|
|
6
|
+
import { parseValue, serializeValue } from './key_value_store_codec.js';
|
|
7
|
+
import { StorageStatsTracker } from './storage_stats.js';
|
|
10
8
|
import { resolveStorageIdentifier } from './storage_instance_manager.js';
|
|
11
9
|
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
10
|
+
import { isBuffer, isStream } from '@crawlee/utils';
|
|
12
11
|
/** @internal */
|
|
13
12
|
const KVS_KEYS_DEFAULT_LIMIT = 1000;
|
|
14
|
-
/**
|
|
15
|
-
* Helper function to possibly stringify value if options.contentType is not set.
|
|
16
|
-
*
|
|
17
|
-
* @ignore
|
|
18
|
-
*/
|
|
19
|
-
export const maybeStringify = (value, options) => {
|
|
20
|
-
// If contentType is missing, value will be stringified to JSON
|
|
21
|
-
if (options.contentType === null || options.contentType === undefined) {
|
|
22
|
-
options.contentType = 'application/json; charset=utf-8';
|
|
23
|
-
try {
|
|
24
|
-
// Format JSON to simplify debugging, the overheads with compression is negligible
|
|
25
|
-
value = jsonStringifyExtended(value, null, 2);
|
|
26
|
-
}
|
|
27
|
-
catch (e) {
|
|
28
|
-
const error = e;
|
|
29
|
-
// Give more meaningful error message
|
|
30
|
-
if (error.message?.includes('Invalid string length')) {
|
|
31
|
-
error.message = 'Object is too large';
|
|
32
|
-
}
|
|
33
|
-
throw new Error(`The "value" parameter cannot be stringified to JSON: ${error.message}`);
|
|
34
|
-
}
|
|
35
|
-
if (value === undefined) {
|
|
36
|
-
throw new Error('The "value" parameter was stringified to JSON and returned undefined. ' +
|
|
37
|
-
"Make sure you're not trying to stringify an undefined value.");
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
return value;
|
|
41
|
-
};
|
|
42
13
|
/**
|
|
43
14
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
44
15
|
* for saving and reading data records or files. Each data record is
|
|
@@ -99,10 +70,16 @@ export class KeyValueStore {
|
|
|
99
70
|
config;
|
|
100
71
|
id;
|
|
101
72
|
name;
|
|
102
|
-
|
|
73
|
+
backend;
|
|
103
74
|
persistStateEventStarted = false;
|
|
104
75
|
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
105
76
|
cache = new Map();
|
|
77
|
+
statsTracker = new StorageStatsTracker({
|
|
78
|
+
readCount: 0,
|
|
79
|
+
writeCount: 0,
|
|
80
|
+
deleteCount: 0,
|
|
81
|
+
listCount: 0,
|
|
82
|
+
});
|
|
106
83
|
/**
|
|
107
84
|
* @internal
|
|
108
85
|
*/
|
|
@@ -110,7 +87,14 @@ export class KeyValueStore {
|
|
|
110
87
|
this.config = config;
|
|
111
88
|
this.id = options.id;
|
|
112
89
|
this.name = options.name;
|
|
113
|
-
this.
|
|
90
|
+
this.backend = options.backend;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Backend-independent usage counters tracked for this key-value store (read / write / delete /
|
|
94
|
+
* list operations issued to the underlying storage backend). Counted per backend call.
|
|
95
|
+
*/
|
|
96
|
+
get stats() {
|
|
97
|
+
return this.statsTracker.current;
|
|
114
98
|
}
|
|
115
99
|
/**
|
|
116
100
|
* Gets a value from the key-value store.
|
|
@@ -147,8 +131,53 @@ export class KeyValueStore {
|
|
|
147
131
|
async getValue(key, defaultValue) {
|
|
148
132
|
checkStorageAccess();
|
|
149
133
|
ow(key, ow.string.nonEmpty);
|
|
150
|
-
|
|
151
|
-
|
|
134
|
+
this.statsTracker.add('readCount');
|
|
135
|
+
const record = await this.backend.getValue(key);
|
|
136
|
+
// A missing record falls back to the default; a record that parses to a falsy value (including
|
|
137
|
+
// a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
|
|
138
|
+
if (!record) {
|
|
139
|
+
return defaultValue ?? null;
|
|
140
|
+
}
|
|
141
|
+
// Storage backends are byte transports — the value is raw bytes; the frontend parses it here.
|
|
142
|
+
return parseValue(record.value, record.contentType ?? null);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Reads a record from the key-value store without parsing the value.
|
|
146
|
+
*
|
|
147
|
+
* Use this when you need the raw bytes and the content type — for example, to run your own
|
|
148
|
+
* parser (`simdjson`, a custom XML library, etc.) or to forward the bytes verbatim.
|
|
149
|
+
*
|
|
150
|
+
* There is no symmetric `setRecord` method, because {@link KeyValueStore.setValue} already
|
|
151
|
+
* passes a `Buffer` (or `string` / `Stream`) through unchanged when an explicit `contentType`
|
|
152
|
+
* is provided. To write pre-serialized bytes, call
|
|
153
|
+
* `setValue(key, buffer, { contentType: 'application/json; charset=utf-8' })`.
|
|
154
|
+
*
|
|
155
|
+
* Returns `null` if the record does not exist.
|
|
156
|
+
*
|
|
157
|
+
* **Example usage:**
|
|
158
|
+
* ```javascript
|
|
159
|
+
* const store = await KeyValueStore.open();
|
|
160
|
+
* const record = await store.getRecord('huge.json');
|
|
161
|
+
* if (record) {
|
|
162
|
+
* const data = simdjson.parse(record.value);
|
|
163
|
+
* }
|
|
164
|
+
* ```
|
|
165
|
+
*
|
|
166
|
+
* @param key
|
|
167
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
168
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
169
|
+
*/
|
|
170
|
+
async getRecord(key) {
|
|
171
|
+
checkStorageAccess();
|
|
172
|
+
ow(key, ow.string.nonEmpty);
|
|
173
|
+
this.statsTracker.add('readCount');
|
|
174
|
+
const record = await this.backend.getValue(key);
|
|
175
|
+
if (!record)
|
|
176
|
+
return null;
|
|
177
|
+
return {
|
|
178
|
+
value: record.value,
|
|
179
|
+
contentType: record.contentType ?? null,
|
|
180
|
+
};
|
|
152
181
|
}
|
|
153
182
|
/**
|
|
154
183
|
* Tests whether a record with the given key exists in the key-value store without retrieving its value.
|
|
@@ -159,7 +188,7 @@ export class KeyValueStore {
|
|
|
159
188
|
async recordExists(key) {
|
|
160
189
|
checkStorageAccess();
|
|
161
190
|
ow(key, ow.string.nonEmpty);
|
|
162
|
-
return this.
|
|
191
|
+
return this.backend.recordExists(key);
|
|
163
192
|
}
|
|
164
193
|
async getAutoSavedValue(key, defaultValue = {}) {
|
|
165
194
|
checkStorageAccess();
|
|
@@ -195,9 +224,12 @@ export class KeyValueStore {
|
|
|
195
224
|
for await (const page of this.fetchKeyPages(options)) {
|
|
196
225
|
const results = [];
|
|
197
226
|
for (const item of page) {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
227
|
+
this.statsTracker.add('readCount');
|
|
228
|
+
const record = await this.backend.getValue(item.key);
|
|
229
|
+
if (record) {
|
|
230
|
+
const parsed = parseValue(record.value, record.contentType ?? null);
|
|
231
|
+
results.push(mapRecord(item.key, parsed));
|
|
232
|
+
}
|
|
201
233
|
}
|
|
202
234
|
yield results;
|
|
203
235
|
}
|
|
@@ -205,11 +237,16 @@ export class KeyValueStore {
|
|
|
205
237
|
async *fetchKeyPages(options, limit = KVS_KEYS_DEFAULT_LIMIT) {
|
|
206
238
|
let exclusiveStartKey;
|
|
207
239
|
while (true) {
|
|
208
|
-
|
|
240
|
+
this.statsTracker.add('listCount');
|
|
241
|
+
const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({
|
|
242
|
+
...options,
|
|
243
|
+
exclusiveStartKey,
|
|
244
|
+
limit,
|
|
245
|
+
});
|
|
209
246
|
yield items;
|
|
210
|
-
if (
|
|
247
|
+
if (!isTruncated)
|
|
211
248
|
break;
|
|
212
|
-
exclusiveStartKey =
|
|
249
|
+
exclusiveStartKey = nextExclusiveStartKey;
|
|
213
250
|
}
|
|
214
251
|
}
|
|
215
252
|
/**
|
|
@@ -262,10 +299,8 @@ export class KeyValueStore {
|
|
|
262
299
|
validator: ow.isValid(k, ow.string.matches(KEY_VALUE_STORE_KEY_REGEX)),
|
|
263
300
|
message: `The "key" argument "${key}" must be at most 256 characters long and only contain the following characters: a-zA-Z0-9!-_.'()`,
|
|
264
301
|
})));
|
|
265
|
-
if (options.contentType &&
|
|
266
|
-
|
|
267
|
-
(ow.isValid(value, ow.object) && typeof value.pipe === 'function'))) {
|
|
268
|
-
throw new ArgumentError('The "value" parameter must be a String, Buffer or Stream when "options.contentType" is specified.', this.setValue);
|
|
302
|
+
if (options.contentType && !(typeof value === 'string' || isBuffer(value) || isStream(value))) {
|
|
303
|
+
throw new ArgumentError('The "value" parameter must be a String, Buffer, ArrayBuffer, TypedArray, or Stream when "options.contentType" is specified.', this.setValue);
|
|
269
304
|
}
|
|
270
305
|
ow(options, ow.object.exactShape({
|
|
271
306
|
contentType: ow.optional.string.nonEmpty,
|
|
@@ -289,13 +324,16 @@ export class KeyValueStore {
|
|
|
289
324
|
}
|
|
290
325
|
}
|
|
291
326
|
// In this case delete the record.
|
|
292
|
-
if (value === null)
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
327
|
+
if (value === null) {
|
|
328
|
+
this.statsTracker.add('deleteCount');
|
|
329
|
+
return this.backend.deleteValue(key);
|
|
330
|
+
}
|
|
331
|
+
const serialized = serializeValue(value, optionsCopy.contentType);
|
|
332
|
+
this.statsTracker.add('writeCount');
|
|
333
|
+
return this.backend.setValue({
|
|
296
334
|
key,
|
|
297
|
-
value,
|
|
298
|
-
contentType:
|
|
335
|
+
value: serialized.value,
|
|
336
|
+
contentType: serialized.contentType,
|
|
299
337
|
});
|
|
300
338
|
}
|
|
301
339
|
/**
|
|
@@ -304,7 +342,7 @@ export class KeyValueStore {
|
|
|
304
342
|
*/
|
|
305
343
|
async drop() {
|
|
306
344
|
checkStorageAccess();
|
|
307
|
-
await this.
|
|
345
|
+
await this.backend.drop();
|
|
308
346
|
serviceLocator.getStorageInstanceManager().removeFromCache(this);
|
|
309
347
|
}
|
|
310
348
|
/** @internal */
|
|
@@ -459,7 +497,7 @@ export class KeyValueStore {
|
|
|
459
497
|
* @param key The key of the record to generate the public URL for.
|
|
460
498
|
*/
|
|
461
499
|
async getPublicUrl(key) {
|
|
462
|
-
return this.
|
|
500
|
+
return this.backend.getPublicUrl(key);
|
|
463
501
|
}
|
|
464
502
|
/**
|
|
465
503
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
@@ -480,16 +518,16 @@ export class KeyValueStore {
|
|
|
480
518
|
checkStorageAccess();
|
|
481
519
|
ow(options, ow.object.exactShape({
|
|
482
520
|
config: ow.optional.object.instanceOf(Configuration),
|
|
483
|
-
|
|
521
|
+
storageBackend: ow.optional.object,
|
|
484
522
|
}));
|
|
485
523
|
options.config ??= Configuration.getGlobalConfig();
|
|
486
|
-
const
|
|
487
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true,
|
|
488
|
-
const resolved = await resolveStorageIdentifier(identifier,
|
|
524
|
+
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
525
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, config: options.config });
|
|
526
|
+
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
|
|
489
527
|
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
490
528
|
...resolved,
|
|
491
|
-
|
|
492
|
-
|
|
529
|
+
backendOpener: () => storageBackend.createKeyValueStoreBackend(resolved),
|
|
530
|
+
backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
|
|
493
531
|
});
|
|
494
532
|
}
|
|
495
533
|
/**
|
|
@@ -525,6 +563,22 @@ export class KeyValueStore {
|
|
|
525
563
|
const store = await this.open();
|
|
526
564
|
return store.getValue(key, defaultValue);
|
|
527
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Reads a record from the default {@link KeyValueStore} associated with the current crawler run
|
|
568
|
+
* without parsing the value.
|
|
569
|
+
*
|
|
570
|
+
* This is just a convenient shortcut for {@link KeyValueStore.getRecord}. Returns `null` if the
|
|
571
|
+
* record does not exist.
|
|
572
|
+
*
|
|
573
|
+
* @param key
|
|
574
|
+
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
575
|
+
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
576
|
+
* @ignore
|
|
577
|
+
*/
|
|
578
|
+
static async getRecord(key) {
|
|
579
|
+
const store = await this.open();
|
|
580
|
+
return store.getRecord(key);
|
|
581
|
+
}
|
|
528
582
|
/**
|
|
529
583
|
* Tests whether a record with the given key exists in the default {@link KeyValueStore} associated with the current crawler run.
|
|
530
584
|
* @param key The queried record key.
|
|
@@ -575,8 +629,9 @@ export class KeyValueStore {
|
|
|
575
629
|
}
|
|
576
630
|
/**
|
|
577
631
|
* Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
|
|
578
|
-
*
|
|
579
|
-
*
|
|
632
|
+
*
|
|
633
|
+
* The input is read from the default {@link KeyValueStore} under the configured input key
|
|
634
|
+
* (`CRAWLEE_INPUT_KEY`, default `INPUT`).
|
|
580
635
|
*
|
|
581
636
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
582
637
|
* If you need to use the input multiple times in your crawler,
|
|
@@ -594,29 +649,7 @@ export class KeyValueStore {
|
|
|
594
649
|
*/
|
|
595
650
|
static async getInput() {
|
|
596
651
|
const store = await this.open();
|
|
597
|
-
|
|
598
|
-
const cwd = process.cwd();
|
|
599
|
-
const possibleExtensions = ['', '.json', '.txt'];
|
|
600
|
-
// Attempt to read input from root file instead of key-value store
|
|
601
|
-
for (const extension of possibleExtensions) {
|
|
602
|
-
const inputFile = join(cwd, `${inputKey}${extension}`);
|
|
603
|
-
let input;
|
|
604
|
-
// Try getting the file from the file system
|
|
605
|
-
try {
|
|
606
|
-
input = await readFile(inputFile);
|
|
607
|
-
}
|
|
608
|
-
catch {
|
|
609
|
-
continue;
|
|
610
|
-
}
|
|
611
|
-
// Attempt to parse as JSON, or return the input as is otherwise
|
|
612
|
-
try {
|
|
613
|
-
return JSON5.parse(input.toString());
|
|
614
|
-
}
|
|
615
|
-
catch {
|
|
616
|
-
return input;
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
return store.getValue(inputKey);
|
|
652
|
+
return store.getValue(store.config.inputKey);
|
|
620
653
|
}
|
|
621
654
|
}
|
|
622
655
|
//# sourceMappingURL=key_value_store.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEvC,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEtE,gBAAgB;AAChB,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACpE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI,CAAC;YACD,kFAAkF;YAClF,KAAK,GAAG,qBAAqB,CAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;QAChF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACnD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;YAC1C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACX,wEAAwE;gBACpE,8DAA8D,CACrE,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAM,OAAO,aAAa;IAcT;IAbJ,EAAE,CAAS;IACX,IAAI,CAAU;IACN,MAAM,CAAsB;IACrC,wBAAwB,GAAG,KAAK,CAAC;IAEzC,wHAAwH;IACvG,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEvD;;OAEG;IACH,YACI,OAA6B,EACpB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IACjC,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAE/C,OAAQ,MAAM,EAAE,KAAW,IAAI,YAAY,IAAI,IAAI,CAAC;IACxD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QAExD,0HAA0H;QAC1H,wHAAwH;QACxH,qEAAqE;QACrE,kIAAkI;QAClI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,OAAO;QACX,CAAC;QAED,cAAc,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,QAAQ,GAAoB,EAAE,CAAC;YAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,QAAQ,CAAC,IAAI,CACT,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,cAAc,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,wCAAwC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC/F,CACJ,CAAC;YACN,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,CAAC,kBAAkB,CAC7B,OAAqC,EACrC,SAA6C;QAE7C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,GAAQ,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpD,IAAI,MAAM;oBAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAChE,CAAC;YACD,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,aAAa,CACxB,OAAqC,EACrC,KAAK,GAAG,sBAAsB;QAE9B,IAAI,iBAAqC,CAAC;QAE1C,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,GAAG,OAAO,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;YACnF,MAAM,KAAK,CAAC;YACZ,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;gBAAE,MAAM;YAChC,iBAAiB,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,EAAE,CACE,GAAG,EACH,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,uBAAuB,GAAG,mGAAmG;SACzI,CAAC,CAAC,CACN,CAAC;QACF,IACI,OAAO,CAAC,WAAW;YACnB,CAAC,CACG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;gBACnD,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CACrF,EACH,CAAC;YACC,MAAM,IAAI,aAAa,CACnB,mGAAmG,EACnG,IAAI,CAAC,QAAQ,CAChB,CAAC;QACN,CAAC;QACD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CACL,CAAC;QAEF,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,KAAoB,CAAC,CAAC;qBAC5C,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAExD,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;YACxB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACzB,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,UAAwC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;SACvD,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAc,UAAwC,EAAE;QAC1D,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAI,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAU,CAAC;YACnF,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CACH,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAc,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAU,CAAC,CAAC;YACnG,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,EAAK,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,IAAI,cAAc,CAAC,gBAAgB,EAAE,CAAC;QAE1E,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;QAErF,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,YAAY,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,yBAAyB,CAAC,QAAQ,CAAC;YAC9D,cAAc,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI;SACjF,CAAC,CAAC;IACP,CAAC;IA4DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAiB,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;QAEvC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,kBAAkB,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAEjD,kEAAkE;QAClE,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC,CAAC;YACvD,IAAI,KAAa,CAAC;YAElB,4CAA4C;YAC5C,IAAI,CAAC;gBACD,KAAK,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,gEAAgE;YAChE,IAAI,CAAC;gBACD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAM,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACL,OAAO,KAAqB,CAAC;YACjC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,QAAQ,CAAI,QAAQ,CAAC,CAAC;IACvC,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEvC,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAExE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAEpD,gBAAgB;AAChB,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AACH,MAAM,OAAO,aAAa;IAqBT;IApBJ,EAAE,CAAS;IACX,IAAI,CAAU;IACN,OAAO,CAAuB;IACvC,wBAAwB,GAAG,KAAK,CAAC;IAEzC,wHAAwH;IACvG,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEtC,YAAY,GAAG,IAAI,mBAAmB,CAAqB;QACxE,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,CAAC;KACf,CAAC,CAAC;IAEH;;OAEG;IACH,YACI,OAA6B,EACpB,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;IACrC,CAAC;IAkED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEhD,+FAA+F;QAC/F,kGAAkG;QAClG,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,YAAY,IAAI,IAAI,CAAC;QAChC,CAAC;QAED,8FAA8F;QAC9F,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,CAAM,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QACvB,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,OAAO;YACH,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;SAC1C,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QAExD,0HAA0H;QAC1H,wHAAwH;QACxH,qEAAqE;QACrE,kIAAkI;QAClI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,OAAO;QACX,CAAC;QAED,cAAc,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,QAAQ,GAAoB,EAAE,CAAC;YAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACpC,QAAQ,CAAC,IAAI,CACT,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,cAAc,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,wCAAwC,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAC/F,CACJ,CAAC;YACN,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,CAAC,kBAAkB,CAC7B,OAAqC,EACrC,SAA6C;QAE7C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,MAAM,OAAO,GAAQ,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrD,IAAI,MAAM,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;oBACpE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;gBAC9C,CAAC;YACL,CAAC;YACD,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,aAAa,CACxB,OAAqC,EACrC,KAAK,GAAG,sBAAsB;QAE9B,IAAI,iBAAqC,CAAC;QAE1C,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACnC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC9E,GAAG,OAAO;gBACV,iBAAiB;gBACjB,KAAK;aACR,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;YACZ,IAAI,CAAC,WAAW;gBAAE,MAAM;YACxB,iBAAiB,GAAG,qBAAqB,CAAC;QAC9C,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,EAAE,CACE,GAAG,EACH,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,uBAAuB,GAAG,mGAAmG;SACzI,CAAC,CAAC,CACN,CAAC;QACF,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC5F,MAAM,IAAI,aAAa,CACnB,6HAA6H,EAC7H,IAAI,CAAC,QAAQ,CAChB,CAAC;QACN,CAAC;QACD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CACL,CAAC;QAEF,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,IAAI,WAAW,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YACvC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACjB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACnC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAK,KAAoB,CAAC,CAAC;qBAC5C,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC;QACL,CAAC;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;QAElE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;YACzB,GAAG;YACH,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,WAAW,EAAE,UAAU,CAAC,WAAW;SACtC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1B,cAAc,CAAC,yBAAyB,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;gBACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,IAAI,CAAC,UAAwC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;YAC9C,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;SACvD,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAc,UAAwC,EAAE;QAC1D,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAI,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAU,CAAC;YACnF,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,OAAO,CACH,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,OAAO,kBAAkB,CAAC;YACtB,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAc,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAU,CAAC,CAAC;YACnG,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;SAC/B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,EAAK,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,UAA8C,EAC9C,UAA8B,EAAE;QAEhC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACrC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,cAAc,CAAC,iBAAiB,EAAE,CAAC;QAEpF,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE5F,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,UAAU,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;QAE7F,OAAO,cAAc,CAAC,yBAAyB,EAAE,CAAC,WAAW,CAAgB,IAAI,EAAE;YAC/E,GAAG,QAAQ;YACX,aAAa,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,0BAA0B,CAAC,QAAQ,CAAC;YACxE,eAAe,EAAE,cAAc,CAAC,yBAAyB,EAAE,EAAE,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI;SACnG,CAAC,CAAC;IACP,CAAC;IA4DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAiB,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAW;QAC9B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;CACJ"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical write path for key-value store records.
|
|
3
|
+
*
|
|
4
|
+
* When a content type is provided, the value passes through unchanged — it is the caller's
|
|
5
|
+
* responsibility to supply a String/Buffer/Stream (the frontend validates this).
|
|
6
|
+
*
|
|
7
|
+
* When no content type is provided, it is inferred from the value's shape:
|
|
8
|
+
* - Buffer / typed array / ArrayBuffer / stream → `application/octet-stream` (passthrough)
|
|
9
|
+
* - `string` → `text/plain; charset=utf-8` (passthrough)
|
|
10
|
+
* - anything else → `application/json; charset=utf-8` (serialized via `jsonStringifyExtended`)
|
|
11
|
+
*
|
|
12
|
+
* Does NOT drain streams — that is storage mechanics and stays in the storage backend.
|
|
13
|
+
*
|
|
14
|
+
* Backend-independent.
|
|
15
|
+
*/
|
|
16
|
+
export declare function serializeValue(value: unknown, contentType?: string): {
|
|
17
|
+
value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
|
|
18
|
+
contentType: string;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Parses a Buffer or ArrayBuffer using the provided content type header.
|
|
22
|
+
*
|
|
23
|
+
* - application/json is returned as a parsed object.
|
|
24
|
+
* - application/*xml and text/* are returned as strings.
|
|
25
|
+
* - everything else is returned as original body.
|
|
26
|
+
*
|
|
27
|
+
* If the header includes a charset, the body will be stringified only
|
|
28
|
+
* if the charset represents a known encoding to Node.js or Browser.
|
|
29
|
+
*
|
|
30
|
+
* Backend-independent — this is the canonical read path for the {@link KeyValueStore} frontend.
|
|
31
|
+
*/
|
|
32
|
+
export declare function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;
|
|
33
|
+
//# sourceMappingURL=key_value_store_codec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"key_value_store_codec.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store_codec.ts"],"names":[],"mappings":"AAUA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAC1B,KAAK,EAAE,OAAO,EACd,WAAW,CAAC,EAAE,MAAM,GACrB;IACC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,eAAe,GAAG,MAAM,GAAG,MAAM,CAAC,cAAc,GAAG,cAAc,CAAC;IAChG,WAAW,EAAE,MAAM,CAAC;CACvB,CAqCA;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,EACnC,iBAAiB,EAAE,MAAM,GAAG,IAAI,GACjC,MAAM,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAuBzD"}
|