opendal 0.1.6.pre.rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (809) hide show
  1. checksums.yaml +7 -0
  2. data/.standard.yml +20 -0
  3. data/.tool-versions +1 -0
  4. data/.yardopts +1 -0
  5. data/Cargo.toml +65 -0
  6. data/DEPENDENCIES.md +9 -0
  7. data/DEPENDENCIES.rust.tsv +277 -0
  8. data/Gemfile +35 -0
  9. data/README.md +159 -0
  10. data/Rakefile +149 -0
  11. data/build.rs +22 -0
  12. data/core/CHANGELOG.md +4929 -0
  13. data/core/CONTRIBUTING.md +61 -0
  14. data/core/Cargo.lock +10259 -0
  15. data/core/Cargo.toml +437 -0
  16. data/core/DEPENDENCIES.md +3 -0
  17. data/core/DEPENDENCIES.rust.tsv +185 -0
  18. data/core/LICENSE +201 -0
  19. data/core/README.md +228 -0
  20. data/core/benches/README.md +18 -0
  21. data/core/benches/ops/README.md +26 -0
  22. data/core/benches/ops/main.rs +25 -0
  23. data/core/benches/ops/read.rs +100 -0
  24. data/core/benches/ops/utils.rs +59 -0
  25. data/core/benches/ops/write.rs +106 -0
  26. data/core/benches/types/README.md +9 -0
  27. data/core/benches/types/buffer.rs +114 -0
  28. data/core/benches/types/main.rs +23 -0
  29. data/core/benches/types/tasks.rs +64 -0
  30. data/core/benches/vs_fs/Cargo.toml +32 -0
  31. data/core/benches/vs_fs/README.md +35 -0
  32. data/core/benches/vs_fs/src/main.rs +83 -0
  33. data/core/benches/vs_s3/Cargo.toml +38 -0
  34. data/core/benches/vs_s3/README.md +55 -0
  35. data/core/benches/vs_s3/src/main.rs +123 -0
  36. data/core/edge/README.md +3 -0
  37. data/core/edge/file_write_on_full_disk/Cargo.toml +31 -0
  38. data/core/edge/file_write_on_full_disk/README.md +14 -0
  39. data/core/edge/file_write_on_full_disk/src/main.rs +43 -0
  40. data/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml +30 -0
  41. data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
  42. data/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs +34 -0
  43. data/core/edge/s3_read_on_wasm/.gitignore +3 -0
  44. data/core/edge/s3_read_on_wasm/Cargo.toml +38 -0
  45. data/core/edge/s3_read_on_wasm/README.md +42 -0
  46. data/core/edge/s3_read_on_wasm/src/lib.rs +60 -0
  47. data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
  48. data/core/examples/README.md +23 -0
  49. data/core/examples/basic/Cargo.toml +29 -0
  50. data/core/examples/basic/README.md +15 -0
  51. data/core/examples/basic/src/main.rs +51 -0
  52. data/core/examples/concurrent-upload/Cargo.toml +29 -0
  53. data/core/examples/concurrent-upload/README.md +15 -0
  54. data/core/examples/concurrent-upload/src/main.rs +68 -0
  55. data/core/examples/multipart-upload/Cargo.toml +29 -0
  56. data/core/examples/multipart-upload/README.md +15 -0
  57. data/core/examples/multipart-upload/src/main.rs +56 -0
  58. data/core/fuzz/.gitignore +5 -0
  59. data/core/fuzz/Cargo.toml +92 -0
  60. data/core/fuzz/README.md +68 -0
  61. data/core/fuzz/fuzz_reader.rs +102 -0
  62. data/core/fuzz/fuzz_writer.rs +123 -0
  63. data/core/src/blocking/delete.rs +74 -0
  64. data/core/src/blocking/list.rs +71 -0
  65. data/core/src/blocking/mod.rs +33 -0
  66. data/core/src/blocking/operator.rs +729 -0
  67. data/core/src/blocking/read/buffer_iterator.rs +66 -0
  68. data/core/src/blocking/read/mod.rs +27 -0
  69. data/core/src/blocking/read/reader.rs +124 -0
  70. data/core/src/blocking/read/std_bytes_iterator.rs +69 -0
  71. data/core/src/blocking/read/std_reader.rs +95 -0
  72. data/core/src/blocking/write/mod.rs +22 -0
  73. data/core/src/blocking/write/std_writer.rs +82 -0
  74. data/core/src/blocking/write/writer.rs +109 -0
  75. data/core/src/docs/comparisons/mod.rs +30 -0
  76. data/core/src/docs/comparisons/vs_object_store.md +183 -0
  77. data/core/src/docs/concepts.rs +135 -0
  78. data/core/src/docs/internals/accessor.rs +306 -0
  79. data/core/src/docs/internals/layer.rs +42 -0
  80. data/core/src/docs/internals/mod.rs +62 -0
  81. data/core/src/docs/mod.rs +43 -0
  82. data/core/src/docs/performance/concurrent_write.md +101 -0
  83. data/core/src/docs/performance/http_optimization.md +124 -0
  84. data/core/src/docs/performance/mod.rs +32 -0
  85. data/core/src/docs/rfcs/0000_example.md +74 -0
  86. data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
  87. data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
  88. data/core/src/docs/rfcs/0044_error_handle.md +198 -0
  89. data/core/src/docs/rfcs/0057_auto_region.md +160 -0
  90. data/core/src/docs/rfcs/0069_object_stream.md +145 -0
  91. data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
  92. data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
  93. data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
  94. data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
  95. data/core/src/docs/rfcs/0221_create_dir.md +89 -0
  96. data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
  97. data/core/src/docs/rfcs/0293_object_id.md +67 -0
  98. data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
  99. data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
  100. data/core/src/docs/rfcs/0413_presign.md +154 -0
  101. data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
  102. data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
  103. data/core/src/docs/rfcs/0438_multipart.md +163 -0
  104. data/core/src/docs/rfcs/0443_gateway.md +73 -0
  105. data/core/src/docs/rfcs/0501_new_builder.md +111 -0
  106. data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
  107. data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
  108. data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
  109. data/core/src/docs/rfcs/0623_redis_service.md +300 -0
  110. data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
  111. data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
  112. data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
  113. data/core/src/docs/rfcs/0926_object_reader.md +93 -0
  114. data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
  115. data/core/src/docs/rfcs/1085_object_handler.md +73 -0
  116. data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
  117. data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
  118. data/core/src/docs/rfcs/1420_object_writer.md +147 -0
  119. data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
  120. data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
  121. data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
  122. data/core/src/docs/rfcs/2133_append_api.md +88 -0
  123. data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
  124. data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
  125. data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
  126. data/core/src/docs/rfcs/2774_lister_api.md +66 -0
  127. data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
  128. data/core/src/docs/rfcs/2852_native_capability.md +58 -0
  129. data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
  130. data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
  131. data/core/src/docs/rfcs/3197_config.md +237 -0
  132. data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
  133. data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
  134. data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
  135. data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
  136. data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
  137. data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
  138. data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
  139. data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
  140. data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
  141. data/core/src/docs/rfcs/4638_executor.md +215 -0
  142. data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
  143. data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
  144. data/core/src/docs/rfcs/5479_context.md +140 -0
  145. data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
  146. data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
  147. data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
  148. data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
  149. data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
  150. data/core/src/docs/rfcs/6209_glob_support.md +132 -0
  151. data/core/src/docs/rfcs/6213_options_api.md +142 -0
  152. data/core/src/docs/rfcs/README.md +62 -0
  153. data/core/src/docs/rfcs/mod.rs +278 -0
  154. data/core/src/docs/upgrade.md +1556 -0
  155. data/core/src/layers/async_backtrace.rs +174 -0
  156. data/core/src/layers/await_tree.rs +202 -0
  157. data/core/src/layers/capability_check.rs +239 -0
  158. data/core/src/layers/chaos.rs +170 -0
  159. data/core/src/layers/complete.rs +385 -0
  160. data/core/src/layers/concurrent_limit.rs +322 -0
  161. data/core/src/layers/correctness_check.rs +440 -0
  162. data/core/src/layers/dtrace.rs +294 -0
  163. data/core/src/layers/error_context.rs +310 -0
  164. data/core/src/layers/fastmetrics.rs +525 -0
  165. data/core/src/layers/fastrace.rs +271 -0
  166. data/core/src/layers/http_client.rs +206 -0
  167. data/core/src/layers/immutable_index.rs +408 -0
  168. data/core/src/layers/logging.rs +842 -0
  169. data/core/src/layers/metrics.rs +182 -0
  170. data/core/src/layers/mime_guess.rs +199 -0
  171. data/core/src/layers/mod.rs +130 -0
  172. data/core/src/layers/observe/metrics.rs +936 -0
  173. data/core/src/layers/observe/mod.rs +93 -0
  174. data/core/src/layers/otelmetrics.rs +496 -0
  175. data/core/src/layers/oteltrace.rs +203 -0
  176. data/core/src/layers/prometheus.rs +686 -0
  177. data/core/src/layers/prometheus_client.rs +519 -0
  178. data/core/src/layers/retry.rs +933 -0
  179. data/core/src/layers/throttle.rs +204 -0
  180. data/core/src/layers/timeout.rs +513 -0
  181. data/core/src/layers/tracing.rs +349 -0
  182. data/core/src/layers/type_eraser.rs +91 -0
  183. data/core/src/lib.rs +204 -0
  184. data/core/src/raw/accessor.rs +856 -0
  185. data/core/src/raw/adapters/kv/api.rs +164 -0
  186. data/core/src/raw/adapters/kv/backend.rs +253 -0
  187. data/core/src/raw/adapters/kv/mod.rs +31 -0
  188. data/core/src/raw/adapters/mod.rs +50 -0
  189. data/core/src/raw/adapters/typed_kv/api.rs +171 -0
  190. data/core/src/raw/adapters/typed_kv/backend.rs +279 -0
  191. data/core/src/raw/adapters/typed_kv/mod.rs +29 -0
  192. data/core/src/raw/atomic_util.rs +57 -0
  193. data/core/src/raw/azure.rs +570 -0
  194. data/core/src/raw/chrono_util.rs +109 -0
  195. data/core/src/raw/enum_utils.rs +201 -0
  196. data/core/src/raw/futures_util.rs +470 -0
  197. data/core/src/raw/http_util/body.rs +144 -0
  198. data/core/src/raw/http_util/bytes_content_range.rs +239 -0
  199. data/core/src/raw/http_util/bytes_range.rs +260 -0
  200. data/core/src/raw/http_util/client.rs +276 -0
  201. data/core/src/raw/http_util/error.rs +68 -0
  202. data/core/src/raw/http_util/header.rs +356 -0
  203. data/core/src/raw/http_util/mod.rs +78 -0
  204. data/core/src/raw/http_util/multipart.rs +1180 -0
  205. data/core/src/raw/http_util/uri.rs +190 -0
  206. data/core/src/raw/layer.rs +295 -0
  207. data/core/src/raw/mod.rs +101 -0
  208. data/core/src/raw/oio/buf/flex_buf.rs +118 -0
  209. data/core/src/raw/oio/buf/mod.rs +25 -0
  210. data/core/src/raw/oio/buf/pooled_buf.rs +126 -0
  211. data/core/src/raw/oio/buf/queue_buf.rs +117 -0
  212. data/core/src/raw/oio/delete/api.rs +102 -0
  213. data/core/src/raw/oio/delete/batch_delete.rs +127 -0
  214. data/core/src/raw/oio/delete/mod.rs +30 -0
  215. data/core/src/raw/oio/delete/one_shot_delete.rs +79 -0
  216. data/core/src/raw/oio/entry.rs +89 -0
  217. data/core/src/raw/oio/list/api.rs +69 -0
  218. data/core/src/raw/oio/list/flat_list.rs +137 -0
  219. data/core/src/raw/oio/list/hierarchy_list.rs +135 -0
  220. data/core/src/raw/oio/list/mod.rs +35 -0
  221. data/core/src/raw/oio/list/page_list.rs +105 -0
  222. data/core/src/raw/oio/list/prefix_list.rs +64 -0
  223. data/core/src/raw/oio/mod.rs +40 -0
  224. data/core/src/raw/oio/read/api.rs +119 -0
  225. data/core/src/raw/oio/read/mod.rs +21 -0
  226. data/core/src/raw/oio/write/api.rs +103 -0
  227. data/core/src/raw/oio/write/append_write.rs +111 -0
  228. data/core/src/raw/oio/write/block_write.rs +405 -0
  229. data/core/src/raw/oio/write/mod.rs +42 -0
  230. data/core/src/raw/oio/write/multipart_write.rs +518 -0
  231. data/core/src/raw/oio/write/one_shot_write.rs +77 -0
  232. data/core/src/raw/oio/write/position_write.rs +284 -0
  233. data/core/src/raw/operation.rs +88 -0
  234. data/core/src/raw/ops.rs +917 -0
  235. data/core/src/raw/path.rs +451 -0
  236. data/core/src/raw/path_cache.rs +244 -0
  237. data/core/src/raw/rps.rs +249 -0
  238. data/core/src/raw/serde_util.rs +423 -0
  239. data/core/src/raw/std_io_util.rs +65 -0
  240. data/core/src/raw/tests/mod.rs +30 -0
  241. data/core/src/raw/tests/read.rs +116 -0
  242. data/core/src/raw/tests/utils.rs +80 -0
  243. data/core/src/raw/tests/write.rs +79 -0
  244. data/core/src/raw/tokio_util.rs +24 -0
  245. data/core/src/raw/version.rs +19 -0
  246. data/core/src/services/aliyun_drive/backend.rs +421 -0
  247. data/core/src/services/aliyun_drive/config.rs +72 -0
  248. data/core/src/services/aliyun_drive/core.rs +651 -0
  249. data/core/src/services/aliyun_drive/delete.rs +51 -0
  250. data/core/src/services/aliyun_drive/docs.md +61 -0
  251. data/core/src/services/aliyun_drive/error.rs +56 -0
  252. data/core/src/services/aliyun_drive/lister.rs +134 -0
  253. data/core/src/services/aliyun_drive/mod.rs +39 -0
  254. data/core/src/services/aliyun_drive/writer.rs +114 -0
  255. data/core/src/services/alluxio/backend.rs +257 -0
  256. data/core/src/services/alluxio/config.rs +50 -0
  257. data/core/src/services/alluxio/core.rs +367 -0
  258. data/core/src/services/alluxio/delete.rs +38 -0
  259. data/core/src/services/alluxio/docs.md +45 -0
  260. data/core/src/services/alluxio/error.rs +99 -0
  261. data/core/src/services/alluxio/lister.rs +73 -0
  262. data/core/src/services/alluxio/mod.rs +39 -0
  263. data/core/src/services/alluxio/writer.rs +74 -0
  264. data/core/src/services/azblob/backend.rs +594 -0
  265. data/core/src/services/azblob/config.rs +220 -0
  266. data/core/src/services/azblob/core.rs +937 -0
  267. data/core/src/services/azblob/delete.rs +108 -0
  268. data/core/src/services/azblob/docs.md +77 -0
  269. data/core/src/services/azblob/error.rs +164 -0
  270. data/core/src/services/azblob/lister.rs +107 -0
  271. data/core/src/services/azblob/mod.rs +38 -0
  272. data/core/src/services/azblob/writer.rs +177 -0
  273. data/core/src/services/azdls/backend.rs +435 -0
  274. data/core/src/services/azdls/config.rs +89 -0
  275. data/core/src/services/azdls/core.rs +388 -0
  276. data/core/src/services/azdls/delete.rs +48 -0
  277. data/core/src/services/azdls/docs.md +73 -0
  278. data/core/src/services/azdls/error.rs +107 -0
  279. data/core/src/services/azdls/lister.rs +165 -0
  280. data/core/src/services/azdls/mod.rs +38 -0
  281. data/core/src/services/azdls/writer.rs +129 -0
  282. data/core/src/services/azfile/backend.rs +373 -0
  283. data/core/src/services/azfile/config.rs +61 -0
  284. data/core/src/services/azfile/core.rs +435 -0
  285. data/core/src/services/azfile/delete.rs +51 -0
  286. data/core/src/services/azfile/docs.md +65 -0
  287. data/core/src/services/azfile/error.rs +108 -0
  288. data/core/src/services/azfile/lister.rs +217 -0
  289. data/core/src/services/azfile/mod.rs +39 -0
  290. data/core/src/services/azfile/writer.rs +92 -0
  291. data/core/src/services/b2/backend.rs +434 -0
  292. data/core/src/services/b2/config.rs +64 -0
  293. data/core/src/services/b2/core.rs +742 -0
  294. data/core/src/services/b2/delete.rs +56 -0
  295. data/core/src/services/b2/docs.md +54 -0
  296. data/core/src/services/b2/error.rs +132 -0
  297. data/core/src/services/b2/lister.rs +110 -0
  298. data/core/src/services/b2/mod.rs +39 -0
  299. data/core/src/services/b2/writer.rs +189 -0
  300. data/core/src/services/cacache/backend.rs +160 -0
  301. data/core/src/services/cacache/config.rs +28 -0
  302. data/core/src/services/cacache/core.rs +96 -0
  303. data/core/src/services/cacache/delete.rs +39 -0
  304. data/core/src/services/cacache/docs.md +38 -0
  305. data/core/src/services/cacache/mod.rs +34 -0
  306. data/core/src/services/cacache/writer.rs +61 -0
  307. data/core/src/services/cloudflare_kv/backend.rs +513 -0
  308. data/core/src/services/cloudflare_kv/config.rs +55 -0
  309. data/core/src/services/cloudflare_kv/core.rs +168 -0
  310. data/core/src/services/cloudflare_kv/delete.rs +119 -0
  311. data/core/src/services/cloudflare_kv/docs.md +21 -0
  312. data/core/src/services/cloudflare_kv/error.rs +79 -0
  313. data/core/src/services/cloudflare_kv/lister.rs +170 -0
  314. data/core/src/services/cloudflare_kv/mod.rs +39 -0
  315. data/core/src/services/cloudflare_kv/model.rs +76 -0
  316. data/core/src/services/cloudflare_kv/writer.rs +68 -0
  317. data/core/src/services/compfs/backend.rs +290 -0
  318. data/core/src/services/compfs/config.rs +30 -0
  319. data/core/src/services/compfs/core.rs +159 -0
  320. data/core/src/services/compfs/delete.rs +53 -0
  321. data/core/src/services/compfs/lister.rs +98 -0
  322. data/core/src/services/compfs/mod.rs +38 -0
  323. data/core/src/services/compfs/reader.rs +79 -0
  324. data/core/src/services/compfs/writer.rs +90 -0
  325. data/core/src/services/cos/backend.rs +442 -0
  326. data/core/src/services/cos/config.rs +54 -0
  327. data/core/src/services/cos/core.rs +761 -0
  328. data/core/src/services/cos/delete.rs +48 -0
  329. data/core/src/services/cos/docs.md +55 -0
  330. data/core/src/services/cos/error.rs +105 -0
  331. data/core/src/services/cos/lister.rs +237 -0
  332. data/core/src/services/cos/mod.rs +39 -0
  333. data/core/src/services/cos/writer.rs +234 -0
  334. data/core/src/services/d1/backend.rs +330 -0
  335. data/core/src/services/d1/config.rs +55 -0
  336. data/core/src/services/d1/docs.md +48 -0
  337. data/core/src/services/d1/error.rs +79 -0
  338. data/core/src/services/d1/mod.rs +29 -0
  339. data/core/src/services/d1/model.rs +125 -0
  340. data/core/src/services/dashmap/backend.rs +203 -0
  341. data/core/src/services/dashmap/config.rs +37 -0
  342. data/core/src/services/dashmap/core.rs +61 -0
  343. data/core/src/services/dashmap/delete.rs +40 -0
  344. data/core/src/services/dashmap/docs.md +38 -0
  345. data/core/src/services/dashmap/lister.rs +63 -0
  346. data/core/src/services/dashmap/mod.rs +36 -0
  347. data/core/src/services/dashmap/writer.rs +87 -0
  348. data/core/src/services/dbfs/backend.rs +258 -0
  349. data/core/src/services/dbfs/config.rs +48 -0
  350. data/core/src/services/dbfs/core.rs +191 -0
  351. data/core/src/services/dbfs/delete.rs +49 -0
  352. data/core/src/services/dbfs/docs.md +57 -0
  353. data/core/src/services/dbfs/error.rs +74 -0
  354. data/core/src/services/dbfs/lister.rs +96 -0
  355. data/core/src/services/dbfs/mod.rs +38 -0
  356. data/core/src/services/dbfs/writer.rs +64 -0
  357. data/core/src/services/dropbox/backend.rs +187 -0
  358. data/core/src/services/dropbox/builder.rs +222 -0
  359. data/core/src/services/dropbox/config.rs +47 -0
  360. data/core/src/services/dropbox/core.rs +496 -0
  361. data/core/src/services/dropbox/delete.rs +54 -0
  362. data/core/src/services/dropbox/docs.md +64 -0
  363. data/core/src/services/dropbox/error.rs +85 -0
  364. data/core/src/services/dropbox/lister.rs +117 -0
  365. data/core/src/services/dropbox/mod.rs +40 -0
  366. data/core/src/services/dropbox/writer.rs +51 -0
  367. data/core/src/services/etcd/backend.rs +345 -0
  368. data/core/src/services/etcd/config.rs +86 -0
  369. data/core/src/services/etcd/core.rs +143 -0
  370. data/core/src/services/etcd/deleter.rs +41 -0
  371. data/core/src/services/etcd/docs.md +45 -0
  372. data/core/src/services/etcd/error.rs +26 -0
  373. data/core/src/services/etcd/lister.rs +79 -0
  374. data/core/src/services/etcd/mod.rs +36 -0
  375. data/core/src/services/etcd/writer.rs +61 -0
  376. data/core/src/services/foundationdb/backend.rs +171 -0
  377. data/core/src/services/foundationdb/config.rs +45 -0
  378. data/core/src/services/foundationdb/docs.md +42 -0
  379. data/core/src/services/foundationdb/mod.rs +24 -0
  380. data/core/src/services/fs/backend.rs +299 -0
  381. data/core/src/services/fs/config.rs +33 -0
  382. data/core/src/services/fs/core.rs +227 -0
  383. data/core/src/services/fs/delete.rs +53 -0
  384. data/core/src/services/fs/docs.md +49 -0
  385. data/core/src/services/fs/error.rs +31 -0
  386. data/core/src/services/fs/lister.rs +81 -0
  387. data/core/src/services/fs/mod.rs +40 -0
  388. data/core/src/services/fs/reader.rs +83 -0
  389. data/core/src/services/fs/writer.rs +212 -0
  390. data/core/src/services/ftp/backend.rs +388 -0
  391. data/core/src/services/ftp/config.rs +46 -0
  392. data/core/src/services/ftp/core.rs +136 -0
  393. data/core/src/services/ftp/delete.rs +62 -0
  394. data/core/src/services/ftp/docs.md +42 -0
  395. data/core/src/services/ftp/err.rs +47 -0
  396. data/core/src/services/ftp/lister.rs +72 -0
  397. data/core/src/services/ftp/mod.rs +41 -0
  398. data/core/src/services/ftp/reader.rs +84 -0
  399. data/core/src/services/ftp/writer.rs +122 -0
  400. data/core/src/services/gcs/backend.rs +499 -0
  401. data/core/src/services/gcs/config.rs +168 -0
  402. data/core/src/services/gcs/core.rs +1079 -0
  403. data/core/src/services/gcs/delete.rs +98 -0
  404. data/core/src/services/gcs/docs.md +76 -0
  405. data/core/src/services/gcs/error.rs +122 -0
  406. data/core/src/services/gcs/lister.rs +136 -0
  407. data/core/src/services/gcs/mod.rs +40 -0
  408. data/core/src/services/gcs/uri.rs +75 -0
  409. data/core/src/services/gcs/writer.rs +163 -0
  410. data/core/src/services/gdrive/backend.rs +176 -0
  411. data/core/src/services/gdrive/builder.rs +228 -0
  412. data/core/src/services/gdrive/config.rs +47 -0
  413. data/core/src/services/gdrive/core.rs +499 -0
  414. data/core/src/services/gdrive/delete.rs +57 -0
  415. data/core/src/services/gdrive/docs.md +65 -0
  416. data/core/src/services/gdrive/error.rs +80 -0
  417. data/core/src/services/gdrive/lister.rs +110 -0
  418. data/core/src/services/gdrive/mod.rs +40 -0
  419. data/core/src/services/gdrive/writer.rs +77 -0
  420. data/core/src/services/ghac/backend.rs +285 -0
  421. data/core/src/services/ghac/config.rs +36 -0
  422. data/core/src/services/ghac/core.rs +459 -0
  423. data/core/src/services/ghac/docs.md +84 -0
  424. data/core/src/services/ghac/error.rs +52 -0
  425. data/core/src/services/ghac/mod.rs +35 -0
  426. data/core/src/services/ghac/writer.rs +201 -0
  427. data/core/src/services/github/backend.rs +285 -0
  428. data/core/src/services/github/config.rs +59 -0
  429. data/core/src/services/github/core.rs +351 -0
  430. data/core/src/services/github/delete.rs +41 -0
  431. data/core/src/services/github/docs.md +52 -0
  432. data/core/src/services/github/error.rs +101 -0
  433. data/core/src/services/github/lister.rs +112 -0
  434. data/core/src/services/github/mod.rs +38 -0
  435. data/core/src/services/github/writer.rs +51 -0
  436. data/core/src/services/gridfs/backend.rs +166 -0
  437. data/core/src/services/gridfs/config.rs +50 -0
  438. data/core/src/services/gridfs/core.rs +154 -0
  439. data/core/src/services/gridfs/docs.md +46 -0
  440. data/core/src/services/gridfs/mod.rs +26 -0
  441. data/core/src/services/hdfs/backend.rs +413 -0
  442. data/core/src/services/hdfs/config.rs +59 -0
  443. data/core/src/services/hdfs/delete.rs +62 -0
  444. data/core/src/services/hdfs/docs.md +140 -0
  445. data/core/src/services/hdfs/lister.rs +70 -0
  446. data/core/src/services/hdfs/mod.rs +36 -0
  447. data/core/src/services/hdfs/reader.rs +79 -0
  448. data/core/src/services/hdfs/writer.rs +104 -0
  449. data/core/src/services/hdfs_native/backend.rs +340 -0
  450. data/core/src/services/hdfs_native/config.rs +45 -0
  451. data/core/src/services/hdfs_native/delete.rs +47 -0
  452. data/core/src/services/hdfs_native/docs.md +35 -0
  453. data/core/src/services/hdfs_native/error.rs +59 -0
  454. data/core/src/services/hdfs_native/lister.rs +85 -0
  455. data/core/src/services/hdfs_native/mod.rs +39 -0
  456. data/core/src/services/hdfs_native/reader.rs +62 -0
  457. data/core/src/services/hdfs_native/writer.rs +61 -0
  458. data/core/src/services/http/backend.rs +291 -0
  459. data/core/src/services/http/config.rs +49 -0
  460. data/core/src/services/http/core.rs +125 -0
  461. data/core/src/services/http/docs.md +45 -0
  462. data/core/src/services/http/error.rs +53 -0
  463. data/core/src/services/http/mod.rs +32 -0
  464. data/core/src/services/huggingface/backend.rs +289 -0
  465. data/core/src/services/huggingface/config.rs +75 -0
  466. data/core/src/services/huggingface/core.rs +406 -0
  467. data/core/src/services/huggingface/docs.md +61 -0
  468. data/core/src/services/huggingface/error.rs +93 -0
  469. data/core/src/services/huggingface/lister.rs +91 -0
  470. data/core/src/services/huggingface/mod.rs +34 -0
  471. data/core/src/services/ipfs/backend.rs +257 -0
  472. data/core/src/services/ipfs/config.rs +32 -0
  473. data/core/src/services/ipfs/core.rs +239 -0
  474. data/core/src/services/ipfs/docs.md +45 -0
  475. data/core/src/services/ipfs/error.rs +52 -0
  476. data/core/src/services/ipfs/ipld.rs +162 -0
  477. data/core/src/services/ipfs/mod.rs +34 -0
  478. data/core/src/services/ipmfs/backend.rs +147 -0
  479. data/core/src/services/ipmfs/builder.rs +166 -0
  480. data/core/src/services/ipmfs/config.rs +32 -0
  481. data/core/src/services/ipmfs/core.rs +142 -0
  482. data/core/src/services/ipmfs/delete.rs +48 -0
  483. data/core/src/services/ipmfs/docs.md +14 -0
  484. data/core/src/services/ipmfs/error.rs +83 -0
  485. data/core/src/services/ipmfs/lister.rs +135 -0
  486. data/core/src/services/ipmfs/mod.rs +40 -0
  487. data/core/src/services/ipmfs/writer.rs +49 -0
  488. data/core/src/services/koofr/backend.rs +361 -0
  489. data/core/src/services/koofr/config.rs +50 -0
  490. data/core/src/services/koofr/core.rs +458 -0
  491. data/core/src/services/koofr/delete.rs +50 -0
  492. data/core/src/services/koofr/docs.md +51 -0
  493. data/core/src/services/koofr/error.rs +72 -0
  494. data/core/src/services/koofr/lister.rs +88 -0
  495. data/core/src/services/koofr/mod.rs +38 -0
  496. data/core/src/services/koofr/writer.rs +53 -0
  497. data/core/src/services/lakefs/backend.rs +309 -0
  498. data/core/src/services/lakefs/config.rs +81 -0
  499. data/core/src/services/lakefs/core.rs +261 -0
  500. data/core/src/services/lakefs/delete.rs +54 -0
  501. data/core/src/services/lakefs/docs.md +62 -0
  502. data/core/src/services/lakefs/error.rs +93 -0
  503. data/core/src/services/lakefs/lister.rs +120 -0
  504. data/core/src/services/lakefs/mod.rs +38 -0
  505. data/core/src/services/lakefs/writer.rs +50 -0
  506. data/core/src/services/memcached/backend.rs +284 -0
  507. data/core/src/services/memcached/binary.rs +289 -0
  508. data/core/src/services/memcached/config.rs +43 -0
  509. data/core/src/services/memcached/docs.md +47 -0
  510. data/core/src/services/memcached/mod.rs +27 -0
  511. data/core/src/services/memory/backend.rs +205 -0
  512. data/core/src/services/memory/config.rs +30 -0
  513. data/core/src/services/memory/core.rs +80 -0
  514. data/core/src/services/memory/delete.rs +42 -0
  515. data/core/src/services/memory/docs.md +36 -0
  516. data/core/src/services/memory/lister.rs +56 -0
  517. data/core/src/services/memory/mod.rs +36 -0
  518. data/core/src/services/memory/writer.rs +85 -0
  519. data/core/src/services/mini_moka/backend.rs +260 -0
  520. data/core/src/services/mini_moka/config.rs +56 -0
  521. data/core/src/services/mini_moka/core.rs +52 -0
  522. data/core/src/services/mini_moka/delete.rs +42 -0
  523. data/core/src/services/mini_moka/docs.md +19 -0
  524. data/core/src/services/mini_moka/lister.rs +68 -0
  525. data/core/src/services/mini_moka/mod.rs +36 -0
  526. data/core/src/services/mini_moka/writer.rs +84 -0
  527. data/core/src/services/mod.rs +206 -0
  528. data/core/src/services/moka/backend.rs +326 -0
  529. data/core/src/services/moka/config.rs +59 -0
  530. data/core/src/services/moka/core.rs +62 -0
  531. data/core/src/services/moka/delete.rs +42 -0
  532. data/core/src/services/moka/docs.md +42 -0
  533. data/core/src/services/moka/lister.rs +65 -0
  534. data/core/src/services/moka/mod.rs +41 -0
  535. data/core/src/services/moka/writer.rs +83 -0
  536. data/core/src/services/mongodb/backend.rs +291 -0
  537. data/core/src/services/mongodb/config.rs +54 -0
  538. data/core/src/services/mongodb/docs.md +49 -0
  539. data/core/src/services/mongodb/mod.rs +24 -0
  540. data/core/src/services/monoiofs/backend.rs +238 -0
  541. data/core/src/services/monoiofs/config.rs +34 -0
  542. data/core/src/services/monoiofs/core.rs +313 -0
  543. data/core/src/services/monoiofs/delete.rs +64 -0
  544. data/core/src/services/monoiofs/docs.md +46 -0
  545. data/core/src/services/monoiofs/mod.rs +36 -0
  546. data/core/src/services/monoiofs/reader.rs +147 -0
  547. data/core/src/services/monoiofs/writer.rs +189 -0
  548. data/core/src/services/mysql/backend.rs +256 -0
  549. data/core/src/services/mysql/config.rs +66 -0
  550. data/core/src/services/mysql/docs.md +47 -0
  551. data/core/src/services/mysql/mod.rs +24 -0
  552. data/core/src/services/obs/backend.rs +442 -0
  553. data/core/src/services/obs/config.rs +53 -0
  554. data/core/src/services/obs/core.rs +608 -0
  555. data/core/src/services/obs/delete.rs +48 -0
  556. data/core/src/services/obs/docs.md +54 -0
  557. data/core/src/services/obs/error.rs +106 -0
  558. data/core/src/services/obs/lister.rs +101 -0
  559. data/core/src/services/obs/mod.rs +38 -0
  560. data/core/src/services/obs/writer.rs +235 -0
  561. data/core/src/services/onedrive/backend.rs +127 -0
  562. data/core/src/services/onedrive/builder.rs +236 -0
  563. data/core/src/services/onedrive/config.rs +49 -0
  564. data/core/src/services/onedrive/core.rs +691 -0
  565. data/core/src/services/onedrive/delete.rs +47 -0
  566. data/core/src/services/onedrive/docs.md +115 -0
  567. data/core/src/services/onedrive/error.rs +61 -0
  568. data/core/src/services/onedrive/graph_model.rs +425 -0
  569. data/core/src/services/onedrive/lister.rs +150 -0
  570. data/core/src/services/onedrive/mod.rs +42 -0
  571. data/core/src/services/onedrive/writer.rs +168 -0
  572. data/core/src/services/opfs/backend.rs +50 -0
  573. data/core/src/services/opfs/config.rs +25 -0
  574. data/core/src/services/opfs/core.rs +74 -0
  575. data/core/src/services/opfs/docs.md +18 -0
  576. data/core/src/services/opfs/error.rs +27 -0
  577. data/core/src/services/opfs/mod.rs +30 -0
  578. data/core/src/services/opfs/utils.rs +70 -0
  579. data/core/src/services/oss/backend.rs +734 -0
  580. data/core/src/services/oss/config.rs +113 -0
  581. data/core/src/services/oss/core.rs +1088 -0
  582. data/core/src/services/oss/delete.rs +109 -0
  583. data/core/src/services/oss/docs.md +74 -0
  584. data/core/src/services/oss/error.rs +109 -0
  585. data/core/src/services/oss/lister.rs +256 -0
  586. data/core/src/services/oss/mod.rs +38 -0
  587. data/core/src/services/oss/writer.rs +228 -0
  588. data/core/src/services/pcloud/backend.rs +358 -0
  589. data/core/src/services/pcloud/config.rs +51 -0
  590. data/core/src/services/pcloud/core.rs +461 -0
  591. data/core/src/services/pcloud/delete.rs +66 -0
  592. data/core/src/services/pcloud/docs.md +51 -0
  593. data/core/src/services/pcloud/error.rs +88 -0
  594. data/core/src/services/pcloud/lister.rs +95 -0
  595. data/core/src/services/pcloud/mod.rs +38 -0
  596. data/core/src/services/pcloud/writer.rs +66 -0
  597. data/core/src/services/persy/backend.rs +226 -0
  598. data/core/src/services/persy/config.rs +32 -0
  599. data/core/src/services/persy/docs.md +43 -0
  600. data/core/src/services/persy/mod.rs +24 -0
  601. data/core/src/services/postgresql/backend.rs +258 -0
  602. data/core/src/services/postgresql/config.rs +66 -0
  603. data/core/src/services/postgresql/docs.md +47 -0
  604. data/core/src/services/postgresql/mod.rs +24 -0
  605. data/core/src/services/redb/backend.rs +280 -0
  606. data/core/src/services/redb/config.rs +34 -0
  607. data/core/src/services/redb/docs.md +41 -0
  608. data/core/src/services/redb/mod.rs +24 -0
  609. data/core/src/services/redis/backend.rs +442 -0
  610. data/core/src/services/redis/config.rs +79 -0
  611. data/core/src/services/redis/core.rs +209 -0
  612. data/core/src/services/redis/delete.rs +40 -0
  613. data/core/src/services/redis/docs.md +43 -0
  614. data/core/src/services/redis/mod.rs +34 -0
  615. data/core/src/services/redis/writer.rs +57 -0
  616. data/core/src/services/rocksdb/backend.rs +159 -0
  617. data/core/src/services/rocksdb/config.rs +34 -0
  618. data/core/src/services/rocksdb/docs.md +54 -0
  619. data/core/src/services/rocksdb/mod.rs +24 -0
  620. data/core/src/services/s3/backend.rs +1293 -0
  621. data/core/src/services/s3/compatible_services.md +126 -0
  622. data/core/src/services/s3/config.rs +327 -0
  623. data/core/src/services/s3/core.rs +1741 -0
  624. data/core/src/services/s3/delete.rs +109 -0
  625. data/core/src/services/s3/docs.md +244 -0
  626. data/core/src/services/s3/error.rs +171 -0
  627. data/core/src/services/s3/lister.rs +405 -0
  628. data/core/src/services/s3/mod.rs +38 -0
  629. data/core/src/services/s3/writer.rs +262 -0
  630. data/core/src/services/seafile/backend.rs +297 -0
  631. data/core/src/services/seafile/config.rs +56 -0
  632. data/core/src/services/seafile/core.rs +475 -0
  633. data/core/src/services/seafile/delete.rs +40 -0
  634. data/core/src/services/seafile/docs.md +54 -0
  635. data/core/src/services/seafile/error.rs +86 -0
  636. data/core/src/services/seafile/lister.rs +83 -0
  637. data/core/src/services/seafile/mod.rs +38 -0
  638. data/core/src/services/seafile/writer.rs +55 -0
  639. data/core/src/services/sftp/backend.rs +397 -0
  640. data/core/src/services/sftp/config.rs +50 -0
  641. data/core/src/services/sftp/core.rs +154 -0
  642. data/core/src/services/sftp/delete.rs +55 -0
  643. data/core/src/services/sftp/docs.md +49 -0
  644. data/core/src/services/sftp/error.rs +57 -0
  645. data/core/src/services/sftp/lister.rs +88 -0
  646. data/core/src/services/sftp/mod.rs +42 -0
  647. data/core/src/services/sftp/reader.rs +78 -0
  648. data/core/src/services/sftp/utils.rs +51 -0
  649. data/core/src/services/sftp/writer.rs +67 -0
  650. data/core/src/services/sled/backend.rs +194 -0
  651. data/core/src/services/sled/config.rs +45 -0
  652. data/core/src/services/sled/docs.md +39 -0
  653. data/core/src/services/sled/mod.rs +24 -0
  654. data/core/src/services/sqlite/backend.rs +326 -0
  655. data/core/src/services/sqlite/config.rs +70 -0
  656. data/core/src/services/sqlite/docs.md +46 -0
  657. data/core/src/services/sqlite/mod.rs +24 -0
  658. data/core/src/services/surrealdb/backend.rs +365 -0
  659. data/core/src/services/surrealdb/config.rs +64 -0
  660. data/core/src/services/surrealdb/docs.md +54 -0
  661. data/core/src/services/surrealdb/mod.rs +24 -0
  662. data/core/src/services/swift/backend.rs +275 -0
  663. data/core/src/services/swift/compatible_services.md +53 -0
  664. data/core/src/services/swift/config.rs +53 -0
  665. data/core/src/services/swift/core.rs +310 -0
  666. data/core/src/services/swift/delete.rs +49 -0
  667. data/core/src/services/swift/docs.md +52 -0
  668. data/core/src/services/swift/error.rs +90 -0
  669. data/core/src/services/swift/lister.rs +119 -0
  670. data/core/src/services/swift/mod.rs +38 -0
  671. data/core/src/services/swift/writer.rs +53 -0
  672. data/core/src/services/tikv/backend.rs +237 -0
  673. data/core/src/services/tikv/config.rs +52 -0
  674. data/core/src/services/tikv/docs.md +43 -0
  675. data/core/src/services/tikv/mod.rs +24 -0
  676. data/core/src/services/upyun/backend.rs +317 -0
  677. data/core/src/services/upyun/config.rs +51 -0
  678. data/core/src/services/upyun/core.rs +521 -0
  679. data/core/src/services/upyun/delete.rs +50 -0
  680. data/core/src/services/upyun/docs.md +51 -0
  681. data/core/src/services/upyun/error.rs +97 -0
  682. data/core/src/services/upyun/lister.rs +101 -0
  683. data/core/src/services/upyun/mod.rs +38 -0
  684. data/core/src/services/upyun/writer.rs +127 -0
  685. data/core/src/services/vercel_artifacts/backend.rs +99 -0
  686. data/core/src/services/vercel_artifacts/builder.rs +117 -0
  687. data/core/src/services/vercel_artifacts/config.rs +39 -0
  688. data/core/src/services/vercel_artifacts/core.rs +112 -0
  689. data/core/src/services/vercel_artifacts/docs.md +40 -0
  690. data/core/src/services/vercel_artifacts/error.rs +50 -0
  691. data/core/src/services/vercel_artifacts/mod.rs +36 -0
  692. data/core/src/services/vercel_artifacts/writer.rs +58 -0
  693. data/core/src/services/vercel_blob/backend.rs +251 -0
  694. data/core/src/services/vercel_blob/config.rs +45 -0
  695. data/core/src/services/vercel_blob/core.rs +449 -0
  696. data/core/src/services/vercel_blob/delete.rs +38 -0
  697. data/core/src/services/vercel_blob/docs.md +45 -0
  698. data/core/src/services/vercel_blob/error.rs +110 -0
  699. data/core/src/services/vercel_blob/lister.rs +69 -0
  700. data/core/src/services/vercel_blob/mod.rs +38 -0
  701. data/core/src/services/vercel_blob/writer.rs +143 -0
  702. data/core/src/services/webdav/backend.rs +318 -0
  703. data/core/src/services/webdav/config.rs +53 -0
  704. data/core/src/services/webdav/core.rs +859 -0
  705. data/core/src/services/webdav/delete.rs +47 -0
  706. data/core/src/services/webdav/docs.md +49 -0
  707. data/core/src/services/webdav/error.rs +53 -0
  708. data/core/src/services/webdav/lister.rs +106 -0
  709. data/core/src/services/webdav/mod.rs +38 -0
  710. data/core/src/services/webdav/writer.rs +56 -0
  711. data/core/src/services/webhdfs/backend.rs +376 -0
  712. data/core/src/services/webhdfs/config.rs +52 -0
  713. data/core/src/services/webhdfs/core.rs +398 -0
  714. data/core/src/services/webhdfs/delete.rs +46 -0
  715. data/core/src/services/webhdfs/docs.md +90 -0
  716. data/core/src/services/webhdfs/error.rs +126 -0
  717. data/core/src/services/webhdfs/lister.rs +130 -0
  718. data/core/src/services/webhdfs/message.rs +249 -0
  719. data/core/src/services/webhdfs/mod.rs +41 -0
  720. data/core/src/services/webhdfs/writer.rs +177 -0
  721. data/core/src/services/yandex_disk/backend.rs +267 -0
  722. data/core/src/services/yandex_disk/config.rs +45 -0
  723. data/core/src/services/yandex_disk/core.rs +340 -0
  724. data/core/src/services/yandex_disk/delete.rs +54 -0
  725. data/core/src/services/yandex_disk/docs.md +45 -0
  726. data/core/src/services/yandex_disk/error.rs +104 -0
  727. data/core/src/services/yandex_disk/lister.rs +113 -0
  728. data/core/src/services/yandex_disk/mod.rs +38 -0
  729. data/core/src/services/yandex_disk/writer.rs +52 -0
  730. data/core/src/types/buffer.rs +991 -0
  731. data/core/src/types/builder.rs +152 -0
  732. data/core/src/types/capability.rs +209 -0
  733. data/core/src/types/context/mod.rs +22 -0
  734. data/core/src/types/context/read.rs +231 -0
  735. data/core/src/types/context/write.rs +441 -0
  736. data/core/src/types/delete/deleter.rs +220 -0
  737. data/core/src/types/delete/futures_delete_sink.rs +176 -0
  738. data/core/src/types/delete/input.rs +97 -0
  739. data/core/src/types/delete/mod.rs +26 -0
  740. data/core/src/types/entry.rs +69 -0
  741. data/core/src/types/error.rs +570 -0
  742. data/core/src/types/execute/api.rs +110 -0
  743. data/core/src/types/execute/executor.rs +96 -0
  744. data/core/src/types/execute/executors/mod.rs +27 -0
  745. data/core/src/types/execute/executors/tokio_executor.rs +60 -0
  746. data/core/src/types/execute/mod.rs +25 -0
  747. data/core/src/types/list.rs +137 -0
  748. data/core/src/types/metadata.rs +436 -0
  749. data/core/src/types/mod.rs +72 -0
  750. data/core/src/types/mode.rs +68 -0
  751. data/core/src/types/operator/builder.rs +535 -0
  752. data/core/src/types/operator/info.rs +63 -0
  753. data/core/src/types/operator/mod.rs +33 -0
  754. data/core/src/types/operator/operator.rs +2236 -0
  755. data/core/src/types/operator/operator_futures.rs +1430 -0
  756. data/core/src/types/operator/registry.rs +129 -0
  757. data/core/src/types/options.rs +548 -0
  758. data/core/src/types/read/buffer_stream.rs +273 -0
  759. data/core/src/types/read/futures_async_reader.rs +289 -0
  760. data/core/src/types/read/futures_bytes_stream.rs +157 -0
  761. data/core/src/types/read/mod.rs +29 -0
  762. data/core/src/types/read/reader.rs +604 -0
  763. data/core/src/types/scheme.rs +475 -0
  764. data/core/src/types/write/buffer_sink.rs +188 -0
  765. data/core/src/types/write/futures_async_writer.rs +136 -0
  766. data/core/src/types/write/futures_bytes_sink.rs +103 -0
  767. data/core/src/types/write/mod.rs +26 -0
  768. data/core/src/types/write/writer.rs +411 -0
  769. data/core/tests/behavior/README.md +77 -0
  770. data/core/tests/behavior/async_copy.rs +314 -0
  771. data/core/tests/behavior/async_create_dir.rs +53 -0
  772. data/core/tests/behavior/async_delete.rs +354 -0
  773. data/core/tests/behavior/async_list.rs +739 -0
  774. data/core/tests/behavior/async_presign.rs +175 -0
  775. data/core/tests/behavior/async_read.rs +871 -0
  776. data/core/tests/behavior/async_rename.rs +210 -0
  777. data/core/tests/behavior/async_stat.rs +628 -0
  778. data/core/tests/behavior/async_write.rs +819 -0
  779. data/core/tests/behavior/main.rs +78 -0
  780. data/core/tests/behavior/utils.rs +187 -0
  781. data/core/tests/data/normal_dir/.gitkeep +0 -0
  782. data/core/tests/data/normal_file.txt +1041 -0
  783. data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
  784. data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
  785. data/core/users.md +13 -0
  786. data/extconf.rb +24 -0
  787. data/lib/opendal.rb +25 -0
  788. data/lib/opendal_ruby/entry.rb +35 -0
  789. data/lib/opendal_ruby/io.rb +70 -0
  790. data/lib/opendal_ruby/metadata.rb +44 -0
  791. data/lib/opendal_ruby/operator.rb +29 -0
  792. data/lib/opendal_ruby/operator_info.rb +26 -0
  793. data/src/capability.rs +146 -0
  794. data/src/io.rs +464 -0
  795. data/src/lib.rs +63 -0
  796. data/src/lister.rs +141 -0
  797. data/src/metadata.rs +111 -0
  798. data/src/middlewares.rs +174 -0
  799. data/src/operator.rs +310 -0
  800. data/src/operator_info.rs +83 -0
  801. data/test/blocking_op_test.rb +112 -0
  802. data/test/capability_test.rb +42 -0
  803. data/test/io_test.rb +172 -0
  804. data/test/lister_test.rb +77 -0
  805. data/test/metadata_test.rb +78 -0
  806. data/test/middlewares_test.rb +46 -0
  807. data/test/operator_info_test.rb +35 -0
  808. data/test/test_helper.rb +36 -0
  809. metadata +857 -0
@@ -0,0 +1,691 @@
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ use std::fmt::Debug;
19
+ use std::fmt::Formatter;
20
+ use std::sync::Arc;
21
+ use std::time::Duration;
22
+
23
+ use bytes::Buf;
24
+ use bytes::Bytes;
25
+ use chrono::DateTime;
26
+ use chrono::Utc;
27
+ use http::header;
28
+ use http::Request;
29
+ use http::Response;
30
+ use http::StatusCode;
31
+ use tokio::sync::Mutex;
32
+
33
+ use super::error::parse_error;
34
+ use super::graph_model::*;
35
+ use crate::raw::*;
36
+ use crate::*;
37
+
38
+ pub struct OneDriveCore {
39
+ pub info: Arc<AccessorInfo>,
40
+ pub root: String,
41
+ pub signer: Arc<Mutex<OneDriveSigner>>,
42
+ }
43
+
44
+ impl Debug for OneDriveCore {
45
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
46
+ f.debug_struct("OneDriveCore")
47
+ .field("root", &self.root)
48
+ .finish_non_exhaustive()
49
+ }
50
+ }
51
+
52
+ // OneDrive returns 400 when try to access a dir with the POSIX special directory entries
53
+ const SPECIAL_POSIX_ENTRIES: [&str; 3] = [".", "/", ""];
54
+
55
+ // organizes a few core module functions
56
+ impl OneDriveCore {
57
+ // OneDrive personal's base URL. `me` is an alias that represents the user's "Drive".
58
+ pub(crate) const DRIVE_ROOT_URL: &str = "https://graph.microsoft.com/v1.0/me/drive/root";
59
+
60
+ /// Get a URL to an OneDrive item
61
+ pub(crate) fn onedrive_item_url(&self, path: &str, build_absolute_path: bool) -> String {
62
+ // OneDrive requires the root to be the same as `DRIVE_ROOT_URL`.
63
+ // For files under the root, the URL pattern becomes `https://graph.microsoft.com/v1.0/me/drive/root:<path>:`
64
+ if self.root == "/" && SPECIAL_POSIX_ENTRIES.contains(&path) {
65
+ Self::DRIVE_ROOT_URL.to_string()
66
+ } else {
67
+ // OneDrive returns 400 when try to access a folder with a ending slash
68
+ let absolute_path = if build_absolute_path {
69
+ let rooted_path = build_rooted_abs_path(&self.root, path);
70
+ rooted_path
71
+ .strip_suffix('/')
72
+ .unwrap_or(rooted_path.as_str())
73
+ .to_string()
74
+ } else {
75
+ path.to_string()
76
+ };
77
+ format!(
78
+ "{}:{}",
79
+ Self::DRIVE_ROOT_URL,
80
+ percent_encode_path(&absolute_path),
81
+ )
82
+ }
83
+ }
84
+
85
+ /// Send a simplest stat request about a particular path
86
+ ///
87
+ /// See also: [`onedrive_stat()`].
88
+ pub(crate) async fn onedrive_get_stat_plain(&self, path: &str) -> Result<Response<Buffer>> {
89
+ let url: String = format!(
90
+ "{}?{}",
91
+ self.onedrive_item_url(path, true),
92
+ GENERAL_SELECT_PARAM
93
+ );
94
+ let request = Request::get(&url);
95
+
96
+ let mut request = request
97
+ .extension(Operation::Stat)
98
+ .body(Buffer::new())
99
+ .map_err(new_request_build_error)?;
100
+
101
+ self.sign(&mut request).await?;
102
+
103
+ self.info.http_client().send(request).await
104
+ }
105
+
106
+ /// Create a directory at path if not exist, return the metadata about the folder
107
+ ///
108
+ /// When the folder exist, this function works exactly the same as [`onedrive_get_stat_plain()`].
109
+ ///
110
+ /// * `path` - a relative folder path
111
+ pub(crate) async fn ensure_directory(&self, path: &str) -> Result<OneDriveItem> {
112
+ let response = self.onedrive_get_stat_plain(path).await?;
113
+ let item: OneDriveItem = match response.status() {
114
+ StatusCode::OK => {
115
+ let bytes = response.into_body();
116
+ serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?
117
+ }
118
+ StatusCode::NOT_FOUND => {
119
+ // We must create directory for the destination
120
+ let response = self.onedrive_create_dir(path).await?;
121
+ match response.status() {
122
+ StatusCode::CREATED | StatusCode::OK => {
123
+ let bytes = response.into_body();
124
+ serde_json::from_reader(bytes.reader())
125
+ .map_err(new_json_deserialize_error)?
126
+ }
127
+ _ => return Err(parse_error(response)),
128
+ }
129
+ }
130
+ _ => return Err(parse_error(response)),
131
+ };
132
+
133
+ Ok(item)
134
+ }
135
+
136
+ pub(crate) async fn sign<T>(&self, request: &mut Request<T>) -> Result<()> {
137
+ let mut signer = self.signer.lock().await;
138
+ signer.sign(request).await
139
+ }
140
+ }
141
+
142
+ // OneDrive copy action is asynchronous. We query an endpoint and wait 1 second.
143
+ // This is the maximum attempts we will wait.
144
+ const MAX_MONITOR_ATTEMPT: i32 = 3600;
145
+ const MONITOR_WAIT_SECOND: u64 = 1;
146
+
147
+ // OneDrive API parameters allows using with a parameter of:
148
+ //
149
+ // - ID
150
+ // - file path
151
+ //
152
+ // `services-onedrive` uses the file path based API for simplicity.
153
+ // Read more at https://learn.microsoft.com/en-us/graph/onedrive-addressing-driveitems
154
+ impl OneDriveCore {
155
+ /// Send a stat request about a particular path, including:
156
+ ///
157
+ /// - Get stat object only if ETag not matches
158
+ /// - whether to get the object version
159
+ ///
160
+ /// See also [`onedrive_get_stat_plain()`].
161
+ pub(crate) async fn onedrive_stat(&self, path: &str, args: OpStat) -> Result<Metadata> {
162
+ let mut url: String = self.onedrive_item_url(path, true);
163
+ if args.version().is_some() {
164
+ url += "?$expand=versions(";
165
+ url += VERSION_SELECT_PARAM;
166
+ url += ")";
167
+ }
168
+
169
+ let mut request = Request::get(&url);
170
+ if let Some(etag) = args.if_none_match() {
171
+ request = request.header(header::IF_NONE_MATCH, etag);
172
+ }
173
+
174
+ let mut request = request
175
+ .extension(Operation::Stat)
176
+ .body(Buffer::new())
177
+ .map_err(new_request_build_error)?;
178
+
179
+ self.sign(&mut request).await?;
180
+
181
+ let response = self.info.http_client().send(request).await?;
182
+ if !response.status().is_success() {
183
+ return Err(parse_error(response));
184
+ }
185
+
186
+ let bytes = response.into_body();
187
+ let decoded_response: OneDriveItem =
188
+ serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
189
+
190
+ let entry_mode: EntryMode = match decoded_response.item_type {
191
+ ItemType::Folder { .. } => EntryMode::DIR,
192
+ ItemType::File { .. } => EntryMode::FILE,
193
+ };
194
+
195
+ let mut meta = Metadata::new(entry_mode)
196
+ .with_etag(decoded_response.e_tag)
197
+ .with_content_length(decoded_response.size.max(0) as u64);
198
+
199
+ if let Some(version) = args.version() {
200
+ for item_version in decoded_response.versions.as_deref().unwrap_or_default() {
201
+ if item_version.id == version {
202
+ meta.set_version(version);
203
+ break; // early exit
204
+ }
205
+ }
206
+
207
+ if meta.version().is_none() {
208
+ return Err(Error::new(
209
+ ErrorKind::NotFound,
210
+ "cannot find this version of the item",
211
+ ));
212
+ }
213
+ }
214
+
215
+ let last_modified = decoded_response.last_modified_date_time;
216
+ let date_utc_last_modified = parse_datetime_from_rfc3339(&last_modified)?;
217
+ meta.set_last_modified(date_utc_last_modified);
218
+
219
+ Ok(meta)
220
+ }
221
+
222
+ /// Return versions of an item
223
+ ///
224
+ /// A folder has no versions.
225
+ ///
226
+ /// * `path` - a relative path
227
+ pub(crate) async fn onedrive_list_versions(
228
+ &self,
229
+ path: &str,
230
+ ) -> Result<Vec<OneDriveItemVersion>> {
231
+ // don't `$select` this endpoint to get the download URL.
232
+ let url: String = format!(
233
+ "{}:/versions?{}",
234
+ self.onedrive_item_url(path, true),
235
+ VERSION_SELECT_PARAM
236
+ );
237
+
238
+ let mut request = Request::get(url)
239
+ .extension(Operation::List)
240
+ .body(Buffer::new())
241
+ .map_err(new_request_build_error)?;
242
+
243
+ self.sign(&mut request).await?;
244
+
245
+ let response = self.info.http_client().send(request).await?;
246
+ let decoded_response: GraphApiOneDriveVersionsResponse =
247
+ serde_json::from_reader(response.into_body().reader())
248
+ .map_err(new_json_deserialize_error)?;
249
+ Ok(decoded_response.value)
250
+ }
251
+
252
+ pub(crate) async fn onedrive_get_next_list_page(&self, url: &str) -> Result<Response<Buffer>> {
253
+ let mut request = Request::get(url)
254
+ .extension(Operation::List)
255
+ .body(Buffer::new())
256
+ .map_err(new_request_build_error)?;
257
+
258
+ self.sign(&mut request).await?;
259
+
260
+ self.info.http_client().send(request).await
261
+ }
262
+
263
+ /// Download a file
264
+ ///
265
+ /// OneDrive handles a download in 2 steps:
266
+ /// 1. Returns a 302 with a presigned URL. If `If-None-Match` succeed, returns 304.
267
+ /// 2. With the presigned URL, we can send a GET:
268
+ /// 1. When getting an item succeed with a `Range` header, we get a 206 Partial Content response.
269
+ /// 2. When succeed, we get a 200 response.
270
+ ///
271
+ /// Read more at https://learn.microsoft.com/en-us/graph/api/driveitem-get-content
272
+ pub(crate) async fn onedrive_get_content(
273
+ &self,
274
+ path: &str,
275
+ args: &OpRead,
276
+ ) -> Result<Response<HttpBody>> {
277
+ // We can't "select" the OneDrive API response fields when reading because "select" shadows not found error
278
+ let url: String = format!("{}:/content", self.onedrive_item_url(path, true));
279
+
280
+ let mut request = Request::get(&url).header(header::RANGE, args.range().to_header());
281
+ if let Some(etag) = args.if_none_match() {
282
+ request = request.header(header::IF_NONE_MATCH, etag);
283
+ }
284
+
285
+ let mut request = request
286
+ .extension(Operation::Read)
287
+ .body(Buffer::new())
288
+ .map_err(new_request_build_error)?;
289
+
290
+ self.sign(&mut request).await?;
291
+
292
+ self.info.http_client().fetch(request).await
293
+ }
294
+
295
+ /// Upload a file
296
+ ///
297
+ /// When creating a file,
298
+ ///
299
+ /// * OneDrive returns 201 if the file is new.
300
+ /// * OneDrive returns 200 if successfully overwrote the file successfully.
301
+ ///
302
+ /// Read more at https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content
303
+ ///
304
+ /// This function is different than uploading a file with chunks.
305
+ /// See also [`create_upload_session()`] and [`OneDriveWriter::write_chunked`].
306
+ pub async fn onedrive_upload_simple(
307
+ &self,
308
+ path: &str,
309
+ args: &OpWrite,
310
+ body: Buffer,
311
+ ) -> Result<Response<Buffer>> {
312
+ let url = format!(
313
+ "{}:/content?@microsoft.graph.conflictBehavior={}&{}",
314
+ self.onedrive_item_url(path, true),
315
+ REPLACE_EXISTING_ITEM_WHEN_CONFLICT,
316
+ GENERAL_SELECT_PARAM
317
+ );
318
+
319
+ // OneDrive upload API documentation requires "text/plain" as the content type.
320
+ // In practice, OneDrive ignores the content type,
321
+ // but decides the type (when stating) based on the extension name.
322
+ // Also, when the extension name is unknown to OneDrive,
323
+ // OneDrive sets the content type as "application/octet-stream".
324
+ // We keep the content type according to the documentation.
325
+ let mut request = Request::put(&url)
326
+ .header(header::CONTENT_LENGTH, body.len())
327
+ .header(header::CONTENT_TYPE, "text/plain");
328
+
329
+ // when creating a new file, `IF-Match` has no effect.
330
+ // when updating a file with the `If-Match`, and if the ETag mismatched,
331
+ // OneDrive will return 412 Precondition Failed
332
+ if let Some(if_match) = args.if_match() {
333
+ request = request.header(header::IF_MATCH, if_match);
334
+ }
335
+
336
+ let mut request = request
337
+ .extension(Operation::Write)
338
+ .body(body)
339
+ .map_err(new_request_build_error)?;
340
+
341
+ self.sign(&mut request).await?;
342
+
343
+ self.info.http_client().send(request).await
344
+ }
345
+
346
+ pub(crate) async fn onedrive_chunked_upload(
347
+ &self,
348
+ url: &str,
349
+ args: &OpWrite,
350
+ offset: usize,
351
+ chunk_end: usize,
352
+ total_len: usize,
353
+ body: Buffer,
354
+ ) -> Result<Response<Buffer>> {
355
+ let mut request = Request::put(url);
356
+
357
+ let range = format!("bytes {offset}-{chunk_end}/{total_len}");
358
+ request = request.header(header::CONTENT_RANGE, range);
359
+
360
+ let size = chunk_end - offset + 1;
361
+ request = request.header(header::CONTENT_LENGTH, size);
362
+
363
+ if let Some(mime) = args.content_type() {
364
+ request = request.header(header::CONTENT_TYPE, mime)
365
+ }
366
+
367
+ let request = request
368
+ .extension(Operation::Write)
369
+ .body(body)
370
+ .map_err(new_request_build_error)?;
371
+ // OneDrive documentation requires not sending the `Authorization` header
372
+
373
+ self.info.http_client().send(request).await
374
+ }
375
+
376
+ /// Create a upload session for chunk uploads
377
+ ///
378
+ /// This endpoint supports `If-None-Match` but [`onedrive_upload_simple()`] doesn't.
379
+ ///
380
+ /// Read more at https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online#upload-bytes-to-the-upload-session
381
+ pub(crate) async fn onedrive_create_upload_session(
382
+ &self,
383
+ path: &str,
384
+ args: &OpWrite,
385
+ ) -> Result<Response<Buffer>> {
386
+ let parent_path = get_parent(path);
387
+ let file_name = get_basename(path);
388
+ let url = format!(
389
+ "{}:/createUploadSession",
390
+ self.onedrive_item_url(parent_path, true),
391
+ );
392
+ let mut request = Request::post(url).header(header::CONTENT_TYPE, "application/json");
393
+
394
+ if let Some(if_match) = args.if_match() {
395
+ request = request.header(header::IF_MATCH, if_match);
396
+ }
397
+
398
+ let body = OneDriveUploadSessionCreationRequestBody::new(file_name.to_string());
399
+ let body_bytes = serde_json::to_vec(&body).map_err(new_json_serialize_error)?;
400
+ let body = Buffer::from(Bytes::from(body_bytes));
401
+ let mut request = request
402
+ .extension(Operation::Write)
403
+ .body(body)
404
+ .map_err(new_request_build_error)?;
405
+
406
+ self.sign(&mut request).await?;
407
+
408
+ self.info.http_client().send(request).await
409
+ }
410
+
411
+ /// Create a directory
412
+ ///
413
+ /// When creating a folder, OneDrive returns a status code with 201.
414
+ /// When using `microsoft.graph.conflictBehavior=replace` to replace a folder, OneDrive returns 200.
415
+ ///
416
+ /// * `path` - the path to the folder without the root
417
+ pub(crate) async fn onedrive_create_dir(&self, path: &str) -> Result<Response<Buffer>> {
418
+ let parent_path = get_parent(path);
419
+ let basename = get_basename(path);
420
+ let folder_name = basename.strip_suffix('/').unwrap_or(basename);
421
+
422
+ let url = format!(
423
+ "{}:/children?{}",
424
+ self.onedrive_item_url(parent_path, true),
425
+ GENERAL_SELECT_PARAM
426
+ );
427
+
428
+ let payload = CreateDirPayload::new(folder_name.to_string());
429
+ let body_bytes = serde_json::to_vec(&payload).map_err(new_json_serialize_error)?;
430
+ let body = Buffer::from(bytes::Bytes::from(body_bytes));
431
+
432
+ let mut request = Request::post(url)
433
+ .header(header::CONTENT_TYPE, "application/json")
434
+ .extension(Operation::CreateDir)
435
+ .body(body)
436
+ .map_err(new_request_build_error)?;
437
+
438
+ self.sign(&mut request).await?;
439
+
440
+ self.info.http_client().send(request).await
441
+ }
442
+
443
+ /// Delete a `DriveItem`
444
+ ///
445
+ /// This moves the items to the recycle bin.
446
+ pub(crate) async fn onedrive_delete(&self, path: &str) -> Result<Response<Buffer>> {
447
+ let url = self.onedrive_item_url(path, true);
448
+
449
+ let mut request = Request::delete(&url)
450
+ .extension(Operation::Delete)
451
+ .body(Buffer::new())
452
+ .map_err(new_request_build_error)?;
453
+
454
+ self.sign(&mut request).await?;
455
+
456
+ self.info.http_client().send(request).await
457
+ }
458
+
459
+ /// Initialize a copy
460
+ ///
461
+ /// * `source` - the path to the source folder without the root
462
+ /// * `destination` - the path to the destination folder without the root
463
+ ///
464
+ /// See also: [`wait_until_complete()`]
465
+ pub(crate) async fn initialize_copy(&self, source: &str, destination: &str) -> Result<String> {
466
+ // we must validate if source exist
467
+ let response = self.onedrive_get_stat_plain(source).await?;
468
+ if !response.status().is_success() {
469
+ return Err(parse_error(response));
470
+ }
471
+
472
+ // We need to stat the destination parent folder to get a parent reference
473
+ let destination_parent = get_parent(destination).to_string();
474
+ let basename = get_basename(destination);
475
+
476
+ let item = self.ensure_directory(&destination_parent).await?;
477
+ let body = OneDrivePatchRequestBody {
478
+ parent_reference: ParentReference {
479
+ path: "".to_string(), // irrelevant for copy
480
+ drive_id: item.parent_reference.drive_id,
481
+ id: item.id,
482
+ },
483
+ name: basename.to_string(),
484
+ };
485
+
486
+ // ensure the destination file or folder doesn't exist
487
+ let response = self.onedrive_get_stat_plain(destination).await?;
488
+ match response.status() {
489
+ // We must remove the file or folder because
490
+ // OneDrive doesn't support `conflictBehavior` for the consumer OneDrive.
491
+ // `conflictBehavior` seems to work for the consumer OneDrive sometimes could be a coincidence.
492
+ // Read more at https://learn.microsoft.com/en-us/graph/api/driveitem-copy
493
+ StatusCode::OK => {
494
+ let response = self.onedrive_delete(destination).await?;
495
+ match response.status() {
496
+ StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => {} // expected, intentionally empty
497
+ _ => return Err(parse_error(response)),
498
+ }
499
+ }
500
+ StatusCode::NOT_FOUND => {} // expected, intentionally empty
501
+ _ => return Err(parse_error(response)),
502
+ }
503
+
504
+ let url: String = format!("{}:/copy", self.onedrive_item_url(source, true));
505
+
506
+ let body_bytes = serde_json::to_vec(&body).map_err(new_json_serialize_error)?;
507
+ let buffer = Buffer::from(Bytes::from(body_bytes));
508
+ let mut request = Request::post(&url)
509
+ .header(header::CONTENT_TYPE, "application/json")
510
+ .extension(Operation::Copy)
511
+ .body(buffer)
512
+ .map_err(new_request_build_error)?;
513
+
514
+ self.sign(&mut request).await?;
515
+
516
+ let response = self.info.http_client().send(request).await?;
517
+ match response.status() {
518
+ StatusCode::ACCEPTED => parse_location(response.headers())?
519
+ .ok_or_else(|| {
520
+ Error::new(
521
+ ErrorKind::Unexpected,
522
+ "OneDrive didn't return a location URL",
523
+ )
524
+ })
525
+ .map(String::from),
526
+ _ => Err(parse_error(response)),
527
+ }
528
+ }
529
+
530
+ pub(crate) async fn wait_until_complete(&self, monitor_url: String) -> Result<()> {
531
+ for _attempt in 0..MAX_MONITOR_ATTEMPT {
532
+ let mut request = Request::get(monitor_url.to_string())
533
+ .header(header::CONTENT_TYPE, "application/json")
534
+ .extension(Operation::Copy)
535
+ .body(Buffer::new())
536
+ .map_err(new_request_build_error)?;
537
+
538
+ self.sign(&mut request).await?;
539
+
540
+ let response = self.info.http_client().send(request).await?;
541
+ let status: OneDriveMonitorStatus =
542
+ serde_json::from_reader(response.into_body().reader())
543
+ .map_err(new_json_deserialize_error)?;
544
+ if status.status == "completed" {
545
+ return Ok(());
546
+ }
547
+
548
+ tokio::time::sleep(Duration::from_secs(MONITOR_WAIT_SECOND)).await;
549
+ }
550
+
551
+ Err(Error::new(
552
+ ErrorKind::Unexpected,
553
+ "Exceed monitoring timeout",
554
+ ))
555
+ }
556
+
557
+ pub(crate) async fn onedrive_move(&self, source: &str, destination: &str) -> Result<()> {
558
+ // We must validate if the source folder exists.
559
+ let response = self.onedrive_get_stat_plain(source).await?;
560
+ if !response.status().is_success() {
561
+ return Err(Error::new(ErrorKind::NotFound, "source not found"));
562
+ }
563
+
564
+ // We want a parent reference about the destination's parent, or the destination folder itself.
565
+ let destination_parent = get_parent(destination).to_string();
566
+ let basename = get_basename(destination);
567
+
568
+ let item = self.ensure_directory(&destination_parent).await?;
569
+ let body = OneDrivePatchRequestBody {
570
+ parent_reference: ParentReference {
571
+ path: "".to_string(), // irrelevant for update
572
+ // reusing `ParentReference` for convenience. The API requires this value to be correct.
573
+ drive_id: item.parent_reference.drive_id,
574
+ id: item.id,
575
+ },
576
+ name: basename.to_string(),
577
+ };
578
+ let body_bytes = serde_json::to_vec(&body).map_err(new_json_serialize_error)?;
579
+ let buffer = Buffer::from(Bytes::from(body_bytes));
580
+ let url: String = format!(
581
+ "{}?@microsoft.graph.conflictBehavior={}&$select=id",
582
+ self.onedrive_item_url(source, true),
583
+ REPLACE_EXISTING_ITEM_WHEN_CONFLICT
584
+ );
585
+ let mut request = Request::patch(&url)
586
+ .header(header::CONTENT_TYPE, "application/json")
587
+ .extension(Operation::Rename)
588
+ .body(buffer)
589
+ .map_err(new_request_build_error)?;
590
+
591
+ self.sign(&mut request).await?;
592
+
593
+ let response = self.info.http_client().send(request).await?;
594
+ match response.status() {
595
+ // can get etag, metadata, etc...
596
+ StatusCode::OK => Ok(()),
597
+ _ => Err(parse_error(response)),
598
+ }
599
+ }
600
+ }
601
+
602
+ // keeps track of OAuth 2.0 tokens and refreshes the access token.
603
+ pub struct OneDriveSigner {
604
+ pub info: Arc<AccessorInfo>, // to use `http_client`
605
+
606
+ pub client_id: String,
607
+ pub client_secret: String,
608
+ pub refresh_token: String,
609
+
610
+ pub access_token: String,
611
+ pub expires_in: DateTime<Utc>,
612
+ }
613
+
614
+ // OneDrive is part of Graph API hence shares the same authentication and authorization processes.
615
+ // `common` applies to account types:
616
+ //
617
+ // - consumers
618
+ // - work and school account
619
+ //
620
+ // set to `common` for simplicity
621
+ const ONEDRIVE_REFRESH_TOKEN: &str = "https://login.microsoftonline.com/common/oauth2/v2.0/token";
622
+
623
+ impl OneDriveSigner {
624
+ pub fn new(info: Arc<AccessorInfo>) -> Self {
625
+ OneDriveSigner {
626
+ info,
627
+
628
+ client_id: "".to_string(),
629
+ client_secret: "".to_string(),
630
+ refresh_token: "".to_string(),
631
+ access_token: "".to_string(),
632
+ expires_in: DateTime::<Utc>::MIN_UTC,
633
+ }
634
+ }
635
+
636
+ async fn refresh_tokens(&mut self) -> Result<()> {
637
+ // OneDrive users must provide at least this required permission scope
638
+ let encoded_payload = format!(
639
+ "client_id={}&client_secret={}&scope=offline_access%20Files.ReadWrite&refresh_token={}&grant_type=refresh_token",
640
+ percent_encode_path(self.client_id.as_str()),
641
+ percent_encode_path(self.client_secret.as_str()),
642
+ percent_encode_path(self.refresh_token.as_str())
643
+ );
644
+ let request = Request::post(ONEDRIVE_REFRESH_TOKEN)
645
+ .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
646
+ .body(Buffer::from(encoded_payload))
647
+ .map_err(new_request_build_error)?;
648
+
649
+ let response = self.info.http_client().send(request).await?;
650
+ match response.status() {
651
+ StatusCode::OK => {
652
+ let resp_body = response.into_body();
653
+ let data: GraphOAuthRefreshTokenResponseBody =
654
+ serde_json::from_reader(resp_body.reader())
655
+ .map_err(new_json_deserialize_error)?;
656
+ self.access_token = data.access_token;
657
+ self.refresh_token = data.refresh_token;
658
+ self.expires_in = Utc::now()
659
+ + chrono::TimeDelta::try_seconds(data.expires_in)
660
+ .expect("expires_in must be valid seconds")
661
+ - chrono::TimeDelta::minutes(2); // assumes 2 mins graceful transmission for implementation simplicity
662
+ Ok(())
663
+ }
664
+ _ => Err(parse_error(response)),
665
+ }
666
+ }
667
+
668
+ /// Sign a request.
669
+ pub async fn sign<T>(&mut self, request: &mut Request<T>) -> Result<()> {
670
+ if !self.access_token.is_empty() && self.expires_in > Utc::now() {
671
+ let value = format!("Bearer {}", self.access_token)
672
+ .parse()
673
+ .expect("access_token must be valid header value");
674
+
675
+ request.headers_mut().insert(header::AUTHORIZATION, value);
676
+ return Ok(());
677
+ }
678
+
679
+ self.refresh_tokens().await?;
680
+
681
+ let auth_header_content = format!("Bearer {}", self.access_token)
682
+ .parse()
683
+ .expect("Fetched access_token is invalid as a header value");
684
+
685
+ request
686
+ .headers_mut()
687
+ .insert(header::AUTHORIZATION, auth_header_content);
688
+
689
+ Ok(())
690
+ }
691
+ }