opendal 0.1.6.pre.rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (809) hide show
  1. checksums.yaml +7 -0
  2. data/.standard.yml +20 -0
  3. data/.tool-versions +1 -0
  4. data/.yardopts +1 -0
  5. data/Cargo.toml +65 -0
  6. data/DEPENDENCIES.md +9 -0
  7. data/DEPENDENCIES.rust.tsv +277 -0
  8. data/Gemfile +35 -0
  9. data/README.md +159 -0
  10. data/Rakefile +149 -0
  11. data/build.rs +22 -0
  12. data/core/CHANGELOG.md +4929 -0
  13. data/core/CONTRIBUTING.md +61 -0
  14. data/core/Cargo.lock +10259 -0
  15. data/core/Cargo.toml +437 -0
  16. data/core/DEPENDENCIES.md +3 -0
  17. data/core/DEPENDENCIES.rust.tsv +185 -0
  18. data/core/LICENSE +201 -0
  19. data/core/README.md +228 -0
  20. data/core/benches/README.md +18 -0
  21. data/core/benches/ops/README.md +26 -0
  22. data/core/benches/ops/main.rs +25 -0
  23. data/core/benches/ops/read.rs +100 -0
  24. data/core/benches/ops/utils.rs +59 -0
  25. data/core/benches/ops/write.rs +106 -0
  26. data/core/benches/types/README.md +9 -0
  27. data/core/benches/types/buffer.rs +114 -0
  28. data/core/benches/types/main.rs +23 -0
  29. data/core/benches/types/tasks.rs +64 -0
  30. data/core/benches/vs_fs/Cargo.toml +32 -0
  31. data/core/benches/vs_fs/README.md +35 -0
  32. data/core/benches/vs_fs/src/main.rs +83 -0
  33. data/core/benches/vs_s3/Cargo.toml +38 -0
  34. data/core/benches/vs_s3/README.md +55 -0
  35. data/core/benches/vs_s3/src/main.rs +123 -0
  36. data/core/edge/README.md +3 -0
  37. data/core/edge/file_write_on_full_disk/Cargo.toml +31 -0
  38. data/core/edge/file_write_on_full_disk/README.md +14 -0
  39. data/core/edge/file_write_on_full_disk/src/main.rs +43 -0
  40. data/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml +30 -0
  41. data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
  42. data/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs +34 -0
  43. data/core/edge/s3_read_on_wasm/.gitignore +3 -0
  44. data/core/edge/s3_read_on_wasm/Cargo.toml +38 -0
  45. data/core/edge/s3_read_on_wasm/README.md +42 -0
  46. data/core/edge/s3_read_on_wasm/src/lib.rs +60 -0
  47. data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
  48. data/core/examples/README.md +23 -0
  49. data/core/examples/basic/Cargo.toml +29 -0
  50. data/core/examples/basic/README.md +15 -0
  51. data/core/examples/basic/src/main.rs +51 -0
  52. data/core/examples/concurrent-upload/Cargo.toml +29 -0
  53. data/core/examples/concurrent-upload/README.md +15 -0
  54. data/core/examples/concurrent-upload/src/main.rs +68 -0
  55. data/core/examples/multipart-upload/Cargo.toml +29 -0
  56. data/core/examples/multipart-upload/README.md +15 -0
  57. data/core/examples/multipart-upload/src/main.rs +56 -0
  58. data/core/fuzz/.gitignore +5 -0
  59. data/core/fuzz/Cargo.toml +92 -0
  60. data/core/fuzz/README.md +68 -0
  61. data/core/fuzz/fuzz_reader.rs +102 -0
  62. data/core/fuzz/fuzz_writer.rs +123 -0
  63. data/core/src/blocking/delete.rs +74 -0
  64. data/core/src/blocking/list.rs +71 -0
  65. data/core/src/blocking/mod.rs +33 -0
  66. data/core/src/blocking/operator.rs +729 -0
  67. data/core/src/blocking/read/buffer_iterator.rs +66 -0
  68. data/core/src/blocking/read/mod.rs +27 -0
  69. data/core/src/blocking/read/reader.rs +124 -0
  70. data/core/src/blocking/read/std_bytes_iterator.rs +69 -0
  71. data/core/src/blocking/read/std_reader.rs +95 -0
  72. data/core/src/blocking/write/mod.rs +22 -0
  73. data/core/src/blocking/write/std_writer.rs +82 -0
  74. data/core/src/blocking/write/writer.rs +109 -0
  75. data/core/src/docs/comparisons/mod.rs +30 -0
  76. data/core/src/docs/comparisons/vs_object_store.md +183 -0
  77. data/core/src/docs/concepts.rs +135 -0
  78. data/core/src/docs/internals/accessor.rs +306 -0
  79. data/core/src/docs/internals/layer.rs +42 -0
  80. data/core/src/docs/internals/mod.rs +62 -0
  81. data/core/src/docs/mod.rs +43 -0
  82. data/core/src/docs/performance/concurrent_write.md +101 -0
  83. data/core/src/docs/performance/http_optimization.md +124 -0
  84. data/core/src/docs/performance/mod.rs +32 -0
  85. data/core/src/docs/rfcs/0000_example.md +74 -0
  86. data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
  87. data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
  88. data/core/src/docs/rfcs/0044_error_handle.md +198 -0
  89. data/core/src/docs/rfcs/0057_auto_region.md +160 -0
  90. data/core/src/docs/rfcs/0069_object_stream.md +145 -0
  91. data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
  92. data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
  93. data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
  94. data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
  95. data/core/src/docs/rfcs/0221_create_dir.md +89 -0
  96. data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
  97. data/core/src/docs/rfcs/0293_object_id.md +67 -0
  98. data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
  99. data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
  100. data/core/src/docs/rfcs/0413_presign.md +154 -0
  101. data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
  102. data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
  103. data/core/src/docs/rfcs/0438_multipart.md +163 -0
  104. data/core/src/docs/rfcs/0443_gateway.md +73 -0
  105. data/core/src/docs/rfcs/0501_new_builder.md +111 -0
  106. data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
  107. data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
  108. data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
  109. data/core/src/docs/rfcs/0623_redis_service.md +300 -0
  110. data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
  111. data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
  112. data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
  113. data/core/src/docs/rfcs/0926_object_reader.md +93 -0
  114. data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
  115. data/core/src/docs/rfcs/1085_object_handler.md +73 -0
  116. data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
  117. data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
  118. data/core/src/docs/rfcs/1420_object_writer.md +147 -0
  119. data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
  120. data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
  121. data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
  122. data/core/src/docs/rfcs/2133_append_api.md +88 -0
  123. data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
  124. data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
  125. data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
  126. data/core/src/docs/rfcs/2774_lister_api.md +66 -0
  127. data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
  128. data/core/src/docs/rfcs/2852_native_capability.md +58 -0
  129. data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
  130. data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
  131. data/core/src/docs/rfcs/3197_config.md +237 -0
  132. data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
  133. data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
  134. data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
  135. data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
  136. data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
  137. data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
  138. data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
  139. data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
  140. data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
  141. data/core/src/docs/rfcs/4638_executor.md +215 -0
  142. data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
  143. data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
  144. data/core/src/docs/rfcs/5479_context.md +140 -0
  145. data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
  146. data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
  147. data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
  148. data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
  149. data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
  150. data/core/src/docs/rfcs/6209_glob_support.md +132 -0
  151. data/core/src/docs/rfcs/6213_options_api.md +142 -0
  152. data/core/src/docs/rfcs/README.md +62 -0
  153. data/core/src/docs/rfcs/mod.rs +278 -0
  154. data/core/src/docs/upgrade.md +1556 -0
  155. data/core/src/layers/async_backtrace.rs +174 -0
  156. data/core/src/layers/await_tree.rs +202 -0
  157. data/core/src/layers/capability_check.rs +239 -0
  158. data/core/src/layers/chaos.rs +170 -0
  159. data/core/src/layers/complete.rs +385 -0
  160. data/core/src/layers/concurrent_limit.rs +322 -0
  161. data/core/src/layers/correctness_check.rs +440 -0
  162. data/core/src/layers/dtrace.rs +294 -0
  163. data/core/src/layers/error_context.rs +310 -0
  164. data/core/src/layers/fastmetrics.rs +525 -0
  165. data/core/src/layers/fastrace.rs +271 -0
  166. data/core/src/layers/http_client.rs +206 -0
  167. data/core/src/layers/immutable_index.rs +408 -0
  168. data/core/src/layers/logging.rs +842 -0
  169. data/core/src/layers/metrics.rs +182 -0
  170. data/core/src/layers/mime_guess.rs +199 -0
  171. data/core/src/layers/mod.rs +130 -0
  172. data/core/src/layers/observe/metrics.rs +936 -0
  173. data/core/src/layers/observe/mod.rs +93 -0
  174. data/core/src/layers/otelmetrics.rs +496 -0
  175. data/core/src/layers/oteltrace.rs +203 -0
  176. data/core/src/layers/prometheus.rs +686 -0
  177. data/core/src/layers/prometheus_client.rs +519 -0
  178. data/core/src/layers/retry.rs +933 -0
  179. data/core/src/layers/throttle.rs +204 -0
  180. data/core/src/layers/timeout.rs +513 -0
  181. data/core/src/layers/tracing.rs +349 -0
  182. data/core/src/layers/type_eraser.rs +91 -0
  183. data/core/src/lib.rs +204 -0
  184. data/core/src/raw/accessor.rs +856 -0
  185. data/core/src/raw/adapters/kv/api.rs +164 -0
  186. data/core/src/raw/adapters/kv/backend.rs +253 -0
  187. data/core/src/raw/adapters/kv/mod.rs +31 -0
  188. data/core/src/raw/adapters/mod.rs +50 -0
  189. data/core/src/raw/adapters/typed_kv/api.rs +171 -0
  190. data/core/src/raw/adapters/typed_kv/backend.rs +279 -0
  191. data/core/src/raw/adapters/typed_kv/mod.rs +29 -0
  192. data/core/src/raw/atomic_util.rs +57 -0
  193. data/core/src/raw/azure.rs +570 -0
  194. data/core/src/raw/chrono_util.rs +109 -0
  195. data/core/src/raw/enum_utils.rs +201 -0
  196. data/core/src/raw/futures_util.rs +470 -0
  197. data/core/src/raw/http_util/body.rs +144 -0
  198. data/core/src/raw/http_util/bytes_content_range.rs +239 -0
  199. data/core/src/raw/http_util/bytes_range.rs +260 -0
  200. data/core/src/raw/http_util/client.rs +276 -0
  201. data/core/src/raw/http_util/error.rs +68 -0
  202. data/core/src/raw/http_util/header.rs +356 -0
  203. data/core/src/raw/http_util/mod.rs +78 -0
  204. data/core/src/raw/http_util/multipart.rs +1180 -0
  205. data/core/src/raw/http_util/uri.rs +190 -0
  206. data/core/src/raw/layer.rs +295 -0
  207. data/core/src/raw/mod.rs +101 -0
  208. data/core/src/raw/oio/buf/flex_buf.rs +118 -0
  209. data/core/src/raw/oio/buf/mod.rs +25 -0
  210. data/core/src/raw/oio/buf/pooled_buf.rs +126 -0
  211. data/core/src/raw/oio/buf/queue_buf.rs +117 -0
  212. data/core/src/raw/oio/delete/api.rs +102 -0
  213. data/core/src/raw/oio/delete/batch_delete.rs +127 -0
  214. data/core/src/raw/oio/delete/mod.rs +30 -0
  215. data/core/src/raw/oio/delete/one_shot_delete.rs +79 -0
  216. data/core/src/raw/oio/entry.rs +89 -0
  217. data/core/src/raw/oio/list/api.rs +69 -0
  218. data/core/src/raw/oio/list/flat_list.rs +137 -0
  219. data/core/src/raw/oio/list/hierarchy_list.rs +135 -0
  220. data/core/src/raw/oio/list/mod.rs +35 -0
  221. data/core/src/raw/oio/list/page_list.rs +105 -0
  222. data/core/src/raw/oio/list/prefix_list.rs +64 -0
  223. data/core/src/raw/oio/mod.rs +40 -0
  224. data/core/src/raw/oio/read/api.rs +119 -0
  225. data/core/src/raw/oio/read/mod.rs +21 -0
  226. data/core/src/raw/oio/write/api.rs +103 -0
  227. data/core/src/raw/oio/write/append_write.rs +111 -0
  228. data/core/src/raw/oio/write/block_write.rs +405 -0
  229. data/core/src/raw/oio/write/mod.rs +42 -0
  230. data/core/src/raw/oio/write/multipart_write.rs +518 -0
  231. data/core/src/raw/oio/write/one_shot_write.rs +77 -0
  232. data/core/src/raw/oio/write/position_write.rs +284 -0
  233. data/core/src/raw/operation.rs +88 -0
  234. data/core/src/raw/ops.rs +917 -0
  235. data/core/src/raw/path.rs +451 -0
  236. data/core/src/raw/path_cache.rs +244 -0
  237. data/core/src/raw/rps.rs +249 -0
  238. data/core/src/raw/serde_util.rs +423 -0
  239. data/core/src/raw/std_io_util.rs +65 -0
  240. data/core/src/raw/tests/mod.rs +30 -0
  241. data/core/src/raw/tests/read.rs +116 -0
  242. data/core/src/raw/tests/utils.rs +80 -0
  243. data/core/src/raw/tests/write.rs +79 -0
  244. data/core/src/raw/tokio_util.rs +24 -0
  245. data/core/src/raw/version.rs +19 -0
  246. data/core/src/services/aliyun_drive/backend.rs +421 -0
  247. data/core/src/services/aliyun_drive/config.rs +72 -0
  248. data/core/src/services/aliyun_drive/core.rs +651 -0
  249. data/core/src/services/aliyun_drive/delete.rs +51 -0
  250. data/core/src/services/aliyun_drive/docs.md +61 -0
  251. data/core/src/services/aliyun_drive/error.rs +56 -0
  252. data/core/src/services/aliyun_drive/lister.rs +134 -0
  253. data/core/src/services/aliyun_drive/mod.rs +39 -0
  254. data/core/src/services/aliyun_drive/writer.rs +114 -0
  255. data/core/src/services/alluxio/backend.rs +257 -0
  256. data/core/src/services/alluxio/config.rs +50 -0
  257. data/core/src/services/alluxio/core.rs +367 -0
  258. data/core/src/services/alluxio/delete.rs +38 -0
  259. data/core/src/services/alluxio/docs.md +45 -0
  260. data/core/src/services/alluxio/error.rs +99 -0
  261. data/core/src/services/alluxio/lister.rs +73 -0
  262. data/core/src/services/alluxio/mod.rs +39 -0
  263. data/core/src/services/alluxio/writer.rs +74 -0
  264. data/core/src/services/azblob/backend.rs +594 -0
  265. data/core/src/services/azblob/config.rs +220 -0
  266. data/core/src/services/azblob/core.rs +937 -0
  267. data/core/src/services/azblob/delete.rs +108 -0
  268. data/core/src/services/azblob/docs.md +77 -0
  269. data/core/src/services/azblob/error.rs +164 -0
  270. data/core/src/services/azblob/lister.rs +107 -0
  271. data/core/src/services/azblob/mod.rs +38 -0
  272. data/core/src/services/azblob/writer.rs +177 -0
  273. data/core/src/services/azdls/backend.rs +435 -0
  274. data/core/src/services/azdls/config.rs +89 -0
  275. data/core/src/services/azdls/core.rs +388 -0
  276. data/core/src/services/azdls/delete.rs +48 -0
  277. data/core/src/services/azdls/docs.md +73 -0
  278. data/core/src/services/azdls/error.rs +107 -0
  279. data/core/src/services/azdls/lister.rs +165 -0
  280. data/core/src/services/azdls/mod.rs +38 -0
  281. data/core/src/services/azdls/writer.rs +129 -0
  282. data/core/src/services/azfile/backend.rs +373 -0
  283. data/core/src/services/azfile/config.rs +61 -0
  284. data/core/src/services/azfile/core.rs +435 -0
  285. data/core/src/services/azfile/delete.rs +51 -0
  286. data/core/src/services/azfile/docs.md +65 -0
  287. data/core/src/services/azfile/error.rs +108 -0
  288. data/core/src/services/azfile/lister.rs +217 -0
  289. data/core/src/services/azfile/mod.rs +39 -0
  290. data/core/src/services/azfile/writer.rs +92 -0
  291. data/core/src/services/b2/backend.rs +434 -0
  292. data/core/src/services/b2/config.rs +64 -0
  293. data/core/src/services/b2/core.rs +742 -0
  294. data/core/src/services/b2/delete.rs +56 -0
  295. data/core/src/services/b2/docs.md +54 -0
  296. data/core/src/services/b2/error.rs +132 -0
  297. data/core/src/services/b2/lister.rs +110 -0
  298. data/core/src/services/b2/mod.rs +39 -0
  299. data/core/src/services/b2/writer.rs +189 -0
  300. data/core/src/services/cacache/backend.rs +160 -0
  301. data/core/src/services/cacache/config.rs +28 -0
  302. data/core/src/services/cacache/core.rs +96 -0
  303. data/core/src/services/cacache/delete.rs +39 -0
  304. data/core/src/services/cacache/docs.md +38 -0
  305. data/core/src/services/cacache/mod.rs +34 -0
  306. data/core/src/services/cacache/writer.rs +61 -0
  307. data/core/src/services/cloudflare_kv/backend.rs +513 -0
  308. data/core/src/services/cloudflare_kv/config.rs +55 -0
  309. data/core/src/services/cloudflare_kv/core.rs +168 -0
  310. data/core/src/services/cloudflare_kv/delete.rs +119 -0
  311. data/core/src/services/cloudflare_kv/docs.md +21 -0
  312. data/core/src/services/cloudflare_kv/error.rs +79 -0
  313. data/core/src/services/cloudflare_kv/lister.rs +170 -0
  314. data/core/src/services/cloudflare_kv/mod.rs +39 -0
  315. data/core/src/services/cloudflare_kv/model.rs +76 -0
  316. data/core/src/services/cloudflare_kv/writer.rs +68 -0
  317. data/core/src/services/compfs/backend.rs +290 -0
  318. data/core/src/services/compfs/config.rs +30 -0
  319. data/core/src/services/compfs/core.rs +159 -0
  320. data/core/src/services/compfs/delete.rs +53 -0
  321. data/core/src/services/compfs/lister.rs +98 -0
  322. data/core/src/services/compfs/mod.rs +38 -0
  323. data/core/src/services/compfs/reader.rs +79 -0
  324. data/core/src/services/compfs/writer.rs +90 -0
  325. data/core/src/services/cos/backend.rs +442 -0
  326. data/core/src/services/cos/config.rs +54 -0
  327. data/core/src/services/cos/core.rs +761 -0
  328. data/core/src/services/cos/delete.rs +48 -0
  329. data/core/src/services/cos/docs.md +55 -0
  330. data/core/src/services/cos/error.rs +105 -0
  331. data/core/src/services/cos/lister.rs +237 -0
  332. data/core/src/services/cos/mod.rs +39 -0
  333. data/core/src/services/cos/writer.rs +234 -0
  334. data/core/src/services/d1/backend.rs +330 -0
  335. data/core/src/services/d1/config.rs +55 -0
  336. data/core/src/services/d1/docs.md +48 -0
  337. data/core/src/services/d1/error.rs +79 -0
  338. data/core/src/services/d1/mod.rs +29 -0
  339. data/core/src/services/d1/model.rs +125 -0
  340. data/core/src/services/dashmap/backend.rs +203 -0
  341. data/core/src/services/dashmap/config.rs +37 -0
  342. data/core/src/services/dashmap/core.rs +61 -0
  343. data/core/src/services/dashmap/delete.rs +40 -0
  344. data/core/src/services/dashmap/docs.md +38 -0
  345. data/core/src/services/dashmap/lister.rs +63 -0
  346. data/core/src/services/dashmap/mod.rs +36 -0
  347. data/core/src/services/dashmap/writer.rs +87 -0
  348. data/core/src/services/dbfs/backend.rs +258 -0
  349. data/core/src/services/dbfs/config.rs +48 -0
  350. data/core/src/services/dbfs/core.rs +191 -0
  351. data/core/src/services/dbfs/delete.rs +49 -0
  352. data/core/src/services/dbfs/docs.md +57 -0
  353. data/core/src/services/dbfs/error.rs +74 -0
  354. data/core/src/services/dbfs/lister.rs +96 -0
  355. data/core/src/services/dbfs/mod.rs +38 -0
  356. data/core/src/services/dbfs/writer.rs +64 -0
  357. data/core/src/services/dropbox/backend.rs +187 -0
  358. data/core/src/services/dropbox/builder.rs +222 -0
  359. data/core/src/services/dropbox/config.rs +47 -0
  360. data/core/src/services/dropbox/core.rs +496 -0
  361. data/core/src/services/dropbox/delete.rs +54 -0
  362. data/core/src/services/dropbox/docs.md +64 -0
  363. data/core/src/services/dropbox/error.rs +85 -0
  364. data/core/src/services/dropbox/lister.rs +117 -0
  365. data/core/src/services/dropbox/mod.rs +40 -0
  366. data/core/src/services/dropbox/writer.rs +51 -0
  367. data/core/src/services/etcd/backend.rs +345 -0
  368. data/core/src/services/etcd/config.rs +86 -0
  369. data/core/src/services/etcd/core.rs +143 -0
  370. data/core/src/services/etcd/deleter.rs +41 -0
  371. data/core/src/services/etcd/docs.md +45 -0
  372. data/core/src/services/etcd/error.rs +26 -0
  373. data/core/src/services/etcd/lister.rs +79 -0
  374. data/core/src/services/etcd/mod.rs +36 -0
  375. data/core/src/services/etcd/writer.rs +61 -0
  376. data/core/src/services/foundationdb/backend.rs +171 -0
  377. data/core/src/services/foundationdb/config.rs +45 -0
  378. data/core/src/services/foundationdb/docs.md +42 -0
  379. data/core/src/services/foundationdb/mod.rs +24 -0
  380. data/core/src/services/fs/backend.rs +299 -0
  381. data/core/src/services/fs/config.rs +33 -0
  382. data/core/src/services/fs/core.rs +227 -0
  383. data/core/src/services/fs/delete.rs +53 -0
  384. data/core/src/services/fs/docs.md +49 -0
  385. data/core/src/services/fs/error.rs +31 -0
  386. data/core/src/services/fs/lister.rs +81 -0
  387. data/core/src/services/fs/mod.rs +40 -0
  388. data/core/src/services/fs/reader.rs +83 -0
  389. data/core/src/services/fs/writer.rs +212 -0
  390. data/core/src/services/ftp/backend.rs +388 -0
  391. data/core/src/services/ftp/config.rs +46 -0
  392. data/core/src/services/ftp/core.rs +136 -0
  393. data/core/src/services/ftp/delete.rs +62 -0
  394. data/core/src/services/ftp/docs.md +42 -0
  395. data/core/src/services/ftp/err.rs +47 -0
  396. data/core/src/services/ftp/lister.rs +72 -0
  397. data/core/src/services/ftp/mod.rs +41 -0
  398. data/core/src/services/ftp/reader.rs +84 -0
  399. data/core/src/services/ftp/writer.rs +122 -0
  400. data/core/src/services/gcs/backend.rs +499 -0
  401. data/core/src/services/gcs/config.rs +168 -0
  402. data/core/src/services/gcs/core.rs +1079 -0
  403. data/core/src/services/gcs/delete.rs +98 -0
  404. data/core/src/services/gcs/docs.md +76 -0
  405. data/core/src/services/gcs/error.rs +122 -0
  406. data/core/src/services/gcs/lister.rs +136 -0
  407. data/core/src/services/gcs/mod.rs +40 -0
  408. data/core/src/services/gcs/uri.rs +75 -0
  409. data/core/src/services/gcs/writer.rs +163 -0
  410. data/core/src/services/gdrive/backend.rs +176 -0
  411. data/core/src/services/gdrive/builder.rs +228 -0
  412. data/core/src/services/gdrive/config.rs +47 -0
  413. data/core/src/services/gdrive/core.rs +499 -0
  414. data/core/src/services/gdrive/delete.rs +57 -0
  415. data/core/src/services/gdrive/docs.md +65 -0
  416. data/core/src/services/gdrive/error.rs +80 -0
  417. data/core/src/services/gdrive/lister.rs +110 -0
  418. data/core/src/services/gdrive/mod.rs +40 -0
  419. data/core/src/services/gdrive/writer.rs +77 -0
  420. data/core/src/services/ghac/backend.rs +285 -0
  421. data/core/src/services/ghac/config.rs +36 -0
  422. data/core/src/services/ghac/core.rs +459 -0
  423. data/core/src/services/ghac/docs.md +84 -0
  424. data/core/src/services/ghac/error.rs +52 -0
  425. data/core/src/services/ghac/mod.rs +35 -0
  426. data/core/src/services/ghac/writer.rs +201 -0
  427. data/core/src/services/github/backend.rs +285 -0
  428. data/core/src/services/github/config.rs +59 -0
  429. data/core/src/services/github/core.rs +351 -0
  430. data/core/src/services/github/delete.rs +41 -0
  431. data/core/src/services/github/docs.md +52 -0
  432. data/core/src/services/github/error.rs +101 -0
  433. data/core/src/services/github/lister.rs +112 -0
  434. data/core/src/services/github/mod.rs +38 -0
  435. data/core/src/services/github/writer.rs +51 -0
  436. data/core/src/services/gridfs/backend.rs +166 -0
  437. data/core/src/services/gridfs/config.rs +50 -0
  438. data/core/src/services/gridfs/core.rs +154 -0
  439. data/core/src/services/gridfs/docs.md +46 -0
  440. data/core/src/services/gridfs/mod.rs +26 -0
  441. data/core/src/services/hdfs/backend.rs +413 -0
  442. data/core/src/services/hdfs/config.rs +59 -0
  443. data/core/src/services/hdfs/delete.rs +62 -0
  444. data/core/src/services/hdfs/docs.md +140 -0
  445. data/core/src/services/hdfs/lister.rs +70 -0
  446. data/core/src/services/hdfs/mod.rs +36 -0
  447. data/core/src/services/hdfs/reader.rs +79 -0
  448. data/core/src/services/hdfs/writer.rs +104 -0
  449. data/core/src/services/hdfs_native/backend.rs +340 -0
  450. data/core/src/services/hdfs_native/config.rs +45 -0
  451. data/core/src/services/hdfs_native/delete.rs +47 -0
  452. data/core/src/services/hdfs_native/docs.md +35 -0
  453. data/core/src/services/hdfs_native/error.rs +59 -0
  454. data/core/src/services/hdfs_native/lister.rs +85 -0
  455. data/core/src/services/hdfs_native/mod.rs +39 -0
  456. data/core/src/services/hdfs_native/reader.rs +62 -0
  457. data/core/src/services/hdfs_native/writer.rs +61 -0
  458. data/core/src/services/http/backend.rs +291 -0
  459. data/core/src/services/http/config.rs +49 -0
  460. data/core/src/services/http/core.rs +125 -0
  461. data/core/src/services/http/docs.md +45 -0
  462. data/core/src/services/http/error.rs +53 -0
  463. data/core/src/services/http/mod.rs +32 -0
  464. data/core/src/services/huggingface/backend.rs +289 -0
  465. data/core/src/services/huggingface/config.rs +75 -0
  466. data/core/src/services/huggingface/core.rs +406 -0
  467. data/core/src/services/huggingface/docs.md +61 -0
  468. data/core/src/services/huggingface/error.rs +93 -0
  469. data/core/src/services/huggingface/lister.rs +91 -0
  470. data/core/src/services/huggingface/mod.rs +34 -0
  471. data/core/src/services/ipfs/backend.rs +257 -0
  472. data/core/src/services/ipfs/config.rs +32 -0
  473. data/core/src/services/ipfs/core.rs +239 -0
  474. data/core/src/services/ipfs/docs.md +45 -0
  475. data/core/src/services/ipfs/error.rs +52 -0
  476. data/core/src/services/ipfs/ipld.rs +162 -0
  477. data/core/src/services/ipfs/mod.rs +34 -0
  478. data/core/src/services/ipmfs/backend.rs +147 -0
  479. data/core/src/services/ipmfs/builder.rs +166 -0
  480. data/core/src/services/ipmfs/config.rs +32 -0
  481. data/core/src/services/ipmfs/core.rs +142 -0
  482. data/core/src/services/ipmfs/delete.rs +48 -0
  483. data/core/src/services/ipmfs/docs.md +14 -0
  484. data/core/src/services/ipmfs/error.rs +83 -0
  485. data/core/src/services/ipmfs/lister.rs +135 -0
  486. data/core/src/services/ipmfs/mod.rs +40 -0
  487. data/core/src/services/ipmfs/writer.rs +49 -0
  488. data/core/src/services/koofr/backend.rs +361 -0
  489. data/core/src/services/koofr/config.rs +50 -0
  490. data/core/src/services/koofr/core.rs +458 -0
  491. data/core/src/services/koofr/delete.rs +50 -0
  492. data/core/src/services/koofr/docs.md +51 -0
  493. data/core/src/services/koofr/error.rs +72 -0
  494. data/core/src/services/koofr/lister.rs +88 -0
  495. data/core/src/services/koofr/mod.rs +38 -0
  496. data/core/src/services/koofr/writer.rs +53 -0
  497. data/core/src/services/lakefs/backend.rs +309 -0
  498. data/core/src/services/lakefs/config.rs +81 -0
  499. data/core/src/services/lakefs/core.rs +261 -0
  500. data/core/src/services/lakefs/delete.rs +54 -0
  501. data/core/src/services/lakefs/docs.md +62 -0
  502. data/core/src/services/lakefs/error.rs +93 -0
  503. data/core/src/services/lakefs/lister.rs +120 -0
  504. data/core/src/services/lakefs/mod.rs +38 -0
  505. data/core/src/services/lakefs/writer.rs +50 -0
  506. data/core/src/services/memcached/backend.rs +284 -0
  507. data/core/src/services/memcached/binary.rs +289 -0
  508. data/core/src/services/memcached/config.rs +43 -0
  509. data/core/src/services/memcached/docs.md +47 -0
  510. data/core/src/services/memcached/mod.rs +27 -0
  511. data/core/src/services/memory/backend.rs +205 -0
  512. data/core/src/services/memory/config.rs +30 -0
  513. data/core/src/services/memory/core.rs +80 -0
  514. data/core/src/services/memory/delete.rs +42 -0
  515. data/core/src/services/memory/docs.md +36 -0
  516. data/core/src/services/memory/lister.rs +56 -0
  517. data/core/src/services/memory/mod.rs +36 -0
  518. data/core/src/services/memory/writer.rs +85 -0
  519. data/core/src/services/mini_moka/backend.rs +260 -0
  520. data/core/src/services/mini_moka/config.rs +56 -0
  521. data/core/src/services/mini_moka/core.rs +52 -0
  522. data/core/src/services/mini_moka/delete.rs +42 -0
  523. data/core/src/services/mini_moka/docs.md +19 -0
  524. data/core/src/services/mini_moka/lister.rs +68 -0
  525. data/core/src/services/mini_moka/mod.rs +36 -0
  526. data/core/src/services/mini_moka/writer.rs +84 -0
  527. data/core/src/services/mod.rs +206 -0
  528. data/core/src/services/moka/backend.rs +326 -0
  529. data/core/src/services/moka/config.rs +59 -0
  530. data/core/src/services/moka/core.rs +62 -0
  531. data/core/src/services/moka/delete.rs +42 -0
  532. data/core/src/services/moka/docs.md +42 -0
  533. data/core/src/services/moka/lister.rs +65 -0
  534. data/core/src/services/moka/mod.rs +41 -0
  535. data/core/src/services/moka/writer.rs +83 -0
  536. data/core/src/services/mongodb/backend.rs +291 -0
  537. data/core/src/services/mongodb/config.rs +54 -0
  538. data/core/src/services/mongodb/docs.md +49 -0
  539. data/core/src/services/mongodb/mod.rs +24 -0
  540. data/core/src/services/monoiofs/backend.rs +238 -0
  541. data/core/src/services/monoiofs/config.rs +34 -0
  542. data/core/src/services/monoiofs/core.rs +313 -0
  543. data/core/src/services/monoiofs/delete.rs +64 -0
  544. data/core/src/services/monoiofs/docs.md +46 -0
  545. data/core/src/services/monoiofs/mod.rs +36 -0
  546. data/core/src/services/monoiofs/reader.rs +147 -0
  547. data/core/src/services/monoiofs/writer.rs +189 -0
  548. data/core/src/services/mysql/backend.rs +256 -0
  549. data/core/src/services/mysql/config.rs +66 -0
  550. data/core/src/services/mysql/docs.md +47 -0
  551. data/core/src/services/mysql/mod.rs +24 -0
  552. data/core/src/services/obs/backend.rs +442 -0
  553. data/core/src/services/obs/config.rs +53 -0
  554. data/core/src/services/obs/core.rs +608 -0
  555. data/core/src/services/obs/delete.rs +48 -0
  556. data/core/src/services/obs/docs.md +54 -0
  557. data/core/src/services/obs/error.rs +106 -0
  558. data/core/src/services/obs/lister.rs +101 -0
  559. data/core/src/services/obs/mod.rs +38 -0
  560. data/core/src/services/obs/writer.rs +235 -0
  561. data/core/src/services/onedrive/backend.rs +127 -0
  562. data/core/src/services/onedrive/builder.rs +236 -0
  563. data/core/src/services/onedrive/config.rs +49 -0
  564. data/core/src/services/onedrive/core.rs +691 -0
  565. data/core/src/services/onedrive/delete.rs +47 -0
  566. data/core/src/services/onedrive/docs.md +115 -0
  567. data/core/src/services/onedrive/error.rs +61 -0
  568. data/core/src/services/onedrive/graph_model.rs +425 -0
  569. data/core/src/services/onedrive/lister.rs +150 -0
  570. data/core/src/services/onedrive/mod.rs +42 -0
  571. data/core/src/services/onedrive/writer.rs +168 -0
  572. data/core/src/services/opfs/backend.rs +50 -0
  573. data/core/src/services/opfs/config.rs +25 -0
  574. data/core/src/services/opfs/core.rs +74 -0
  575. data/core/src/services/opfs/docs.md +18 -0
  576. data/core/src/services/opfs/error.rs +27 -0
  577. data/core/src/services/opfs/mod.rs +30 -0
  578. data/core/src/services/opfs/utils.rs +70 -0
  579. data/core/src/services/oss/backend.rs +734 -0
  580. data/core/src/services/oss/config.rs +113 -0
  581. data/core/src/services/oss/core.rs +1088 -0
  582. data/core/src/services/oss/delete.rs +109 -0
  583. data/core/src/services/oss/docs.md +74 -0
  584. data/core/src/services/oss/error.rs +109 -0
  585. data/core/src/services/oss/lister.rs +256 -0
  586. data/core/src/services/oss/mod.rs +38 -0
  587. data/core/src/services/oss/writer.rs +228 -0
  588. data/core/src/services/pcloud/backend.rs +358 -0
  589. data/core/src/services/pcloud/config.rs +51 -0
  590. data/core/src/services/pcloud/core.rs +461 -0
  591. data/core/src/services/pcloud/delete.rs +66 -0
  592. data/core/src/services/pcloud/docs.md +51 -0
  593. data/core/src/services/pcloud/error.rs +88 -0
  594. data/core/src/services/pcloud/lister.rs +95 -0
  595. data/core/src/services/pcloud/mod.rs +38 -0
  596. data/core/src/services/pcloud/writer.rs +66 -0
  597. data/core/src/services/persy/backend.rs +226 -0
  598. data/core/src/services/persy/config.rs +32 -0
  599. data/core/src/services/persy/docs.md +43 -0
  600. data/core/src/services/persy/mod.rs +24 -0
  601. data/core/src/services/postgresql/backend.rs +258 -0
  602. data/core/src/services/postgresql/config.rs +66 -0
  603. data/core/src/services/postgresql/docs.md +47 -0
  604. data/core/src/services/postgresql/mod.rs +24 -0
  605. data/core/src/services/redb/backend.rs +280 -0
  606. data/core/src/services/redb/config.rs +34 -0
  607. data/core/src/services/redb/docs.md +41 -0
  608. data/core/src/services/redb/mod.rs +24 -0
  609. data/core/src/services/redis/backend.rs +442 -0
  610. data/core/src/services/redis/config.rs +79 -0
  611. data/core/src/services/redis/core.rs +209 -0
  612. data/core/src/services/redis/delete.rs +40 -0
  613. data/core/src/services/redis/docs.md +43 -0
  614. data/core/src/services/redis/mod.rs +34 -0
  615. data/core/src/services/redis/writer.rs +57 -0
  616. data/core/src/services/rocksdb/backend.rs +159 -0
  617. data/core/src/services/rocksdb/config.rs +34 -0
  618. data/core/src/services/rocksdb/docs.md +54 -0
  619. data/core/src/services/rocksdb/mod.rs +24 -0
  620. data/core/src/services/s3/backend.rs +1293 -0
  621. data/core/src/services/s3/compatible_services.md +126 -0
  622. data/core/src/services/s3/config.rs +327 -0
  623. data/core/src/services/s3/core.rs +1741 -0
  624. data/core/src/services/s3/delete.rs +109 -0
  625. data/core/src/services/s3/docs.md +244 -0
  626. data/core/src/services/s3/error.rs +171 -0
  627. data/core/src/services/s3/lister.rs +405 -0
  628. data/core/src/services/s3/mod.rs +38 -0
  629. data/core/src/services/s3/writer.rs +262 -0
  630. data/core/src/services/seafile/backend.rs +297 -0
  631. data/core/src/services/seafile/config.rs +56 -0
  632. data/core/src/services/seafile/core.rs +475 -0
  633. data/core/src/services/seafile/delete.rs +40 -0
  634. data/core/src/services/seafile/docs.md +54 -0
  635. data/core/src/services/seafile/error.rs +86 -0
  636. data/core/src/services/seafile/lister.rs +83 -0
  637. data/core/src/services/seafile/mod.rs +38 -0
  638. data/core/src/services/seafile/writer.rs +55 -0
  639. data/core/src/services/sftp/backend.rs +397 -0
  640. data/core/src/services/sftp/config.rs +50 -0
  641. data/core/src/services/sftp/core.rs +154 -0
  642. data/core/src/services/sftp/delete.rs +55 -0
  643. data/core/src/services/sftp/docs.md +49 -0
  644. data/core/src/services/sftp/error.rs +57 -0
  645. data/core/src/services/sftp/lister.rs +88 -0
  646. data/core/src/services/sftp/mod.rs +42 -0
  647. data/core/src/services/sftp/reader.rs +78 -0
  648. data/core/src/services/sftp/utils.rs +51 -0
  649. data/core/src/services/sftp/writer.rs +67 -0
  650. data/core/src/services/sled/backend.rs +194 -0
  651. data/core/src/services/sled/config.rs +45 -0
  652. data/core/src/services/sled/docs.md +39 -0
  653. data/core/src/services/sled/mod.rs +24 -0
  654. data/core/src/services/sqlite/backend.rs +326 -0
  655. data/core/src/services/sqlite/config.rs +70 -0
  656. data/core/src/services/sqlite/docs.md +46 -0
  657. data/core/src/services/sqlite/mod.rs +24 -0
  658. data/core/src/services/surrealdb/backend.rs +365 -0
  659. data/core/src/services/surrealdb/config.rs +64 -0
  660. data/core/src/services/surrealdb/docs.md +54 -0
  661. data/core/src/services/surrealdb/mod.rs +24 -0
  662. data/core/src/services/swift/backend.rs +275 -0
  663. data/core/src/services/swift/compatible_services.md +53 -0
  664. data/core/src/services/swift/config.rs +53 -0
  665. data/core/src/services/swift/core.rs +310 -0
  666. data/core/src/services/swift/delete.rs +49 -0
  667. data/core/src/services/swift/docs.md +52 -0
  668. data/core/src/services/swift/error.rs +90 -0
  669. data/core/src/services/swift/lister.rs +119 -0
  670. data/core/src/services/swift/mod.rs +38 -0
  671. data/core/src/services/swift/writer.rs +53 -0
  672. data/core/src/services/tikv/backend.rs +237 -0
  673. data/core/src/services/tikv/config.rs +52 -0
  674. data/core/src/services/tikv/docs.md +43 -0
  675. data/core/src/services/tikv/mod.rs +24 -0
  676. data/core/src/services/upyun/backend.rs +317 -0
  677. data/core/src/services/upyun/config.rs +51 -0
  678. data/core/src/services/upyun/core.rs +521 -0
  679. data/core/src/services/upyun/delete.rs +50 -0
  680. data/core/src/services/upyun/docs.md +51 -0
  681. data/core/src/services/upyun/error.rs +97 -0
  682. data/core/src/services/upyun/lister.rs +101 -0
  683. data/core/src/services/upyun/mod.rs +38 -0
  684. data/core/src/services/upyun/writer.rs +127 -0
  685. data/core/src/services/vercel_artifacts/backend.rs +99 -0
  686. data/core/src/services/vercel_artifacts/builder.rs +117 -0
  687. data/core/src/services/vercel_artifacts/config.rs +39 -0
  688. data/core/src/services/vercel_artifacts/core.rs +112 -0
  689. data/core/src/services/vercel_artifacts/docs.md +40 -0
  690. data/core/src/services/vercel_artifacts/error.rs +50 -0
  691. data/core/src/services/vercel_artifacts/mod.rs +36 -0
  692. data/core/src/services/vercel_artifacts/writer.rs +58 -0
  693. data/core/src/services/vercel_blob/backend.rs +251 -0
  694. data/core/src/services/vercel_blob/config.rs +45 -0
  695. data/core/src/services/vercel_blob/core.rs +449 -0
  696. data/core/src/services/vercel_blob/delete.rs +38 -0
  697. data/core/src/services/vercel_blob/docs.md +45 -0
  698. data/core/src/services/vercel_blob/error.rs +110 -0
  699. data/core/src/services/vercel_blob/lister.rs +69 -0
  700. data/core/src/services/vercel_blob/mod.rs +38 -0
  701. data/core/src/services/vercel_blob/writer.rs +143 -0
  702. data/core/src/services/webdav/backend.rs +318 -0
  703. data/core/src/services/webdav/config.rs +53 -0
  704. data/core/src/services/webdav/core.rs +859 -0
  705. data/core/src/services/webdav/delete.rs +47 -0
  706. data/core/src/services/webdav/docs.md +49 -0
  707. data/core/src/services/webdav/error.rs +53 -0
  708. data/core/src/services/webdav/lister.rs +106 -0
  709. data/core/src/services/webdav/mod.rs +38 -0
  710. data/core/src/services/webdav/writer.rs +56 -0
  711. data/core/src/services/webhdfs/backend.rs +376 -0
  712. data/core/src/services/webhdfs/config.rs +52 -0
  713. data/core/src/services/webhdfs/core.rs +398 -0
  714. data/core/src/services/webhdfs/delete.rs +46 -0
  715. data/core/src/services/webhdfs/docs.md +90 -0
  716. data/core/src/services/webhdfs/error.rs +126 -0
  717. data/core/src/services/webhdfs/lister.rs +130 -0
  718. data/core/src/services/webhdfs/message.rs +249 -0
  719. data/core/src/services/webhdfs/mod.rs +41 -0
  720. data/core/src/services/webhdfs/writer.rs +177 -0
  721. data/core/src/services/yandex_disk/backend.rs +267 -0
  722. data/core/src/services/yandex_disk/config.rs +45 -0
  723. data/core/src/services/yandex_disk/core.rs +340 -0
  724. data/core/src/services/yandex_disk/delete.rs +54 -0
  725. data/core/src/services/yandex_disk/docs.md +45 -0
  726. data/core/src/services/yandex_disk/error.rs +104 -0
  727. data/core/src/services/yandex_disk/lister.rs +113 -0
  728. data/core/src/services/yandex_disk/mod.rs +38 -0
  729. data/core/src/services/yandex_disk/writer.rs +52 -0
  730. data/core/src/types/buffer.rs +991 -0
  731. data/core/src/types/builder.rs +152 -0
  732. data/core/src/types/capability.rs +209 -0
  733. data/core/src/types/context/mod.rs +22 -0
  734. data/core/src/types/context/read.rs +231 -0
  735. data/core/src/types/context/write.rs +441 -0
  736. data/core/src/types/delete/deleter.rs +220 -0
  737. data/core/src/types/delete/futures_delete_sink.rs +176 -0
  738. data/core/src/types/delete/input.rs +97 -0
  739. data/core/src/types/delete/mod.rs +26 -0
  740. data/core/src/types/entry.rs +69 -0
  741. data/core/src/types/error.rs +570 -0
  742. data/core/src/types/execute/api.rs +110 -0
  743. data/core/src/types/execute/executor.rs +96 -0
  744. data/core/src/types/execute/executors/mod.rs +27 -0
  745. data/core/src/types/execute/executors/tokio_executor.rs +60 -0
  746. data/core/src/types/execute/mod.rs +25 -0
  747. data/core/src/types/list.rs +137 -0
  748. data/core/src/types/metadata.rs +436 -0
  749. data/core/src/types/mod.rs +72 -0
  750. data/core/src/types/mode.rs +68 -0
  751. data/core/src/types/operator/builder.rs +535 -0
  752. data/core/src/types/operator/info.rs +63 -0
  753. data/core/src/types/operator/mod.rs +33 -0
  754. data/core/src/types/operator/operator.rs +2236 -0
  755. data/core/src/types/operator/operator_futures.rs +1430 -0
  756. data/core/src/types/operator/registry.rs +129 -0
  757. data/core/src/types/options.rs +548 -0
  758. data/core/src/types/read/buffer_stream.rs +273 -0
  759. data/core/src/types/read/futures_async_reader.rs +289 -0
  760. data/core/src/types/read/futures_bytes_stream.rs +157 -0
  761. data/core/src/types/read/mod.rs +29 -0
  762. data/core/src/types/read/reader.rs +604 -0
  763. data/core/src/types/scheme.rs +475 -0
  764. data/core/src/types/write/buffer_sink.rs +188 -0
  765. data/core/src/types/write/futures_async_writer.rs +136 -0
  766. data/core/src/types/write/futures_bytes_sink.rs +103 -0
  767. data/core/src/types/write/mod.rs +26 -0
  768. data/core/src/types/write/writer.rs +411 -0
  769. data/core/tests/behavior/README.md +77 -0
  770. data/core/tests/behavior/async_copy.rs +314 -0
  771. data/core/tests/behavior/async_create_dir.rs +53 -0
  772. data/core/tests/behavior/async_delete.rs +354 -0
  773. data/core/tests/behavior/async_list.rs +739 -0
  774. data/core/tests/behavior/async_presign.rs +175 -0
  775. data/core/tests/behavior/async_read.rs +871 -0
  776. data/core/tests/behavior/async_rename.rs +210 -0
  777. data/core/tests/behavior/async_stat.rs +628 -0
  778. data/core/tests/behavior/async_write.rs +819 -0
  779. data/core/tests/behavior/main.rs +78 -0
  780. data/core/tests/behavior/utils.rs +187 -0
  781. data/core/tests/data/normal_dir/.gitkeep +0 -0
  782. data/core/tests/data/normal_file.txt +1041 -0
  783. data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
  784. data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
  785. data/core/users.md +13 -0
  786. data/extconf.rb +24 -0
  787. data/lib/opendal.rb +25 -0
  788. data/lib/opendal_ruby/entry.rb +35 -0
  789. data/lib/opendal_ruby/io.rb +70 -0
  790. data/lib/opendal_ruby/metadata.rb +44 -0
  791. data/lib/opendal_ruby/operator.rb +29 -0
  792. data/lib/opendal_ruby/operator_info.rb +26 -0
  793. data/src/capability.rs +146 -0
  794. data/src/io.rs +464 -0
  795. data/src/lib.rs +63 -0
  796. data/src/lister.rs +141 -0
  797. data/src/metadata.rs +111 -0
  798. data/src/middlewares.rs +174 -0
  799. data/src/operator.rs +310 -0
  800. data/src/operator_info.rs +83 -0
  801. data/test/blocking_op_test.rb +112 -0
  802. data/test/capability_test.rb +42 -0
  803. data/test/io_test.rb +172 -0
  804. data/test/lister_test.rb +77 -0
  805. data/test/metadata_test.rb +78 -0
  806. data/test/middlewares_test.rb +46 -0
  807. data/test/operator_info_test.rb +35 -0
  808. data/test/test_helper.rb +36 -0
  809. metadata +857 -0
@@ -0,0 +1,2236 @@
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ use std::future::Future;
19
+ use std::time::Duration;
20
+
21
+ use futures::Stream;
22
+ use futures::StreamExt;
23
+ use futures::TryStreamExt;
24
+
25
+ use crate::operator_futures::*;
26
+ use crate::raw::oio::DeleteDyn;
27
+ use crate::raw::*;
28
+ use crate::types::delete::Deleter;
29
+ use crate::*;
30
+
31
+ /// The `Operator` serves as the entry point for all public asynchronous APIs.
32
+ ///
33
+ /// For more details about the `Operator`, refer to the [`concepts`][crate::docs::concepts] section.
34
+ ///
35
+ /// All cloned `Operator` instances share the same internal state, such as
36
+ /// `HttpClient` and `Runtime`. Some layers may modify the internal state of
37
+ /// the `Operator` too like inject logging and metrics for `HttpClient`.
38
+ ///
39
+ /// ## Build
40
+ ///
41
+ /// Users can initialize an `Operator` through the following methods:
42
+ ///
43
+ /// - [`Operator::new`]: Creates an operator using a [`services`] builder, such as [`services::S3`].
44
+ /// - [`Operator::from_config`]: Creates an operator using a [`services`] configuration, such as [`services::S3Config`].
45
+ /// - [`Operator::from_iter`]: Creates an operator from an iterator of configuration key-value pairs.
46
+ ///
47
+ /// ```
48
+ /// # use anyhow::Result;
49
+ /// use opendal::services::Memory;
50
+ /// use opendal::Operator;
51
+ /// async fn test() -> Result<()> {
52
+ /// // Build an `Operator` to start operating the storage.
53
+ /// let _: Operator = Operator::new(Memory::default())?.finish();
54
+ ///
55
+ /// Ok(())
56
+ /// }
57
+ /// ```
58
+ ///
59
+ /// ## Layer
60
+ ///
61
+ /// After the operator is built, users can add the layers they need on top of it.
62
+ ///
63
+ /// OpenDAL offers various layers for users to choose from, such as `RetryLayer`, `LoggingLayer`, and more. Visit [`layers`] for further details.
64
+ ///
65
+ /// Please note that `Layer` can modify internal contexts such as `HttpClient`
66
+ /// and `Runtime` for all clones of given operator. Therefore, it is recommended
67
+ /// to add layers before interacting with the storage. Adding or duplicating
68
+ /// layers after accessing the storage may result in unexpected behavior.
69
+ ///
70
+ /// ```
71
+ /// # use anyhow::Result;
72
+ /// use opendal::layers::RetryLayer;
73
+ /// use opendal::services::Memory;
74
+ /// use opendal::Operator;
75
+ /// async fn test() -> Result<()> {
76
+ /// let op: Operator = Operator::new(Memory::default())?.finish();
77
+ ///
78
+ /// // OpenDAL will retry failed operations now.
79
+ /// let op = op.layer(RetryLayer::default());
80
+ ///
81
+ /// Ok(())
82
+ /// }
83
+ /// ```
84
+ ///
85
+ /// ## Operate
86
+ ///
87
+ /// After the operator is built and the layers are added, users can start operating the storage.
88
+ ///
89
+ /// The operator is `Send`, `Sync`, and `Clone`. It has no internal state, and all APIs only take
90
+ /// a `&self` reference, making it safe to share the operator across threads.
91
+ ///
92
+ /// Operator provides a consistent API pattern for data operations. For reading operations, it exposes:
93
+ ///
94
+ /// - [`Operator::read`]: Executes a read operation.
95
+ /// - [`Operator::read_with`]: Executes a read operation with additional options using the builder pattern.
96
+ /// - [`Operator::read_options`]: Executes a read operation with extra options provided via a [`options::ReadOptions`] struct.
97
+ /// - [`Operator::reader`]: Creates a reader for streaming data, allowing for flexible access.
98
+ /// - [`Operator::reader_with`]: Creates a reader with advanced options using the builder pattern.
99
+ /// - [`Operator::reader_options`]: Creates a reader with extra options provided via a [`options::ReadOptions`] struct.
100
+ ///
101
+ /// The [`Reader`] created by [`Operator`] supports custom read control methods and can be converted
102
+ /// into [`futures::AsyncRead`] or [`futures::Stream`] for broader ecosystem compatibility.
103
+ ///
104
+ /// ```no_run
105
+ /// use opendal::layers::LoggingLayer;
106
+ /// use opendal::options;
107
+ /// use opendal::services;
108
+ /// use opendal::Operator;
109
+ /// use opendal::Result;
110
+ ///
111
+ /// #[tokio::main]
112
+ /// async fn main() -> Result<()> {
113
+ /// // Pick a builder and configure it.
114
+ /// let mut builder = services::S3::default().bucket("test");
115
+ ///
116
+ /// // Init an operator
117
+ /// let op = Operator::new(builder)?
118
+ /// // Init with logging layer enabled.
119
+ /// .layer(LoggingLayer::default())
120
+ /// .finish();
121
+ ///
122
+ /// // Fetch this file's metadata
123
+ /// let meta = op.stat("hello.txt").await?;
124
+ /// let length = meta.content_length();
125
+ ///
126
+ /// // Read data from `hello.txt` with options.
127
+ /// let bs = op
128
+ /// .read_with("hello.txt")
129
+ /// .range(0..8 * 1024 * 1024)
130
+ /// .chunk(1024 * 1024)
131
+ /// .concurrent(4)
132
+ /// .await?;
133
+ ///
134
+ /// // The same to:
135
+ /// let bs = op
136
+ /// .read_options("hello.txt", options::ReadOptions {
137
+ /// range: (0..8 * 1024 * 1024).into(),
138
+ /// chunk: Some(1024 * 1024),
139
+ /// concurrent: 4,
140
+ /// ..Default::default()
141
+ /// })
142
+ /// .await?;
143
+ ///
144
+ /// Ok(())
145
+ /// }
146
+ /// ```
147
+ #[derive(Clone, Debug)]
148
+ pub struct Operator {
149
+ // accessor is what Operator delegates for
150
+ accessor: Accessor,
151
+ }
152
+
153
+ /// # Operator basic API.
154
+ impl Operator {
155
+ /// Fetch the internal accessor.
156
+ pub fn inner(&self) -> &Accessor {
157
+ &self.accessor
158
+ }
159
+
160
+ /// Convert inner accessor into operator.
161
+ pub fn from_inner(accessor: Accessor) -> Self {
162
+ Self { accessor }
163
+ }
164
+
165
+ /// Convert operator into inner accessor.
166
+ pub fn into_inner(self) -> Accessor {
167
+ self.accessor
168
+ }
169
+
170
+ /// Get information of underlying accessor.
171
+ ///
172
+ /// # Examples
173
+ ///
174
+ /// ```
175
+ /// # use std::sync::Arc;
176
+ /// # use anyhow::Result;
177
+ /// use opendal::Operator;
178
+ ///
179
+ /// # async fn test(op: Operator) -> Result<()> {
180
+ /// let info = op.info();
181
+ /// # Ok(())
182
+ /// # }
183
+ /// ```
184
+ pub fn info(&self) -> OperatorInfo {
185
+ OperatorInfo::new(self.accessor.info())
186
+ }
187
+
188
+ /// Get the executor used by current operator.
189
+ pub fn executor(&self) -> Executor {
190
+ self.accessor.info().executor()
191
+ }
192
+
193
+ /// Update executor for the context.
194
+ ///
195
+ /// All cloned `Operator` instances share the same internal state, such as
196
+ /// `HttpClient` and `Runtime`. Some layers may modify the internal state of
197
+ /// the `Operator` too like inject logging and metrics for `HttpClient`.
198
+ ///
199
+ /// # Note
200
+ ///
201
+ /// Tasks must be forwarded to the old executor after the update. Otherwise, features such as retry, timeout, and metrics may not function properly.
202
+ pub fn update_executor(&self, f: impl FnOnce(Executor) -> Executor) {
203
+ self.accessor.info().update_executor(f);
204
+ }
205
+
206
+ /// Get the http client used by current operator.
207
+ #[deprecated(
208
+ since = "0.54.0",
209
+ note = "Use HttpClientLayer instead. This method will be removed in next version."
210
+ )]
211
+ pub fn http_client(&self) -> HttpClient {
212
+ self.accessor.info().http_client()
213
+ }
214
+
215
+ /// Update http client for the context.
216
+ ///
217
+ /// All cloned `Operator` instances share the same internal state, such as
218
+ /// `HttpClient` and `Runtime`. Some layers may modify the internal state of
219
+ /// the `Operator` too like inject logging and metrics for `HttpClient`.
220
+ ///
221
+ /// # Note
222
+ ///
223
+ /// Tasks must be forwarded to the old executor after the update. Otherwise, features such as retry, timeout, and metrics may not function properly.
224
+ ///
225
+ /// # Deprecated
226
+ ///
227
+ /// This method is deprecated since v0.54.0. Use [`HttpClientLayer`] instead.
228
+ ///
229
+ /// ## Migration Example
230
+ ///
231
+ /// Instead of:
232
+ /// ```ignore
233
+ /// let operator = Operator::new(service)?;
234
+ /// operator.update_http_client(|_| custom_client);
235
+ /// ```
236
+ ///
237
+ /// Use:
238
+ /// ```ignore
239
+ /// use opendal::layers::HttpClientLayer;
240
+ ///
241
+ /// let operator = Operator::new(service)?
242
+ /// .layer(HttpClientLayer::new(custom_client))
243
+ /// .finish();
244
+ /// ```
245
+ ///
246
+ /// [`HttpClientLayer`]: crate::layers::HttpClientLayer
247
+ #[deprecated(
248
+ since = "0.54.0",
249
+ note = "Use HttpClientLayer instead. This method will be removed in next version"
250
+ )]
251
+ pub fn update_http_client(&self, f: impl FnOnce(HttpClient) -> HttpClient) {
252
+ self.accessor.info().update_http_client(f);
253
+ }
254
+ }
255
+
256
+ /// # Operator async API.
257
+ impl Operator {
258
+ /// Check if this operator can work correctly.
259
+ ///
260
+ /// We will send a `list` request to path and return any errors we met.
261
+ ///
262
+ /// ```
263
+ /// # use std::sync::Arc;
264
+ /// # use anyhow::Result;
265
+ /// use opendal::Operator;
266
+ ///
267
+ /// # async fn test(op: Operator) -> Result<()> {
268
+ /// op.check().await?;
269
+ /// # Ok(())
270
+ /// # }
271
+ /// ```
272
+ pub async fn check(&self) -> Result<()> {
273
+ let mut ds = self.lister_with("/").limit(1).await?;
274
+
275
+ match ds.next().await {
276
+ Some(Err(e)) if e.kind() != ErrorKind::NotFound => Err(e),
277
+ _ => Ok(()),
278
+ }
279
+ }
280
+
281
+ /// Retrieve the metadata for the specified path.
282
+ ///
283
+ /// # Notes
284
+ ///
285
+ /// ## Extra Options
286
+ ///
287
+ /// [`Operator::stat`] is a wrapper around [`Operator::stat_with`] that uses no additional options.
288
+ /// To specify extra options such as `if_match` and `if_none_match`, please use [`Operator::stat_with`] instead.
289
+ ///
290
+ /// # Examples
291
+ ///
292
+ /// ## Check if file exists
293
+ ///
294
+ /// ```
295
+ /// # use anyhow::Result;
296
+ /// # use futures::io;
297
+ /// # use opendal::Operator;
298
+ /// use opendal::ErrorKind;
299
+ /// #
300
+ /// # async fn test(op: Operator) -> Result<()> {
301
+ /// if let Err(e) = op.stat("test").await {
302
+ /// if e.kind() == ErrorKind::NotFound {
303
+ /// println!("file not exist")
304
+ /// }
305
+ /// }
306
+ /// # Ok(())
307
+ /// # }
308
+ /// ```
309
+ pub async fn stat(&self, path: &str) -> Result<Metadata> {
310
+ self.stat_with(path).await
311
+ }
312
+
313
+ /// Retrieve the metadata of the specified path with additional options.
314
+ ///
315
+ /// # Options
316
+ ///
317
+ /// Check [`options::StatOptions`] for all available options.
318
+ ///
319
+ /// # Examples
320
+ ///
321
+ /// ## Get metadata while `ETag` matches
322
+ ///
323
+ /// `stat_with` will
324
+ ///
325
+ /// - return `Ok(metadata)` if `ETag` matches
326
+ /// - return `Err(error)` and `error.kind() == ErrorKind::ConditionNotMatch` if file exists but
327
+ /// `ETag` mismatch
328
+ /// - return `Err(err)` if other errors occur, for example, `NotFound`.
329
+ ///
330
+ /// ```
331
+ /// # use anyhow::Result;
332
+ /// # use futures::io;
333
+ /// # use opendal::Operator;
334
+ /// use opendal::ErrorKind;
335
+ /// #
336
+ /// # async fn test(op: Operator) -> Result<()> {
337
+ /// if let Err(e) = op.stat_with("test").if_match("<etag>").await {
338
+ /// if e.kind() == ErrorKind::ConditionNotMatch {
339
+ /// println!("file exists, but etag mismatch")
340
+ /// }
341
+ /// if e.kind() == ErrorKind::NotFound {
342
+ /// println!("file not exist")
343
+ /// }
344
+ /// }
345
+ /// # Ok(())
346
+ /// # }
347
+ /// ```
348
+ pub fn stat_with(&self, path: &str) -> FutureStat<impl Future<Output = Result<Metadata>>> {
349
+ let path = normalize_path(path);
350
+ OperatorFuture::new(
351
+ self.inner().clone(),
352
+ path,
353
+ options::StatOptions::default(),
354
+ Self::stat_inner,
355
+ )
356
+ }
357
+
358
+ /// Retrieve the metadata of the specified path with additional options.
359
+ ///
360
+ /// # Examples
361
+ ///
362
+ /// ## Get metadata while `ETag` matches
363
+ ///
364
+ /// `stat_with` will
365
+ ///
366
+ /// - return `Ok(metadata)` if `ETag` matches
367
+ /// - return `Err(error)` and `error.kind() == ErrorKind::ConditionNotMatch` if file exists but
368
+ /// `ETag` mismatch
369
+ /// - return `Err(err)` if other errors occur, for example, `NotFound`.
370
+ ///
371
+ /// ```
372
+ /// # use anyhow::Result;
373
+ /// # use futures::io;
374
+ /// # use opendal::Operator;
375
+ /// use opendal::options;
376
+ /// use opendal::ErrorKind;
377
+ /// #
378
+ /// # async fn test(op: Operator) -> Result<()> {
379
+ /// let res = op
380
+ /// .stat_options("test", options::StatOptions {
381
+ /// if_match: Some("<etag>".to_string()),
382
+ /// ..Default::default()
383
+ /// })
384
+ /// .await;
385
+ /// if let Err(e) = res {
386
+ /// if e.kind() == ErrorKind::ConditionNotMatch {
387
+ /// println!("file exists, but etag mismatch")
388
+ /// }
389
+ /// if e.kind() == ErrorKind::NotFound {
390
+ /// println!("file not exist")
391
+ /// }
392
+ /// }
393
+ /// # Ok(())
394
+ /// # }
395
+ /// ```
396
+ pub async fn stat_options(&self, path: &str, opts: options::StatOptions) -> Result<Metadata> {
397
+ let path = normalize_path(path);
398
+ Self::stat_inner(self.accessor.clone(), path, opts).await
399
+ }
400
+
401
+ #[inline]
402
+ async fn stat_inner(
403
+ acc: Accessor,
404
+ path: String,
405
+ opts: options::StatOptions,
406
+ ) -> Result<Metadata> {
407
+ let rp = acc.stat(&path, opts.into()).await?;
408
+ Ok(rp.into_metadata())
409
+ }
410
+
411
+ /// Check whether this path exists.
412
+ ///
413
+ /// # Example
414
+ ///
415
+ /// ```
416
+ /// use anyhow::Result;
417
+ /// use futures::io;
418
+ /// use opendal::Operator;
419
+ ///
420
+ /// async fn test(op: Operator) -> Result<()> {
421
+ /// let _ = op.exists("test").await?;
422
+ ///
423
+ /// Ok(())
424
+ /// }
425
+ /// ```
426
+ pub async fn exists(&self, path: &str) -> Result<bool> {
427
+ let r = self.stat(path).await;
428
+ match r {
429
+ Ok(_) => Ok(true),
430
+ Err(err) if err.kind() == ErrorKind::NotFound => Ok(false),
431
+ Err(err) => Err(err),
432
+ }
433
+ }
434
+
435
+ /// Create a directory at the specified path.
436
+ ///
437
+ /// # Notes
438
+ ///
439
+ /// To specify that a path is a directory, you must include a trailing slash (/).
440
+ /// Omitting the trailing slash may cause OpenDAL to return a `NotADirectory` error.
441
+ ///
442
+ /// # Behavior
443
+ ///
444
+ /// - Creating a directory that already exists will succeed.
445
+ /// - Directory creation is always recursive, functioning like `mkdir -p`.
446
+ ///
447
+ /// # Examples
448
+ ///
449
+ /// ```
450
+ /// # use opendal::Result;
451
+ /// # use opendal::Operator;
452
+ /// # async fn test(op: Operator) -> Result<()> {
453
+ /// op.create_dir("path/to/dir/").await?;
454
+ /// # Ok(())
455
+ /// # }
456
+ /// ```
457
+ pub async fn create_dir(&self, path: &str) -> Result<()> {
458
+ let path = normalize_path(path);
459
+
460
+ if !validate_path(&path, EntryMode::DIR) {
461
+ return Err(Error::new(
462
+ ErrorKind::NotADirectory,
463
+ "the path trying to create should end with `/`",
464
+ )
465
+ .with_operation("create_dir")
466
+ .with_context("service", self.inner().info().scheme())
467
+ .with_context("path", &path));
468
+ }
469
+
470
+ self.inner().create_dir(&path, OpCreateDir::new()).await?;
471
+
472
+ Ok(())
473
+ }
474
+
475
+ /// Read the entire file into bytes from given path.
476
+ ///
477
+ /// # Notes
478
+ ///
479
+ /// ## Additional Options
480
+ ///
481
+ /// [`Operator::read`] is a simplified method that does not support additional options. To access features like `range` and `if_match`, please use [`Operator::read_with`] or [`Operator::read_options`] instead.
482
+ ///
483
+ /// ## Streaming Read
484
+ ///
485
+ /// This function reads all content into memory at once. For more precise memory management or to read big file lazily, please use [`Operator::reader`].
486
+ ///
487
+ /// # Examples
488
+ ///
489
+ /// ```
490
+ /// # use opendal::Result;
491
+ /// # use opendal::Operator;
492
+ /// # use futures::TryStreamExt;
493
+ /// # async fn test(op: Operator) -> Result<()> {
494
+ /// let bs = op.read("path/to/file").await?;
495
+ /// # Ok(())
496
+ /// # }
497
+ /// ```
498
+ pub async fn read(&self, path: &str) -> Result<Buffer> {
499
+ self.read_options(path, options::ReadOptions::default())
500
+ .await
501
+ }
502
+
503
+ /// Read the entire file into bytes from given path with additional options.
504
+ ///
505
+ /// # Notes
506
+ ///
507
+ /// ## Streaming Read
508
+ ///
509
+ /// This function reads all content into memory at once. For more precise memory management or to read big file lazily, please use [`Operator::reader`].
510
+ ///
511
+ /// # Options
512
+ ///
513
+ /// Visit [`options::ReadOptions`] for all available options.
514
+ ///
515
+ /// # Examples
516
+ ///
517
+ /// Read the first 10 bytes of a file:
518
+ ///
519
+ /// ```
520
+ /// # use opendal::Result;
521
+ /// # use opendal::Operator;
522
+ /// # async fn test(op: Operator) -> Result<()> {
523
+ /// let bs = op.read_with("path/to/file").range(0..10).await?;
524
+ /// # Ok(())
525
+ /// # }
526
+ /// ```
527
+ pub fn read_with(&self, path: &str) -> FutureRead<impl Future<Output = Result<Buffer>>> {
528
+ let path = normalize_path(path);
529
+
530
+ OperatorFuture::new(
531
+ self.inner().clone(),
532
+ path,
533
+ options::ReadOptions::default(),
534
+ Self::read_inner,
535
+ )
536
+ }
537
+
538
+ /// Read the entire file into bytes from given path with additional options.
539
+ ///
540
+ /// # Notes
541
+ ///
542
+ /// ## Streaming Read
543
+ ///
544
+ /// This function reads all content into memory at once. For more precise memory management or to read big file lazily, please use [`Operator::reader`].
545
+ ///
546
+ /// # Examples
547
+ ///
548
+ /// Read the first 10 bytes of a file:
549
+ ///
550
+ /// ```
551
+ /// # use opendal::Result;
552
+ /// # use opendal::Operator;
553
+ /// use opendal::options;
554
+ /// # async fn test(op: Operator) -> Result<()> {
555
+ /// let bs = op
556
+ /// .read_options("path/to/file", options::ReadOptions {
557
+ /// range: (0..10).into(),
558
+ /// ..Default::default()
559
+ /// })
560
+ /// .await?;
561
+ /// # Ok(())
562
+ /// # }
563
+ /// ```
564
+ pub async fn read_options(&self, path: &str, opts: options::ReadOptions) -> Result<Buffer> {
565
+ let path = normalize_path(path);
566
+ Self::read_inner(self.inner().clone(), path, opts).await
567
+ }
568
+
569
+ #[inline]
570
+ async fn read_inner(acc: Accessor, path: String, opts: options::ReadOptions) -> Result<Buffer> {
571
+ if !validate_path(&path, EntryMode::FILE) {
572
+ return Err(
573
+ Error::new(ErrorKind::IsADirectory, "read path is a directory")
574
+ .with_operation("read")
575
+ .with_context("service", acc.info().scheme())
576
+ .with_context("path", &path),
577
+ );
578
+ }
579
+
580
+ let (args, opts) = opts.into();
581
+ let range = args.range();
582
+ let context = ReadContext::new(acc, path, args, opts);
583
+ let r = Reader::new(context);
584
+ let buf = r.read(range.to_range()).await?;
585
+ Ok(buf)
586
+ }
587
+
588
+ /// Create a new reader of given path.
589
+ ///
590
+ /// # Notes
591
+ ///
592
+ /// ## Extra Options
593
+ ///
594
+ /// [`Operator::reader`] is a simplified method without any options. To use additional options such as `concurrent` or `if_match`, please use [`Operator::reader_with`] or [`Operator::reader_options`] instead.
595
+ ///
596
+ /// # Examples
597
+ ///
598
+ /// ```
599
+ /// # use opendal::Result;
600
+ /// # use opendal::Operator;
601
+ /// # use futures::TryStreamExt;
602
+ /// # use opendal::Scheme;
603
+ /// # async fn test(op: Operator) -> Result<()> {
604
+ /// let r = op.reader("path/to/file").await?;
605
+ /// // Read the first 10 bytes of the file
606
+ /// let data = r.read(0..10).await?;
607
+ /// # Ok(())
608
+ /// # }
609
+ /// ```
610
+ pub async fn reader(&self, path: &str) -> Result<Reader> {
611
+ self.reader_with(path).await
612
+ }
613
+
614
+ /// Create a new reader of given path with additional options.
615
+ ///
616
+ /// # Options
617
+ ///
618
+ /// Visit [`options::ReaderOptions`] for all available options.
619
+ ///
620
+ /// # Examples
621
+ ///
622
+ /// Create a reader with a specific version ID:
623
+ ///
624
+ /// ```
625
+ /// # use opendal::Result;
626
+ /// # use opendal::Operator;
627
+ /// # use opendal::Scheme;
628
+ /// # async fn test(op: Operator) -> Result<()> {
629
+ /// let r = op.reader_with("path/to/file").version("version_id").await?;
630
+ /// // Read the first 10 bytes of the file
631
+ /// let data = r.read(0..10).await?;
632
+ /// # Ok(())
633
+ /// # }
634
+ /// ```
635
+ pub fn reader_with(&self, path: &str) -> FutureReader<impl Future<Output = Result<Reader>>> {
636
+ let path = normalize_path(path);
637
+
638
+ OperatorFuture::new(
639
+ self.inner().clone(),
640
+ path,
641
+ options::ReaderOptions::default(),
642
+ Self::reader_inner,
643
+ )
644
+ }
645
+
646
+ /// Create a new reader of given path with additional options.
647
+ ///
648
+ /// # Examples
649
+ ///
650
+ /// Create a reader with a specific version ID:
651
+ ///
652
+ /// ```
653
+ /// # use opendal::Result;
654
+ /// # use opendal::Operator;
655
+ /// use opendal::options;
656
+ /// # async fn test(op: Operator) -> Result<()> {
657
+ /// let r = op
658
+ /// .reader_options("path/to/file", options::ReaderOptions {
659
+ /// version: Some("version_id".to_string()),
660
+ /// ..Default::default()
661
+ /// })
662
+ /// .await?;
663
+ /// // Read the first 10 bytes of the file
664
+ /// let data = r.read(0..10).await?;
665
+ /// # Ok(())
666
+ /// # }
667
+ /// ```
668
+ pub async fn reader_options(&self, path: &str, opts: options::ReaderOptions) -> Result<Reader> {
669
+ let path = normalize_path(path);
670
+ Self::reader_inner(self.inner().clone(), path, opts).await
671
+ }
672
+
673
+ /// Allow this unused async since we don't want
674
+ /// to change our public API.
675
+ #[allow(clippy::unused_async)]
676
+ #[inline]
677
+ async fn reader_inner(
678
+ acc: Accessor,
679
+ path: String,
680
+ options: options::ReaderOptions,
681
+ ) -> Result<Reader> {
682
+ if !validate_path(&path, EntryMode::FILE) {
683
+ return Err(
684
+ Error::new(ErrorKind::IsADirectory, "read path is a directory")
685
+ .with_operation("Operator::reader")
686
+ .with_context("service", acc.info().scheme())
687
+ .with_context("path", path),
688
+ );
689
+ }
690
+
691
+ let (args, opts) = options.into();
692
+ let context = ReadContext::new(acc, path, args, opts);
693
+ Ok(Reader::new(context))
694
+ }
695
+
696
+ /// Write all data to the specified path at once.
697
+ ///
698
+ /// # Notes
699
+ ///
700
+ /// Visit [`performance::concurrent_write`][crate::docs::performance::concurrent_write] for more details on concurrent writes.
701
+ ///
702
+ /// ## Extra Options
703
+ ///
704
+ /// [`Operator::write`] is a simplified method that does not include additional options.
705
+ /// For advanced features such as `chunk` and `concurrent`, use [`Operator::write_with`] or [`Operator::write_options`] instead.
706
+ ///
707
+ /// ## Streaming Write
708
+ ///
709
+ /// This method executes a single bulk write operation. For more precise memory management
710
+ /// or to write data in a streaming fashion, use [`Operator::writer`] instead.
711
+ ///
712
+ /// ## Multipart Uploads
713
+ ///
714
+ /// OpenDAL offers multipart upload capabilities through the [`Writer`] abstraction,
715
+ /// automatically managing all upload details for you. You can fine-tune the upload process
716
+ /// by adjusting the `chunk` size and the number of `concurrent` operations using [`Operator::writer_with`].
717
+ ///
718
+ /// # Examples
719
+ ///
720
+ /// ```
721
+ /// # use opendal::Result;
722
+ /// # use opendal::Operator;
723
+ /// # use futures::StreamExt;
724
+ /// # use futures::SinkExt;
725
+ /// use bytes::Bytes;
726
+ ///
727
+ /// # async fn test(op: Operator) -> Result<()> {
728
+ /// op.write("path/to/file", vec![0; 4096]).await?;
729
+ /// # Ok(())
730
+ /// # }
731
+ /// ```
732
+ pub async fn write(&self, path: &str, bs: impl Into<Buffer>) -> Result<Metadata> {
733
+ let bs = bs.into();
734
+ self.write_with(path, bs).await
735
+ }
736
+
737
+ /// Write all data to the specified path at once with additional options.
738
+ ///
739
+ /// # Notes
740
+ ///
741
+ /// Visit [`performance::concurrent_write`][crate::docs::performance::concurrent_write] for more details on concurrent writes.
742
+ ///
743
+ /// ## Streaming Write
744
+ ///
745
+ /// This method executes a single bulk write operation. For more precise memory management
746
+ /// or to write data in a streaming fashion, use [`Operator::writer`] instead.
747
+ ///
748
+ /// ## Multipart Uploads
749
+ ///
750
+ /// OpenDAL offers multipart upload capabilities through the [`Writer`] abstraction,
751
+ /// automatically managing all upload details for you. You can fine-tune the upload process
752
+ /// by adjusting the `chunk` size and the number of `concurrent` operations using [`Operator::writer_with`].
753
+ ///
754
+ /// # Options
755
+ ///
756
+ /// Visit [`options::WriteOptions`] for all available options.
757
+ ///
758
+ /// # Examples
759
+ ///
760
+ /// Write data to a file only when it does not already exist:
761
+ ///
762
+ /// ```
763
+ /// # use opendal::Result;
764
+ /// # use opendal::Operator;
765
+ /// use bytes::Bytes;
766
+ ///
767
+ /// # async fn test(op: Operator) -> Result<()> {
768
+ /// let _ = op
769
+ /// .write_with("path/to/file", vec![0; 4096])
770
+ /// .if_not_exists(true)
771
+ /// .await?;
772
+ /// # Ok(())
773
+ /// # }
774
+ /// ```
775
+ pub fn write_with(
776
+ &self,
777
+ path: &str,
778
+ bs: impl Into<Buffer>,
779
+ ) -> FutureWrite<impl Future<Output = Result<Metadata>>> {
780
+ let path = normalize_path(path);
781
+ let bs = bs.into();
782
+
783
+ OperatorFuture::new(
784
+ self.inner().clone(),
785
+ path,
786
+ (options::WriteOptions::default(), bs),
787
+ |inner, path, (opts, bs)| Self::write_inner(inner, path, bs, opts),
788
+ )
789
+ }
790
+
791
+ /// Write all data to the specified path at once with additional options.
792
+ ///
793
+ /// # Notes
794
+ ///
795
+ /// Visit [`performance::concurrent_write`][crate::docs::performance::concurrent_write] for more details on concurrent writes.
796
+ ///
797
+ /// ## Streaming Write
798
+ ///
799
+ /// This method executes a single bulk write operation. For more precise memory management
800
+ /// or to write data in a streaming fashion, use [`Operator::writer`] instead.
801
+ ///
802
+ /// ## Multipart Uploads
803
+ ///
804
+ /// OpenDAL offers multipart upload capabilities through the [`Writer`] abstraction,
805
+ /// automatically managing all upload details for you. You can fine-tune the upload process
806
+ /// by adjusting the `chunk` size and the number of `concurrent` operations using [`Operator::writer_with`].
807
+ ///
808
+ /// # Examples
809
+ ///
810
+ /// Write data to a file only when it does not already exist:
811
+ ///
812
+ /// ```
813
+ /// # use opendal::Result;
814
+ /// # use opendal::Operator;
815
+ /// use opendal::options;
816
+ ///
817
+ /// # async fn test(op: Operator) -> Result<()> {
818
+ /// let _ = op
819
+ /// .write_options("path/to/file", vec![0; 4096], options::WriteOptions {
820
+ /// if_not_exists: true,
821
+ /// ..Default::default()
822
+ /// })
823
+ /// .await?;
824
+ /// # Ok(())
825
+ /// # }
826
+ /// ```
827
+ pub async fn write_options(
828
+ &self,
829
+ path: &str,
830
+ bs: impl Into<Buffer>,
831
+ opts: options::WriteOptions,
832
+ ) -> Result<Metadata> {
833
+ let path = normalize_path(path);
834
+ Self::write_inner(self.inner().clone(), path, bs.into(), opts).await
835
+ }
836
+
837
+ #[inline]
838
+ async fn write_inner(
839
+ acc: Accessor,
840
+ path: String,
841
+ bs: Buffer,
842
+ opts: options::WriteOptions,
843
+ ) -> Result<Metadata> {
844
+ if !validate_path(&path, EntryMode::FILE) {
845
+ return Err(
846
+ Error::new(ErrorKind::IsADirectory, "write path is a directory")
847
+ .with_operation("Operator::write")
848
+ .with_context("service", acc.info().scheme())
849
+ .with_context("path", &path),
850
+ );
851
+ }
852
+
853
+ let (args, opts) = opts.into();
854
+ let context = WriteContext::new(acc, path, args, opts);
855
+ let mut w = Writer::new(context).await?;
856
+ w.write(bs).await?;
857
+ w.close().await
858
+ }
859
+
860
+ /// Create a new writer of given path.
861
+ ///
862
+ /// # Notes
863
+ ///
864
+ /// ## Writer Features
865
+ ///
866
+ /// The writer provides several powerful capabilities:
867
+ /// - Streaming writes for continuous data transfer
868
+ /// - Automatic multipart upload handling
869
+ /// - Memory-efficient chunk-based writing
870
+ ///
871
+ /// ## Extra Options
872
+ ///
873
+ /// [`Operator::writer`] is a simplified version that does not include additional options.
874
+ /// For advanced features such as `chunk` and `concurrent`, use [`Operator::writer_with`] or [`Operator::writer_options`] instead.
875
+ ///
876
+ /// # Examples
877
+ ///
878
+ /// ```
879
+ /// # use opendal::Result;
880
+ /// # use opendal::Operator;
881
+ /// use bytes::Bytes;
882
+ ///
883
+ /// # async fn test(op: Operator) -> Result<()> {
884
+ /// let mut w = op.writer("path/to/file").await?;
885
+ /// w.write(vec![0; 4096]).await?;
886
+ /// w.write(vec![1; 4096]).await?;
887
+ /// w.close().await?;
888
+ /// # Ok(())
889
+ /// # }
890
+ /// ```
891
+ pub async fn writer(&self, path: &str) -> Result<Writer> {
892
+ self.writer_with(path).await
893
+ }
894
+
895
+ /// Create a new writer of given path with additional options.
896
+ ///
897
+ /// # Notes
898
+ ///
899
+ /// ## Writer Features
900
+ ///
901
+ /// The writer provides several powerful capabilities:
902
+ /// - Streaming writes for continuous data transfer
903
+ /// - Automatic multipart upload handling
904
+ /// - Memory-efficient chunk-based writing
905
+ ///
906
+ /// ## Chunk Size Handling
907
+ ///
908
+ /// Storage services often have specific requirements for chunk sizes:
909
+ /// - Services like `s3` may return `EntityTooSmall` errors for undersized chunks
910
+ /// - Using small chunks in cloud storage services can lead to increased costs
911
+ ///
912
+ /// OpenDAL automatically determines optimal chunk sizes based on the service's
913
+ /// [Capability](crate::types::Capability). However, you can override this by explicitly
914
+ /// setting the `chunk` parameter.
915
+ ///
916
+ /// Visit [`performance::concurrent_write`][crate::docs::performance::concurrent_write] for more details on concurrent writes.
917
+ ///
918
+ /// # Examples
919
+ ///
920
+ /// ```
921
+ /// # use opendal::Result;
922
+ /// # use opendal::Operator;
923
+ /// use bytes::Bytes;
924
+ ///
925
+ /// # async fn test(op: Operator) -> Result<()> {
926
+ /// let mut w = op
927
+ /// .writer_with("path/to/file")
928
+ /// .chunk(4 * 1024 * 1024)
929
+ /// .concurrent(8)
930
+ /// .await?;
931
+ /// w.write(vec![0; 4096]).await?;
932
+ /// w.write(vec![1; 4096]).await?;
933
+ /// w.close().await?;
934
+ /// # Ok(())
935
+ /// # }
936
+ /// ```
937
+ pub fn writer_with(&self, path: &str) -> FutureWriter<impl Future<Output = Result<Writer>>> {
938
+ let path = normalize_path(path);
939
+
940
+ OperatorFuture::new(
941
+ self.inner().clone(),
942
+ path,
943
+ options::WriteOptions::default(),
944
+ Self::writer_inner,
945
+ )
946
+ }
947
+
948
+ /// Create a new writer of given path with additional options.
949
+ ///
950
+ /// # Notes
951
+ ///
952
+ /// ## Writer Features
953
+ ///
954
+ /// The writer provides several powerful capabilities:
955
+ /// - Streaming writes for continuous data transfer
956
+ /// - Automatic multipart upload handling
957
+ /// - Memory-efficient chunk-based writing
958
+ ///
959
+ /// ## Chunk Size Handling
960
+ ///
961
+ /// Storage services often have specific requirements for chunk sizes:
962
+ /// - Services like `s3` may return `EntityTooSmall` errors for undersized chunks
963
+ /// - Using small chunks in cloud storage services can lead to increased costs
964
+ ///
965
+ /// OpenDAL automatically determines optimal chunk sizes based on the service's
966
+ /// [Capability](crate::types::Capability). However, you can override this by explicitly
967
+ /// setting the `chunk` parameter.
968
+ ///
969
+ /// Visit [`performance::concurrent_write`][crate::docs::performance::concurrent_write] for more details on concurrent writes.
970
+ ///
971
+ /// # Examples
972
+ ///
973
+ /// Write data to a file in 4MiB chunk size and at 8 concurrency:
974
+ ///
975
+ /// ```
976
+ /// # use opendal::Result;
977
+ /// # use opendal::Operator;
978
+ /// use bytes::Bytes;
979
+ ///
980
+ /// # async fn test(op: Operator) -> Result<()> {
981
+ /// let mut w = op
982
+ /// .writer_with("path/to/file")
983
+ /// .chunk(4 * 1024 * 1024)
984
+ /// .concurrent(8)
985
+ /// .await?;
986
+ /// w.write(vec![0; 4096]).await?;
987
+ /// w.write(vec![1; 4096]).await?;
988
+ /// w.close().await?;
989
+ /// # Ok(())
990
+ /// # }
991
+ /// ```
992
+ pub async fn writer_options(&self, path: &str, opts: options::WriteOptions) -> Result<Writer> {
993
+ let path = normalize_path(path);
994
+ Self::writer_inner(self.inner().clone(), path, opts).await
995
+ }
996
+
997
+ #[inline]
998
+ async fn writer_inner(
999
+ acc: Accessor,
1000
+ path: String,
1001
+ opts: options::WriteOptions,
1002
+ ) -> Result<Writer> {
1003
+ if !validate_path(&path, EntryMode::FILE) {
1004
+ return Err(
1005
+ Error::new(ErrorKind::IsADirectory, "write path is a directory")
1006
+ .with_operation("Operator::writer")
1007
+ .with_context("service", acc.info().scheme())
1008
+ .with_context("path", &path),
1009
+ );
1010
+ }
1011
+
1012
+ let (args, opts) = opts.into();
1013
+ let context = WriteContext::new(acc, path, args, opts);
1014
+ let w = Writer::new(context).await?;
1015
+ Ok(w)
1016
+ }
1017
+
1018
+ /// Copy a file from `from` to `to`.
1019
+ ///
1020
+ /// # Notes
1021
+ ///
1022
+ /// - `from` and `to` must be a file.
1023
+ /// - `to` will be overwritten if it exists.
1024
+ /// - If `from` and `to` are the same, an `IsSameFile` error will occur.
1025
+ /// - `copy` is idempotent. For same `from` and `to` input, the result will be the same.
1026
+ ///
1027
+ /// # Examples
1028
+ ///
1029
+ /// ```
1030
+ /// # use opendal::Result;
1031
+ /// # use opendal::Operator;
1032
+ ///
1033
+ /// # async fn test(op: Operator) -> Result<()> {
1034
+ /// op.copy("path/to/file", "path/to/file2").await?;
1035
+ /// # Ok(())
1036
+ /// # }
1037
+ /// ```
1038
+ pub async fn copy(&self, from: &str, to: &str) -> Result<()> {
1039
+ let from = normalize_path(from);
1040
+
1041
+ if !validate_path(&from, EntryMode::FILE) {
1042
+ return Err(
1043
+ Error::new(ErrorKind::IsADirectory, "from path is a directory")
1044
+ .with_operation("Operator::copy")
1045
+ .with_context("service", self.info().scheme())
1046
+ .with_context("from", from),
1047
+ );
1048
+ }
1049
+
1050
+ let to = normalize_path(to);
1051
+
1052
+ if !validate_path(&to, EntryMode::FILE) {
1053
+ return Err(
1054
+ Error::new(ErrorKind::IsADirectory, "to path is a directory")
1055
+ .with_operation("Operator::copy")
1056
+ .with_context("service", self.info().scheme())
1057
+ .with_context("to", to),
1058
+ );
1059
+ }
1060
+
1061
+ if from == to {
1062
+ return Err(
1063
+ Error::new(ErrorKind::IsSameFile, "from and to paths are same")
1064
+ .with_operation("Operator::copy")
1065
+ .with_context("service", self.info().scheme())
1066
+ .with_context("from", from)
1067
+ .with_context("to", to),
1068
+ );
1069
+ }
1070
+
1071
+ self.inner().copy(&from, &to, OpCopy::new()).await?;
1072
+
1073
+ Ok(())
1074
+ }
1075
+
1076
+ /// Copy a file from `from` to `to` with additional options.
1077
+ ///
1078
+ /// # Notes
1079
+ ///
1080
+ /// - `from` and `to` must be a file.
1081
+ /// - If `from` and `to` are the same, an `IsSameFile` error will occur.
1082
+ /// - `copy` is idempotent. For same `from` and `to` input, the result will be the same.
1083
+ ///
1084
+ /// # Options
1085
+ ///
1086
+ /// Visit [`options::CopyOptions`] for all available options.
1087
+ ///
1088
+ /// # Examples
1089
+ ///
1090
+ /// Copy a file only if the destination doesn't exist:
1091
+ ///
1092
+ /// ```
1093
+ /// # use opendal::Result;
1094
+ /// # use opendal::Operator;
1095
+ ///
1096
+ /// # async fn test(op: Operator) -> Result<()> {
1097
+ /// op.copy_with("path/to/file", "path/to/file2")
1098
+ /// .if_not_exists(true)
1099
+ /// .await?;
1100
+ /// # Ok(())
1101
+ /// # }
1102
+ /// ```
1103
+ pub fn copy_with(&self, from: &str, to: &str) -> FutureCopy<impl Future<Output = Result<()>>> {
1104
+ let from = normalize_path(from);
1105
+ let to = normalize_path(to);
1106
+
1107
+ OperatorFuture::new(
1108
+ self.inner().clone(),
1109
+ from,
1110
+ (options::CopyOptions::default(), to),
1111
+ Self::copy_inner,
1112
+ )
1113
+ }
1114
+
1115
+ /// Copy a file from `from` to `to` with additional options.
1116
+ ///
1117
+ /// # Notes
1118
+ ///
1119
+ /// - `from` and `to` must be a file.
1120
+ /// - If `from` and `to` are the same, an `IsSameFile` error will occur.
1121
+ /// - `copy` is idempotent. For same `from` and `to` input, the result will be the same.
1122
+ ///
1123
+ /// # Options
1124
+ ///
1125
+ /// Check [`options::CopyOptions`] for all available options.
1126
+ ///
1127
+ /// # Examples
1128
+ ///
1129
+ /// Copy a file only if the destination doesn't exist:
1130
+ ///
1131
+ /// ```
1132
+ /// # use opendal::Result;
1133
+ /// # use opendal::Operator;
1134
+ /// # use opendal::options::CopyOptions;
1135
+ ///
1136
+ /// # async fn test(op: Operator) -> Result<()> {
1137
+ /// let mut opts = CopyOptions::default();
1138
+ /// opts.if_not_exists = true;
1139
+ /// op.copy_options("path/to/file", "path/to/file2", opts).await?;
1140
+ /// # Ok(())
1141
+ /// # }
1142
+ /// ```
1143
+ pub async fn copy_options(
1144
+ &self,
1145
+ from: &str,
1146
+ to: &str,
1147
+ opts: impl Into<options::CopyOptions>,
1148
+ ) -> Result<()> {
1149
+ let from = normalize_path(from);
1150
+ let to = normalize_path(to);
1151
+ let opts = opts.into();
1152
+
1153
+ Self::copy_inner(self.inner().clone(), from, (opts, to)).await
1154
+ }
1155
+
1156
+ async fn copy_inner(
1157
+ acc: Accessor,
1158
+ from: String,
1159
+ (opts, to): (options::CopyOptions, String),
1160
+ ) -> Result<()> {
1161
+ if !validate_path(&from, EntryMode::FILE) {
1162
+ return Err(
1163
+ Error::new(ErrorKind::IsADirectory, "from path is a directory")
1164
+ .with_operation("Operator::copy")
1165
+ .with_context("service", acc.info().scheme())
1166
+ .with_context("from", from),
1167
+ );
1168
+ }
1169
+
1170
+ if !validate_path(&to, EntryMode::FILE) {
1171
+ return Err(
1172
+ Error::new(ErrorKind::IsADirectory, "to path is a directory")
1173
+ .with_operation("Operator::copy")
1174
+ .with_context("service", acc.info().scheme())
1175
+ .with_context("to", to),
1176
+ );
1177
+ }
1178
+
1179
+ if from == to {
1180
+ return Err(
1181
+ Error::new(ErrorKind::IsSameFile, "from and to paths are same")
1182
+ .with_operation("Operator::copy")
1183
+ .with_context("service", acc.info().scheme())
1184
+ .with_context("from", &from)
1185
+ .with_context("to", &to),
1186
+ );
1187
+ }
1188
+
1189
+ let mut op = OpCopy::new();
1190
+ if opts.if_not_exists {
1191
+ op = op.with_if_not_exists(true);
1192
+ }
1193
+
1194
+ acc.copy(&from, &to, op).await.map(|_| ())
1195
+ }
1196
+
1197
+ /// Rename a file from `from` to `to`.
1198
+ ///
1199
+ /// # Notes
1200
+ ///
1201
+ /// - `from` and `to` must be a file.
1202
+ /// - `to` will be overwritten if it exists.
1203
+ /// - If `from` and `to` are the same, an `IsSameFile` error will occur.
1204
+ ///
1205
+ /// # Examples
1206
+ ///
1207
+ /// ```
1208
+ /// # use opendal::Result;
1209
+ /// # use opendal::Operator;
1210
+ ///
1211
+ /// # async fn test(op: Operator) -> Result<()> {
1212
+ /// op.rename("path/to/file", "path/to/file2").await?;
1213
+ /// # Ok(())
1214
+ /// # }
1215
+ /// ```
1216
+ pub async fn rename(&self, from: &str, to: &str) -> Result<()> {
1217
+ let from = normalize_path(from);
1218
+
1219
+ if !validate_path(&from, EntryMode::FILE) {
1220
+ return Err(
1221
+ Error::new(ErrorKind::IsADirectory, "from path is a directory")
1222
+ .with_operation("Operator::move_")
1223
+ .with_context("service", self.info().scheme())
1224
+ .with_context("from", from),
1225
+ );
1226
+ }
1227
+
1228
+ let to = normalize_path(to);
1229
+
1230
+ if !validate_path(&to, EntryMode::FILE) {
1231
+ return Err(
1232
+ Error::new(ErrorKind::IsADirectory, "to path is a directory")
1233
+ .with_operation("Operator::move_")
1234
+ .with_context("service", self.info().scheme())
1235
+ .with_context("to", to),
1236
+ );
1237
+ }
1238
+
1239
+ if from == to {
1240
+ return Err(
1241
+ Error::new(ErrorKind::IsSameFile, "from and to paths are same")
1242
+ .with_operation("Operator::move_")
1243
+ .with_context("service", self.info().scheme())
1244
+ .with_context("from", from)
1245
+ .with_context("to", to),
1246
+ );
1247
+ }
1248
+
1249
+ self.inner().rename(&from, &to, OpRename::new()).await?;
1250
+
1251
+ Ok(())
1252
+ }
1253
+
1254
+ /// Delete the given path.
1255
+ ///
1256
+ /// # Notes
1257
+ ///
1258
+ /// - Deleting a file that does not exist won't return errors.
1259
+ ///
1260
+ /// # Examples
1261
+ ///
1262
+ /// ```
1263
+ /// # use anyhow::Result;
1264
+ /// # use futures::io;
1265
+ /// # use opendal::Operator;
1266
+ /// # async fn test(op: Operator) -> Result<()> {
1267
+ /// op.delete("test").await?;
1268
+ /// # Ok(())
1269
+ /// # }
1270
+ /// ```
1271
+ pub async fn delete(&self, path: &str) -> Result<()> {
1272
+ self.delete_with(path).await
1273
+ }
1274
+
1275
+ /// Delete the given path with additional options.
1276
+ ///
1277
+ /// # Notes
1278
+ ///
1279
+ /// - Deleting a file that does not exist won't return errors.
1280
+ ///
1281
+ /// # Options
1282
+ ///
1283
+ /// Visit [`options::DeleteOptions`] for all available options.
1284
+ ///
1285
+ /// # Examples
1286
+ ///
1287
+ /// Delete a specific version of a file:
1288
+ ///
1289
+ /// ```
1290
+ /// # use opendal::Result;
1291
+ /// # use opendal::Operator;
1292
+ ///
1293
+ /// # async fn test(op: Operator, version: &str) -> Result<()> {
1294
+ /// op.delete_with("path/to/file").version(version).await?;
1295
+ /// # Ok(())
1296
+ /// # }
1297
+ /// ```
1298
+ pub fn delete_with(&self, path: &str) -> FutureDelete<impl Future<Output = Result<()>>> {
1299
+ let path = normalize_path(path);
1300
+
1301
+ OperatorFuture::new(
1302
+ self.inner().clone(),
1303
+ path,
1304
+ options::DeleteOptions::default(),
1305
+ Self::delete_inner,
1306
+ )
1307
+ }
1308
+
1309
+ /// Delete the given path with additional options.
1310
+ ///
1311
+ /// # Notes
1312
+ ///
1313
+ /// - Deleting a file that does not exist won't return errors.
1314
+ ///
1315
+ /// # Examples
1316
+ ///
1317
+ /// Delete a specific version of a file:
1318
+ ///
1319
+ /// ```
1320
+ /// # use opendal::Result;
1321
+ /// # use opendal::Operator;
1322
+ /// use opendal::options;
1323
+ ///
1324
+ /// # async fn test(op: Operator, version: &str) -> Result<()> {
1325
+ /// op.delete_options("path/to/file", options::DeleteOptions {
1326
+ /// version: Some(version.to_string()),
1327
+ /// ..Default::default()
1328
+ /// })
1329
+ /// .await?;
1330
+ /// # Ok(())
1331
+ /// # }
1332
+ /// ```
1333
+ pub async fn delete_options(&self, path: &str, opts: options::DeleteOptions) -> Result<()> {
1334
+ let path = normalize_path(path);
1335
+ Self::delete_inner(self.inner().clone(), path, opts).await
1336
+ }
1337
+
1338
+ async fn delete_inner(acc: Accessor, path: String, opts: options::DeleteOptions) -> Result<()> {
1339
+ let (_, mut deleter) = acc.delete_dyn().await?;
1340
+ let args = opts.into();
1341
+ deleter.delete_dyn(&path, args)?;
1342
+ deleter.flush_dyn().await?;
1343
+ Ok(())
1344
+ }
1345
+
1346
+ /// Delete an infallible iterator of paths.
1347
+ ///
1348
+ /// Also see:
1349
+ ///
1350
+ /// - [`Operator::delete_try_iter`]: delete a fallible iterator of paths.
1351
+ /// - [`Operator::delete_stream`]: delete an infallible stream of paths.
1352
+ /// - [`Operator::delete_try_stream`]: delete a fallible stream of paths.
1353
+ pub async fn delete_iter<I, D>(&self, iter: I) -> Result<()>
1354
+ where
1355
+ I: IntoIterator<Item = D>,
1356
+ D: IntoDeleteInput,
1357
+ {
1358
+ let mut deleter = self.deleter().await?;
1359
+ deleter.delete_iter(iter).await?;
1360
+ deleter.close().await?;
1361
+ Ok(())
1362
+ }
1363
+
1364
+ /// Delete a fallible iterator of paths.
1365
+ ///
1366
+ /// Also see:
1367
+ ///
1368
+ /// - [`Operator::delete_iter`]: delete an infallible iterator of paths.
1369
+ /// - [`Operator::delete_stream`]: delete an infallible stream of paths.
1370
+ /// - [`Operator::delete_try_stream`]: delete a fallible stream of paths.
1371
+ pub async fn delete_try_iter<I, D>(&self, try_iter: I) -> Result<()>
1372
+ where
1373
+ I: IntoIterator<Item = Result<D>>,
1374
+ D: IntoDeleteInput,
1375
+ {
1376
+ let mut deleter = self.deleter().await?;
1377
+ deleter.delete_try_iter(try_iter).await?;
1378
+ deleter.close().await?;
1379
+ Ok(())
1380
+ }
1381
+
1382
+ /// Delete an infallible stream of paths.
1383
+ ///
1384
+ /// Also see:
1385
+ ///
1386
+ /// - [`Operator::delete_iter`]: delete an infallible iterator of paths.
1387
+ /// - [`Operator::delete_try_iter`]: delete a fallible iterator of paths.
1388
+ /// - [`Operator::delete_try_stream`]: delete a fallible stream of paths.
1389
+ pub async fn delete_stream<S, D>(&self, stream: S) -> Result<()>
1390
+ where
1391
+ S: Stream<Item = D>,
1392
+ D: IntoDeleteInput,
1393
+ {
1394
+ let mut deleter = self.deleter().await?;
1395
+ deleter.delete_stream(stream).await?;
1396
+ deleter.close().await?;
1397
+ Ok(())
1398
+ }
1399
+
1400
+ /// Delete a fallible stream of paths.
1401
+ ///
1402
+ /// Also see:
1403
+ ///
1404
+ /// - [`Operator::delete_iter`]: delete an infallible iterator of paths.
1405
+ /// - [`Operator::delete_try_iter`]: delete a fallible iterator of paths.
1406
+ /// - [`Operator::delete_stream`]: delete an infallible stream of paths.
1407
+ pub async fn delete_try_stream<S, D>(&self, try_stream: S) -> Result<()>
1408
+ where
1409
+ S: Stream<Item = Result<D>>,
1410
+ D: IntoDeleteInput,
1411
+ {
1412
+ let mut deleter = self.deleter().await?;
1413
+ deleter.delete_try_stream(try_stream).await?;
1414
+ deleter.close().await?;
1415
+ Ok(())
1416
+ }
1417
+
1418
+ /// Create a [`Deleter`] to continuously remove content from storage.
1419
+ ///
1420
+ /// It leverages batch deletion capabilities provided by storage services for efficient removal.
1421
+ ///
1422
+ /// Users can have more control over the deletion process by using [`Deleter`] directly.
1423
+ pub async fn deleter(&self) -> Result<Deleter> {
1424
+ Deleter::create(self.inner().clone()).await
1425
+ }
1426
+
1427
+ /// Remove the path and all nested dirs and files recursively.
1428
+ ///
1429
+ /// # Notes
1430
+ ///
1431
+ /// If underlying services support delete in batch, we will use batch
1432
+ /// delete instead.
1433
+ ///
1434
+ /// # Examples
1435
+ ///
1436
+ /// ```
1437
+ /// # use anyhow::Result;
1438
+ /// # use futures::io;
1439
+ /// # use opendal::Operator;
1440
+ /// #
1441
+ /// # async fn test(op: Operator) -> Result<()> {
1442
+ /// op.remove_all("path/to/dir").await?;
1443
+ /// # Ok(())
1444
+ /// # }
1445
+ /// ```
1446
+ pub async fn remove_all(&self, path: &str) -> Result<()> {
1447
+ match self.stat(path).await {
1448
+ // If object exists.
1449
+ Ok(metadata) => {
1450
+ // If the object is a file, we can delete it.
1451
+ if metadata.mode() != EntryMode::DIR {
1452
+ self.delete(path).await?;
1453
+ // There may still be objects prefixed with the path in some backend, so we can't return here.
1454
+ }
1455
+ }
1456
+
1457
+ // If dir not found, it may be a prefix in object store like S3,
1458
+ // and we still need to delete objects under the prefix.
1459
+ Err(e) if e.kind() == ErrorKind::NotFound => {}
1460
+
1461
+ // Pass on any other error.
1462
+ Err(e) => return Err(e),
1463
+ };
1464
+
1465
+ let lister = self.lister_with(path).recursive(true).await?;
1466
+ self.delete_try_stream(lister).await?;
1467
+ Ok(())
1468
+ }
1469
+
1470
+ /// List entries in the parent directory that start with the specified `path`.
1471
+ ///
1472
+ /// # Notes
1473
+ ///
1474
+ /// ## Recursively List
1475
+ ///
1476
+ /// This function only reads the immediate children of the specified directory.
1477
+ /// To list all entries recursively, use `Operator::list_with("path").recursive(true)` instead.
1478
+ ///
1479
+ /// ## Streaming List
1480
+ ///
1481
+ /// This function reads all entries in the specified directory. If the directory contains many entries, this process may take a long time and use significant memory.
1482
+ ///
1483
+ /// To prevent this, consider using [`Operator::lister`] to stream the entries instead.
1484
+ ///
1485
+ /// # Examples
1486
+ ///
1487
+ /// This example will list all entries under the dir `path/to/dir/`.
1488
+ ///
1489
+ /// ```
1490
+ /// # use anyhow::Result;
1491
+ /// use opendal::EntryMode;
1492
+ /// use opendal::Operator;
1493
+ /// # async fn test(op: Operator) -> Result<()> {
1494
+ /// let mut entries = op.list("path/to/dir/").await?;
1495
+ /// for entry in entries {
1496
+ /// match entry.metadata().mode() {
1497
+ /// EntryMode::FILE => {
1498
+ /// println!("Handling file")
1499
+ /// }
1500
+ /// EntryMode::DIR => {
1501
+ /// println!("Handling dir {}", entry.path())
1502
+ /// }
1503
+ /// EntryMode::Unknown => continue,
1504
+ /// }
1505
+ /// }
1506
+ /// # Ok(())
1507
+ /// # }
1508
+ /// ```
1509
+ pub async fn list(&self, path: &str) -> Result<Vec<Entry>> {
1510
+ self.list_with(path).await
1511
+ }
1512
+
1513
+ /// List entries in the parent directory that start with the specified `path` with additional options.
1514
+ ///
1515
+ /// # Notes
1516
+ ///
1517
+ /// ## Streaming List
1518
+ ///
1519
+ /// This function reads all entries in the specified directory. If the directory contains many entries, this process may take a long time and use significant memory.
1520
+ ///
1521
+ /// To prevent this, consider using [`Operator::lister`] to stream the entries instead.
1522
+ ///
1523
+ /// # Options
1524
+ ///
1525
+ /// Visit [`options::ListOptions`] for all available options.
1526
+ ///
1527
+ /// # Examples
1528
+ ///
1529
+ /// This example will list all entries recursively under the prefix `path/to/prefix`.
1530
+ ///
1531
+ /// ```
1532
+ /// # use anyhow::Result;
1533
+ /// use opendal::EntryMode;
1534
+ /// use opendal::Operator;
1535
+ /// # async fn test(op: Operator) -> Result<()> {
1536
+ /// let mut entries = op.list_with("path/to/prefix").recursive(true).await?;
1537
+ /// for entry in entries {
1538
+ /// match entry.metadata().mode() {
1539
+ /// EntryMode::FILE => {
1540
+ /// println!("Handling file")
1541
+ /// }
1542
+ /// EntryMode::DIR => {
1543
+ /// println!("Handling dir like start a new list via meta.path()")
1544
+ /// }
1545
+ /// EntryMode::Unknown => continue,
1546
+ /// }
1547
+ /// }
1548
+ /// # Ok(())
1549
+ /// # }
1550
+ /// ```
1551
+ pub fn list_with(&self, path: &str) -> FutureList<impl Future<Output = Result<Vec<Entry>>>> {
1552
+ let path = normalize_path(path);
1553
+
1554
+ OperatorFuture::new(
1555
+ self.inner().clone(),
1556
+ path,
1557
+ options::ListOptions::default(),
1558
+ Self::list_inner,
1559
+ )
1560
+ }
1561
+
1562
+ /// List entries in the parent directory that start with the specified `path` with additional options.
1563
+ ///
1564
+ /// # Notes
1565
+ ///
1566
+ /// ## Streaming List
1567
+ ///
1568
+ /// This function reads all entries in the specified directory. If the directory contains many entries, this process may take a long time and use significant memory.
1569
+ ///
1570
+ /// To prevent this, consider using [`Operator::lister`] to stream the entries instead.
1571
+ ///
1572
+ /// # Options
1573
+ ///
1574
+ /// Visit [`options::ListOptions`] for all available options.
1575
+ ///
1576
+ /// # Examples
1577
+ ///
1578
+ /// This example will list all entries recursively under the prefix `path/to/prefix`.
1579
+ ///
1580
+ /// ```
1581
+ /// # use anyhow::Result;
1582
+ /// use opendal::options;
1583
+ /// use opendal::EntryMode;
1584
+ /// use opendal::Operator;
1585
+ /// # async fn test(op: Operator) -> Result<()> {
1586
+ /// let mut entries = op
1587
+ /// .list_options("path/to/prefix", options::ListOptions {
1588
+ /// recursive: true,
1589
+ /// ..Default::default()
1590
+ /// })
1591
+ /// .await?;
1592
+ /// for entry in entries {
1593
+ /// match entry.metadata().mode() {
1594
+ /// EntryMode::FILE => {
1595
+ /// println!("Handling file")
1596
+ /// }
1597
+ /// EntryMode::DIR => {
1598
+ /// println!("Handling dir like start a new list via meta.path()")
1599
+ /// }
1600
+ /// EntryMode::Unknown => continue,
1601
+ /// }
1602
+ /// }
1603
+ /// # Ok(())
1604
+ /// # }
1605
+ /// ```
1606
+ pub async fn list_options(&self, path: &str, opts: options::ListOptions) -> Result<Vec<Entry>> {
1607
+ let path = normalize_path(path);
1608
+ Self::list_inner(self.inner().clone(), path, opts).await
1609
+ }
1610
+
1611
+ #[inline]
1612
+ async fn list_inner(
1613
+ acc: Accessor,
1614
+ path: String,
1615
+ opts: options::ListOptions,
1616
+ ) -> Result<Vec<Entry>> {
1617
+ let args = opts.into();
1618
+ let lister = Lister::create(acc, &path, args).await?;
1619
+ lister.try_collect().await
1620
+ }
1621
+
1622
+ /// Create a new lister to list entries that starts with given `path` in parent dir.
1623
+ ///
1624
+ /// # Notes
1625
+ ///
1626
+ /// ## Recursively list
1627
+ ///
1628
+ /// This function only reads the immediate children of the specified directory.
1629
+ /// To retrieve all entries recursively, use [`Operator::lister_with`] with `recursive(true)` instead.
1630
+ ///
1631
+ /// # Examples
1632
+ ///
1633
+ /// ```
1634
+ /// # use anyhow::Result;
1635
+ /// # use futures::io;
1636
+ /// use futures::TryStreamExt;
1637
+ /// use opendal::EntryMode;
1638
+ /// use opendal::Operator;
1639
+ /// # async fn test(op: Operator) -> Result<()> {
1640
+ /// let mut ds = op.lister("path/to/dir/").await?;
1641
+ /// while let Some(mut de) = ds.try_next().await? {
1642
+ /// match de.metadata().mode() {
1643
+ /// EntryMode::FILE => {
1644
+ /// println!("Handling file")
1645
+ /// }
1646
+ /// EntryMode::DIR => {
1647
+ /// println!("Handling dir like start a new list via meta.path()")
1648
+ /// }
1649
+ /// EntryMode::Unknown => continue,
1650
+ /// }
1651
+ /// }
1652
+ /// # Ok(())
1653
+ /// # }
1654
+ /// ```
1655
+ pub async fn lister(&self, path: &str) -> Result<Lister> {
1656
+ self.lister_with(path).await
1657
+ }
1658
+
1659
+ /// Create a new lister to list entries that starts with given `path` in parent dir with additional options.
1660
+ ///
1661
+ /// # Options
1662
+ ///
1663
+ /// Visit [`options::ListOptions`] for all available options.
1664
+ ///
1665
+ /// # Examples
1666
+ ///
1667
+ /// ## List all files recursively
1668
+ ///
1669
+ /// ```
1670
+ /// # use anyhow::Result;
1671
+ /// use futures::TryStreamExt;
1672
+ /// use opendal::EntryMode;
1673
+ /// use opendal::Operator;
1674
+ /// # async fn test(op: Operator) -> Result<()> {
1675
+ /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
1676
+ /// while let Some(mut entry) = lister.try_next().await? {
1677
+ /// match entry.metadata().mode() {
1678
+ /// EntryMode::FILE => {
1679
+ /// println!("Handling file {}", entry.path())
1680
+ /// }
1681
+ /// EntryMode::DIR => {
1682
+ /// println!("Handling dir {}", entry.path())
1683
+ /// }
1684
+ /// EntryMode::Unknown => continue,
1685
+ /// }
1686
+ /// }
1687
+ /// # Ok(())
1688
+ /// # }
1689
+ /// ```
1690
+ pub fn lister_with(&self, path: &str) -> FutureLister<impl Future<Output = Result<Lister>>> {
1691
+ let path = normalize_path(path);
1692
+
1693
+ OperatorFuture::new(
1694
+ self.inner().clone(),
1695
+ path,
1696
+ options::ListOptions::default(),
1697
+ Self::lister_inner,
1698
+ )
1699
+ }
1700
+
1701
+ /// Create a new lister to list entries that starts with given `path` in parent dir with additional options.
1702
+ ///
1703
+ /// # Examples
1704
+ ///
1705
+ /// ## List all files recursively
1706
+ ///
1707
+ /// ```
1708
+ /// # use anyhow::Result;
1709
+ /// use futures::TryStreamExt;
1710
+ /// use opendal::options;
1711
+ /// use opendal::EntryMode;
1712
+ /// use opendal::Operator;
1713
+ /// # async fn test(op: Operator) -> Result<()> {
1714
+ /// let mut lister = op
1715
+ /// .lister_options("path/to/dir/", options::ListOptions {
1716
+ /// recursive: true,
1717
+ /// ..Default::default()
1718
+ /// })
1719
+ /// .await?;
1720
+ /// while let Some(mut entry) = lister.try_next().await? {
1721
+ /// match entry.metadata().mode() {
1722
+ /// EntryMode::FILE => {
1723
+ /// println!("Handling file {}", entry.path())
1724
+ /// }
1725
+ /// EntryMode::DIR => {
1726
+ /// println!("Handling dir {}", entry.path())
1727
+ /// }
1728
+ /// EntryMode::Unknown => continue,
1729
+ /// }
1730
+ /// }
1731
+ /// # Ok(())
1732
+ /// # }
1733
+ /// ```
1734
+ pub async fn lister_options(&self, path: &str, opts: options::ListOptions) -> Result<Lister> {
1735
+ let path = normalize_path(path);
1736
+ Self::lister_inner(self.inner().clone(), path, opts).await
1737
+ }
1738
+
1739
+ #[inline]
1740
+ async fn lister_inner(
1741
+ acc: Accessor,
1742
+ path: String,
1743
+ opts: options::ListOptions,
1744
+ ) -> Result<Lister> {
1745
+ let args = opts.into();
1746
+ let lister = Lister::create(acc, &path, args).await?;
1747
+ Ok(lister)
1748
+ }
1749
+ }
1750
+
1751
+ /// Operator presign API.
1752
+ impl Operator {
1753
+ /// Presign an operation for stat(head).
1754
+ ///
1755
+ /// # Example
1756
+ ///
1757
+ /// ```
1758
+ /// use anyhow::Result;
1759
+ /// use futures::io;
1760
+ /// use opendal::Operator;
1761
+ /// use std::time::Duration;
1762
+ ///
1763
+ /// async fn test(op: Operator) -> Result<()> {
1764
+ /// let signed_req = op.presign_stat("test",Duration::from_secs(3600)).await?;
1765
+ /// let req = http::Request::builder()
1766
+ /// .method(signed_req.method())
1767
+ /// .uri(signed_req.uri())
1768
+ /// .body(())?;
1769
+ ///
1770
+ /// # Ok(())
1771
+ /// # }
1772
+ /// ```
1773
+ pub async fn presign_stat(&self, path: &str, expire: Duration) -> Result<PresignedRequest> {
1774
+ let path = normalize_path(path);
1775
+
1776
+ let op = OpPresign::new(OpStat::new(), expire);
1777
+
1778
+ let rp = self.inner().presign(&path, op).await?;
1779
+ Ok(rp.into_presigned_request())
1780
+ }
1781
+
1782
+ /// Presign an operation for stat(head).
1783
+ ///
1784
+ /// # Example
1785
+ ///
1786
+ /// ```
1787
+ /// use anyhow::Result;
1788
+ /// use futures::io;
1789
+ /// use opendal::Operator;
1790
+ /// use std::time::Duration;
1791
+ ///
1792
+ /// async fn test(op: Operator) -> Result<()> {
1793
+ /// let signed_req = op.presign_stat_with("test",Duration::from_secs(3600)).override_content_disposition("attachment; filename=\"othertext.txt\"").await?;
1794
+ /// # Ok(())
1795
+ /// # }
1796
+ /// ```
1797
+ pub fn presign_stat_with(
1798
+ &self,
1799
+ path: &str,
1800
+ expire: Duration,
1801
+ ) -> FuturePresignStat<impl Future<Output = Result<PresignedRequest>>> {
1802
+ let path = normalize_path(path);
1803
+
1804
+ OperatorFuture::new(
1805
+ self.inner().clone(),
1806
+ path,
1807
+ (options::StatOptions::default(), expire),
1808
+ Self::presign_stat_inner,
1809
+ )
1810
+ }
1811
+
1812
+ /// Presign an operation for stat(head) with additional options.
1813
+ ///
1814
+ /// # Options
1815
+ ///
1816
+ /// Visit [`options::StatOptions`] for all available options.
1817
+ ///
1818
+ /// # Example
1819
+ ///
1820
+ /// ```
1821
+ /// use anyhow::Result;
1822
+ /// use opendal::Operator;
1823
+ /// use opendal::options;
1824
+ /// use std::time::Duration;
1825
+ ///
1826
+ /// async fn test(op: Operator) -> Result<()> {
1827
+ /// let signed_req = op.presign_stat_options(
1828
+ /// "test",
1829
+ /// Duration::from_secs(3600),
1830
+ /// options::StatOptions {
1831
+ /// if_match: Some("<etag>".to_string()),
1832
+ /// ..Default::default()
1833
+ /// }
1834
+ /// ).await?;
1835
+ /// let req = http::Request::builder()
1836
+ /// .method(signed_req.method())
1837
+ /// .uri(signed_req.uri())
1838
+ /// .body(())?;
1839
+ ///
1840
+ /// # Ok(())
1841
+ /// # }
1842
+ /// ```
1843
+ pub async fn presign_stat_options(
1844
+ &self,
1845
+ path: &str,
1846
+ expire: Duration,
1847
+ opts: options::StatOptions,
1848
+ ) -> Result<PresignedRequest> {
1849
+ let path = normalize_path(path);
1850
+ Self::presign_stat_inner(self.inner().clone(), path, (opts, expire)).await
1851
+ }
1852
+
1853
+ #[inline]
1854
+ async fn presign_stat_inner(
1855
+ acc: Accessor,
1856
+ path: String,
1857
+ (opts, expire): (options::StatOptions, Duration),
1858
+ ) -> Result<PresignedRequest> {
1859
+ let op = OpPresign::new(OpStat::from(opts), expire);
1860
+ let rp = acc.presign(&path, op).await?;
1861
+ Ok(rp.into_presigned_request())
1862
+ }
1863
+
1864
+ /// Presign an operation for read.
1865
+ ///
1866
+ /// # Notes
1867
+ ///
1868
+ /// ## Extra Options
1869
+ ///
1870
+ /// `presign_read` is a wrapper of [`Self::presign_read_with`] without any options. To use
1871
+ /// extra options like `override_content_disposition`, please use [`Self::presign_read_with`] or
1872
+ /// [`Self::presign_read_options] instead.
1873
+ ///
1874
+ /// # Example
1875
+ ///
1876
+ /// ```
1877
+ /// use anyhow::Result;
1878
+ /// use futures::io;
1879
+ /// use opendal::Operator;
1880
+ /// use std::time::Duration;
1881
+ ///
1882
+ /// async fn test(op: Operator) -> Result<()> {
1883
+ /// let signed_req = op.presign_read("test.txt", Duration::from_secs(3600)).await?;
1884
+ /// # Ok(())
1885
+ /// # }
1886
+ /// ```
1887
+ ///
1888
+ /// - `signed_req.method()`: `GET`
1889
+ /// - `signed_req.uri()`: `https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>`
1890
+ /// - `signed_req.headers()`: `{ "host": "s3.amazonaws.com" }`
1891
+ ///
1892
+ /// We can download this file via `curl` or other tools without credentials:
1893
+ ///
1894
+ /// ```shell
1895
+ /// curl "https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>" -O /tmp/test.txt
1896
+ /// ```
1897
+ pub async fn presign_read(&self, path: &str, expire: Duration) -> Result<PresignedRequest> {
1898
+ let path = normalize_path(path);
1899
+
1900
+ let op = OpPresign::new(OpRead::new(), expire);
1901
+
1902
+ let rp = self.inner().presign(&path, op).await?;
1903
+ Ok(rp.into_presigned_request())
1904
+ }
1905
+
1906
+ /// Presign an operation for read with extra options.
1907
+ ///
1908
+ /// # Options
1909
+ ///
1910
+ /// Visit [`options::ReadOptions`] for all available options.
1911
+ ///
1912
+ /// # Example
1913
+ ///
1914
+ /// ```
1915
+ /// use std::time::Duration;
1916
+ ///
1917
+ /// use anyhow::Result;
1918
+ /// use futures::io;
1919
+ /// use opendal::Operator;
1920
+ ///
1921
+ /// async fn test(op: Operator) -> Result<()> {
1922
+ /// let signed_req = op
1923
+ /// .presign_read_with("test.txt", Duration::from_secs(3600))
1924
+ /// .override_content_type("text/plain")
1925
+ /// .await?;
1926
+ /// Ok(())
1927
+ /// }
1928
+ /// ```
1929
+ pub fn presign_read_with(
1930
+ &self,
1931
+ path: &str,
1932
+ expire: Duration,
1933
+ ) -> FuturePresignRead<impl Future<Output = Result<PresignedRequest>>> {
1934
+ let path = normalize_path(path);
1935
+
1936
+ OperatorFuture::new(
1937
+ self.inner().clone(),
1938
+ path,
1939
+ (options::ReadOptions::default(), expire),
1940
+ Self::presign_read_inner,
1941
+ )
1942
+ }
1943
+
1944
+ /// Presign an operation for read with additional options.
1945
+ ///
1946
+ /// # Options
1947
+ ///
1948
+ /// Visit [`options::ReadOptions`] for all available options.
1949
+ ///
1950
+ /// # Example
1951
+ ///
1952
+ /// ```
1953
+ /// use anyhow::Result;
1954
+ /// use opendal::Operator;
1955
+ /// use opendal::options;
1956
+ /// use std::time::Duration;
1957
+ ///
1958
+ /// async fn test(op: Operator) -> Result<()> {
1959
+ /// let signed_req = op.presign_read_options(
1960
+ /// "file",
1961
+ /// Duration::from_secs(3600),
1962
+ /// options::ReadOptions {
1963
+ /// override_content_disposition: Some("attachment; filename=\"othertext.txt\"".to_string()),
1964
+ /// ..Default::default()
1965
+ /// }
1966
+ /// ).await?;
1967
+ /// let req = http::Request::builder()
1968
+ /// .method(signed_req.method())
1969
+ /// .uri(signed_req.uri())
1970
+ /// .body(())?;
1971
+ ///
1972
+ /// # Ok(())
1973
+ /// # }
1974
+ /// ```
1975
+ pub async fn presign_read_options(
1976
+ &self,
1977
+ path: &str,
1978
+ expire: Duration,
1979
+ opts: options::ReadOptions,
1980
+ ) -> Result<PresignedRequest> {
1981
+ let path = normalize_path(path);
1982
+ Self::presign_read_inner(self.inner().clone(), path, (opts, expire)).await
1983
+ }
1984
+
1985
+ #[inline]
1986
+ async fn presign_read_inner(
1987
+ acc: Accessor,
1988
+ path: String,
1989
+ (opts, expire): (options::ReadOptions, Duration),
1990
+ ) -> Result<PresignedRequest> {
1991
+ let (op_read, _) = opts.into();
1992
+ let op = OpPresign::new(op_read, expire);
1993
+ let rp = acc.presign(&path, op).await?;
1994
+ Ok(rp.into_presigned_request())
1995
+ }
1996
+
1997
+ /// Presign an operation for write.
1998
+ ///
1999
+ /// # Notes
2000
+ ///
2001
+ /// ## Extra Options
2002
+ ///
2003
+ /// `presign_write` is a wrapper of [`Self::presign_write_with`] without any options. To use
2004
+ /// extra options like `content_type`, please use [`Self::presign_write_with`] or
2005
+ /// [`Self::presign_write_options`] instead.
2006
+ ///
2007
+ /// # Example
2008
+ ///
2009
+ /// ```
2010
+ /// use std::time::Duration;
2011
+ ///
2012
+ /// use anyhow::Result;
2013
+ /// use opendal::Operator;
2014
+ ///
2015
+ /// async fn test(op: Operator) -> Result<()> {
2016
+ /// let signed_req = op
2017
+ /// .presign_write("test.txt", Duration::from_secs(3600))
2018
+ /// .await?;
2019
+ /// Ok(())
2020
+ /// }
2021
+ /// ```
2022
+ ///
2023
+ /// - `signed_req.method()`: `PUT`
2024
+ /// - `signed_req.uri()`: `https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>`
2025
+ /// - `signed_req.headers()`: `{ "host": "s3.amazonaws.com" }`
2026
+ ///
2027
+ /// We can upload file as this file via `curl` or other tools without credential:
2028
+ ///
2029
+ /// ```shell
2030
+ /// curl -X PUT "https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>" -d "Hello, World!"
2031
+ /// ```
2032
+ pub async fn presign_write(&self, path: &str, expire: Duration) -> Result<PresignedRequest> {
2033
+ self.presign_write_with(path, expire).await
2034
+ }
2035
+
2036
+ /// Presign an operation for write with extra options.
2037
+ ///
2038
+ /// # Options
2039
+ ///
2040
+ /// Visit [`options::WriteOptions`] for all available options.
2041
+ ///
2042
+ /// # Example
2043
+ ///
2044
+ /// ```
2045
+ /// use std::time::Duration;
2046
+ ///
2047
+ /// use anyhow::Result;
2048
+ /// use opendal::Operator;
2049
+ ///
2050
+ /// async fn test(op: Operator) -> Result<()> {
2051
+ /// let signed_req = op
2052
+ /// .presign_write_with("test", Duration::from_secs(3600))
2053
+ /// .cache_control("no-store")
2054
+ /// .await?;
2055
+ /// let req = http::Request::builder()
2056
+ /// .method(signed_req.method())
2057
+ /// .uri(signed_req.uri())
2058
+ /// .body(())?;
2059
+ ///
2060
+ /// Ok(())
2061
+ /// }
2062
+ /// ```
2063
+ pub fn presign_write_with(
2064
+ &self,
2065
+ path: &str,
2066
+ expire: Duration,
2067
+ ) -> FuturePresignWrite<impl Future<Output = Result<PresignedRequest>>> {
2068
+ let path = normalize_path(path);
2069
+
2070
+ OperatorFuture::new(
2071
+ self.inner().clone(),
2072
+ path,
2073
+ (options::WriteOptions::default(), expire),
2074
+ Self::presign_write_inner,
2075
+ )
2076
+ }
2077
+
2078
+ /// Presign an operation for write with additional options.
2079
+ ///
2080
+ /// # Options
2081
+ ///
2082
+ /// Check [`options::WriteOptions`] for all available options.
2083
+ ///
2084
+ /// # Example
2085
+ ///
2086
+ /// ```
2087
+ /// use anyhow::Result;
2088
+ /// use opendal::Operator;
2089
+ /// use opendal::options;
2090
+ /// use std::time::Duration;
2091
+ ///
2092
+ /// async fn test(op: Operator) -> Result<()> {
2093
+ /// let signed_req = op.presign_write_options(
2094
+ /// "file",
2095
+ /// Duration::from_secs(3600),
2096
+ /// options::WriteOptions {
2097
+ /// content_type: Some("application/json".to_string()),
2098
+ /// cache_control: Some("max-age=3600".to_string()),
2099
+ /// if_not_exists: true,
2100
+ /// ..Default::default()
2101
+ /// }
2102
+ /// ).await?;
2103
+ /// let req = http::Request::builder()
2104
+ /// .method(signed_req.method())
2105
+ /// .uri(signed_req.uri())
2106
+ /// .body(())?;
2107
+ ///
2108
+ /// # Ok(())
2109
+ /// # }
2110
+ /// ```
2111
+ pub async fn presign_write_options(
2112
+ &self,
2113
+ path: &str,
2114
+ expire: Duration,
2115
+ opts: options::WriteOptions,
2116
+ ) -> Result<PresignedRequest> {
2117
+ let path = normalize_path(path);
2118
+ Self::presign_write_inner(self.inner().clone(), path, (opts, expire)).await
2119
+ }
2120
+
2121
+ #[inline]
2122
+ async fn presign_write_inner(
2123
+ acc: Accessor,
2124
+ path: String,
2125
+ (opts, expire): (options::WriteOptions, Duration),
2126
+ ) -> Result<PresignedRequest> {
2127
+ let (op_write, _) = opts.into();
2128
+ let op = OpPresign::new(op_write, expire);
2129
+ let rp = acc.presign(&path, op).await?;
2130
+ Ok(rp.into_presigned_request())
2131
+ }
2132
+
2133
+ /// Presign an operation for delete.
2134
+ ///
2135
+ /// # Notes
2136
+ ///
2137
+ /// ## Extra Options
2138
+ ///
2139
+ /// `presign_delete` is a wrapper of [`Self::presign_delete_with`] without any options.
2140
+ ///
2141
+ /// # Example
2142
+ ///
2143
+ /// ```
2144
+ /// use std::time::Duration;
2145
+ ///
2146
+ /// use anyhow::Result;
2147
+ /// use opendal::Operator;
2148
+ ///
2149
+ /// async fn test(op: Operator) -> Result<()> {
2150
+ /// let signed_req = op
2151
+ /// .presign_delete("test.txt", Duration::from_secs(3600))
2152
+ /// .await?;
2153
+ /// Ok(())
2154
+ /// }
2155
+ /// ```
2156
+ ///
2157
+ /// - `signed_req.method()`: `DELETE`
2158
+ /// - `signed_req.uri()`: `https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>`
2159
+ /// - `signed_req.headers()`: `{ "host": "s3.amazonaws.com" }`
2160
+ ///
2161
+ /// We can delete file as this file via `curl` or other tools without credential:
2162
+ ///
2163
+ /// ```shell
2164
+ /// curl -X DELETE "https://s3.amazonaws.com/examplebucket/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=access_key_id/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=<signature-value>"
2165
+ /// ```
2166
+ pub async fn presign_delete(&self, path: &str, expire: Duration) -> Result<PresignedRequest> {
2167
+ self.presign_delete_with(path, expire).await
2168
+ }
2169
+
2170
+ /// Presign an operation for delete without extra options.
2171
+ pub fn presign_delete_with(
2172
+ &self,
2173
+ path: &str,
2174
+ expire: Duration,
2175
+ ) -> FuturePresignDelete<impl Future<Output = Result<PresignedRequest>>> {
2176
+ let path = normalize_path(path);
2177
+
2178
+ OperatorFuture::new(
2179
+ self.inner().clone(),
2180
+ path,
2181
+ (options::DeleteOptions::default(), expire),
2182
+ Self::presign_delete_inner,
2183
+ )
2184
+ }
2185
+
2186
+ /// Presign an operation for delete with additional options.
2187
+ ///
2188
+ /// # Options
2189
+ ///
2190
+ /// Visit [`options::DeleteOptions`] for all available options.
2191
+ ///
2192
+ /// # Example
2193
+ ///
2194
+ /// ```
2195
+ /// use anyhow::Result;
2196
+ /// use opendal::Operator;
2197
+ /// use opendal::options;
2198
+ /// use std::time::Duration;
2199
+ ///
2200
+ /// async fn test(op: Operator) -> Result<()> {
2201
+ /// let signed_req = op.presign_delete_options(
2202
+ /// "path/to/file",
2203
+ /// Duration::from_secs(3600),
2204
+ /// options::DeleteOptions {
2205
+ /// ..Default::default()
2206
+ /// }
2207
+ /// ).await?;
2208
+ /// let req = http::Request::builder()
2209
+ /// .method(signed_req.method())
2210
+ /// .uri(signed_req.uri())
2211
+ /// .body(())?;
2212
+ ///
2213
+ /// # Ok(())
2214
+ /// # }
2215
+ /// ```
2216
+ pub async fn presign_delete_options(
2217
+ &self,
2218
+ path: &str,
2219
+ expire: Duration,
2220
+ opts: options::DeleteOptions,
2221
+ ) -> Result<PresignedRequest> {
2222
+ let path = normalize_path(path);
2223
+ Self::presign_delete_inner(self.inner().clone(), path, (opts, expire)).await
2224
+ }
2225
+
2226
+ #[inline]
2227
+ async fn presign_delete_inner(
2228
+ acc: Accessor,
2229
+ path: String,
2230
+ (opts, expire): (options::DeleteOptions, Duration),
2231
+ ) -> Result<PresignedRequest> {
2232
+ let op = OpPresign::new(OpDelete::from(opts), expire);
2233
+ let rp = acc.presign(&path, op).await?;
2234
+ Ok(rp.into_presigned_request())
2235
+ }
2236
+ }