opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -75,7 +75,7 @@ impl oio::PageList for SwiftLister {
75
75
  let status_code = response.status();
76
76
 
77
77
  if !status_code.is_success() {
78
- let error = parse_error(response);
78
+ let error = parse_error(ErrorContext::new(ServiceOperation("ListObjects")), response);
79
79
  return Err(error);
80
80
  }
81
81
 
@@ -101,7 +101,7 @@ impl oio::PageList for SwiftLister {
101
101
  if path.is_empty() {
102
102
  path = "/".to_string();
103
103
  }
104
- let meta = Metadata::new(EntryMode::DIR);
104
+ let meta = MetadataBuilder::dir().build();
105
105
  oio::Entry::with(path, meta)
106
106
  }
107
107
  ListOpResponse::FileInfo {
@@ -115,9 +115,12 @@ impl oio::PageList for SwiftLister {
115
115
  if path.is_empty() {
116
116
  path = "/".to_string();
117
117
  }
118
- let mut meta = Metadata::new(EntryMode::from_path(path.as_str()));
119
- meta.set_content_length(bytes);
120
- meta.set_content_md5(hash.as_str());
118
+ let mut meta = if path.ends_with('/') {
119
+ MetadataBuilder::dir()
120
+ } else {
121
+ MetadataBuilder::file(bytes)
122
+ };
123
+ meta.content_md5(hash.as_str());
121
124
 
122
125
  // OpenStack Swift returns time without 'Z' at the end,
123
126
  // which causes an error in parse_datetime_from_rfc3339.
@@ -125,13 +128,13 @@ impl oio::PageList for SwiftLister {
125
128
  if !last_modified.ends_with('Z') {
126
129
  last_modified.push('Z');
127
130
  }
128
- meta.set_last_modified(last_modified.parse::<Timestamp>()?);
131
+ meta.last_modified(last_modified.parse::<Timestamp>()?);
129
132
 
130
133
  if let Some(content_type) = content_type {
131
- meta.set_content_type(content_type.as_str());
134
+ meta.content_type(content_type.as_str());
132
135
  }
133
136
 
134
- oio::Entry::with(path, meta)
137
+ oio::Entry::with(path, meta.build())
135
138
  }
136
139
  };
137
140
  ctx.entries.push_back(entry);
@@ -16,7 +16,7 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use super::core::parse_error;
19
+ use super::core::{ErrorContext, parse_error};
20
20
  use http::Response;
21
21
  use http::StatusCode;
22
22
  use opendal_core::raw::*;
@@ -66,7 +66,10 @@ impl oio::StreamRead for SwiftReader {
66
66
  _ => {
67
67
  let (part, mut body) = resp.into_parts();
68
68
  let buf = body.to_buffer().await?;
69
- return Err(parse_error(Response::from_parts(part, buf)));
69
+ return Err(parse_error(
70
+ ErrorContext::new(ServiceOperation("GetObject")),
71
+ Response::from_parts(part, buf),
72
+ ));
70
73
  }
71
74
  };
72
75
 
@@ -20,8 +20,8 @@ use std::sync::Arc;
20
20
  use http::StatusCode;
21
21
 
22
22
  use super::core::SloManifestEntry;
23
- use super::core::SwiftCore;
24
23
  use super::core::parse_error;
24
+ use super::core::{ErrorContext, SwiftCore};
25
25
  use opendal_core::raw::*;
26
26
  use opendal_core::*;
27
27
 
@@ -43,17 +43,17 @@ impl SwiftWriter {
43
43
  }
44
44
 
45
45
  fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
46
- let mut metadata = Metadata::default();
46
+ let mut metadata = MetadataBuilder::unknown();
47
47
 
48
48
  if let Some(etag) = parse_etag(headers)? {
49
- metadata.set_etag(etag);
49
+ metadata.etag(etag);
50
50
  }
51
51
 
52
52
  if let Some(last_modified) = parse_last_modified(headers)? {
53
- metadata.set_last_modified(last_modified);
53
+ metadata.last_modified(last_modified);
54
54
  }
55
55
 
56
- Ok(metadata)
56
+ Ok(metadata.build())
57
57
  }
58
58
  }
59
59
 
@@ -71,7 +71,10 @@ impl oio::MultipartWrite for SwiftWriter {
71
71
  let metadata = SwiftWriter::parse_metadata(resp.headers())?;
72
72
  Ok(metadata)
73
73
  }
74
- _ => Err(parse_error(resp)),
74
+ _ => Err(parse_error(
75
+ ErrorContext::new(ServiceOperation("CreateObject")),
76
+ resp,
77
+ )),
75
78
  }
76
79
  }
77
80
 
@@ -113,7 +116,10 @@ impl oio::MultipartWrite for SwiftWriter {
113
116
  size: Some(size),
114
117
  })
115
118
  }
116
- _ => Err(parse_error(resp)),
119
+ _ => Err(parse_error(
120
+ ErrorContext::new(ServiceOperation("UploadSegment")),
121
+ resp,
122
+ )),
117
123
  }
118
124
  }
119
125
 
@@ -148,7 +154,10 @@ impl oio::MultipartWrite for SwiftWriter {
148
154
  let metadata = SwiftWriter::parse_metadata(resp.headers())?;
149
155
  Ok(metadata)
150
156
  }
151
- _ => Err(parse_error(resp)),
157
+ _ => Err(parse_error(
158
+ ErrorContext::new(ServiceOperation("UploadManifest")),
159
+ resp,
160
+ )),
152
161
  }
153
162
  }
154
163
 
@@ -32,8 +32,8 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- mea = { workspace = true }
36
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
35
+ asyncband = { workspace = true }
36
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
37
37
  serde = { workspace = true, features = ["derive"] }
38
38
  tikv-client = { version = "0.4.0", default-features = false }
39
39
 
@@ -18,7 +18,7 @@
18
18
  use std::fmt::Debug;
19
19
  use std::sync::Arc;
20
20
 
21
- use mea::once::OnceCell;
21
+ use asyncband::once::OnceCell;
22
22
 
23
23
  use super::TIKV_SCHEME;
24
24
  use super::config::TikvConfig;
@@ -149,6 +149,7 @@ impl Service for TikvBackend {
149
149
  type Lister = ();
150
150
  type Deleter = oio::OneShotDeleter<TikvDeleter>;
151
151
  type Copier = ();
152
+ type Composer = ();
152
153
 
153
154
  fn info(&self) -> ServiceInfo {
154
155
  self.info.clone()
@@ -174,13 +175,14 @@ impl Service for TikvBackend {
174
175
  let p = build_abs_path(&self.root, path);
175
176
 
176
177
  if p == build_abs_path(&self.root, "") {
177
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
178
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
178
179
  } else {
179
180
  let bs = self.core.get(&p).await?;
180
181
  match bs {
181
- Some(bs) => Ok(RpStat::new(
182
- Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
183
- )),
182
+ Some(bs) => Ok(RpStat::new({
183
+ let metadata = MetadataBuilder::file(bs.len() as u64);
184
+ metadata.build()
185
+ })),
184
186
  None => Err(Error::new(ErrorKind::NotFound, "kv not found in tikv")),
185
187
  }
186
188
  }
@@ -230,7 +232,6 @@ impl Service for TikvBackend {
230
232
  _from: &str,
231
233
  _to: &str,
232
234
  _args: OpCopy,
233
- _opts: OpCopier,
234
235
  ) -> Result<Self::Copier> {
235
236
  Err(Error::new(
236
237
  ErrorKind::Unsupported,
@@ -17,13 +17,25 @@
17
17
 
18
18
  use std::fmt::Debug;
19
19
 
20
- use mea::once::OnceCell;
20
+ use asyncband::once::OnceCell;
21
21
  use tikv_client::Config;
22
22
  use tikv_client::RawClient;
23
23
 
24
24
  use super::TIKV_SCHEME;
25
+ use opendal_core::raw::ServiceOperation;
25
26
  use opendal_core::*;
26
27
 
28
+ #[derive(Clone, Copy, Debug)]
29
+ struct ErrorContext {
30
+ service_operation: ServiceOperation,
31
+ }
32
+
33
+ impl ErrorContext {
34
+ const fn new(service_operation: ServiceOperation) -> Self {
35
+ Self { service_operation }
36
+ }
37
+ }
38
+
27
39
  /// TikvCore holds the configuration and client for interacting with TiKV.
28
40
  #[derive(Clone)]
29
41
  pub struct TikvCore {
@@ -49,9 +61,9 @@ impl TikvCore {
49
61
  self.client
50
62
  .get_or_try_init(|| async {
51
63
  if self.insecure {
52
- return RawClient::new(self.endpoints.clone())
53
- .await
54
- .map_err(parse_tikv_config_error);
64
+ return RawClient::new(self.endpoints.clone()).await.map_err(|err| {
65
+ parse_error(ErrorContext::new(ServiceOperation("Connect")), err)
66
+ });
55
67
  }
56
68
 
57
69
  if let Some(ca_path) = self.ca_path.as_ref()
@@ -61,7 +73,9 @@ impl TikvCore {
61
73
  let config = Config::default().with_security(ca_path, cert_path, key_path);
62
74
  return RawClient::new_with_config(self.endpoints.clone(), config)
63
75
  .await
64
- .map_err(parse_tikv_config_error);
76
+ .map_err(|err| {
77
+ parse_error(ErrorContext::new(ServiceOperation("Connect")), err)
78
+ });
65
79
  }
66
80
 
67
81
  Err(
@@ -79,7 +93,7 @@ impl TikvCore {
79
93
  .await?
80
94
  .get(path.to_owned())
81
95
  .await
82
- .map_err(parse_tikv_error)?;
96
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Get")), err))?;
83
97
  Ok(result.map(Buffer::from))
84
98
  }
85
99
 
@@ -88,7 +102,7 @@ impl TikvCore {
88
102
  .await?
89
103
  .put(path.to_owned(), value.to_vec())
90
104
  .await
91
- .map_err(parse_tikv_error)
105
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Put")), err))
92
106
  }
93
107
 
94
108
  pub async fn delete(&self, path: &str) -> Result<()> {
@@ -96,16 +110,19 @@ impl TikvCore {
96
110
  .await?
97
111
  .delete(path.to_owned())
98
112
  .await
99
- .map_err(parse_tikv_error)
113
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Delete")), err))
100
114
  }
101
115
  }
102
116
 
103
- fn parse_tikv_error(e: tikv_client::Error) -> Error {
104
- Error::new(ErrorKind::Unexpected, "error from tikv").set_source(e)
105
- }
117
+ fn parse_error(ctx: ErrorContext, err: tikv_client::Error) -> Error {
118
+ let (kind, message) = if ctx.service_operation.0 == "Connect" {
119
+ (ErrorKind::ConfigInvalid, "invalid configuration")
120
+ } else {
121
+ (ErrorKind::Unexpected, "error from tikv")
122
+ };
106
123
 
107
- fn parse_tikv_config_error(e: tikv_client::Error) -> Error {
108
- Error::new(ErrorKind::ConfigInvalid, "invalid configuration")
124
+ Error::new(kind, message)
109
125
  .with_context("service", TIKV_SCHEME)
110
- .set_source(e)
126
+ .with_context("service_operation", ctx.service_operation.0)
127
+ .set_source(err)
111
128
  }
@@ -45,7 +45,10 @@ impl oio::StreamRead for TikvReader {
45
45
  None => return Err(Error::new(ErrorKind::NotFound, "kv not found in tikv")),
46
46
  };
47
47
  let content = bs.slice(range.to_content_range(bs.len())?);
48
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
48
+ let metadata = {
49
+ let metadata = MetadataBuilder::file(bs.len() as u64);
50
+ metadata.build()
51
+ };
49
52
  Ok((
50
53
  RpRead::new(metadata),
51
54
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -48,8 +48,12 @@ impl oio::Write for TikvWriter {
48
48
  let length = buf.len() as u64;
49
49
  self.core.set(&self.path, buf).await?;
50
50
 
51
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
52
- Ok(meta)
51
+ let metadata = if self.path.ends_with('/') {
52
+ MetadataBuilder::dir()
53
+ } else {
54
+ MetadataBuilder::file(length)
55
+ };
56
+ Ok(metadata.build())
53
57
  }
54
58
 
55
59
  async fn abort(&mut self) -> Result<()> {
@@ -34,12 +34,12 @@ all-features = true
34
34
  [dependencies]
35
35
  bytes = { workspace = true }
36
36
  http = { workspace = true }
37
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false, features = [
37
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false, features = [
38
38
  "reqsign",
39
39
  ] }
40
40
  quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
41
41
  reqsign-core = { version = "3.2.0", default-features = false }
42
42
  reqsign-file-read-tokio = { version = "3.0.3", default-features = false }
43
- reqsign-volcengine-tos = "3.0.3"
43
+ reqsign-volcengine-tos = "3.1.1"
44
44
  serde = { workspace = true, features = ["derive"] }
45
45
  serde_json = { workspace = true }
@@ -33,7 +33,7 @@ use crate::writer::TosWriter;
33
33
  use http::StatusCode;
34
34
  use opendal_core::OperationContext;
35
35
  use opendal_core::raw::*;
36
- use opendal_core::{Builder, Capability, EntryMode, Error, ErrorKind, Result};
36
+ use opendal_core::{Buffer, Builder, Capability, Error, ErrorKind, Result};
37
37
  use reqsign_core::{Context, OsEnv, ProvideCredentialChain, Signer};
38
38
  use reqsign_file_read_tokio::TokioFileRead;
39
39
  use reqsign_volcengine_tos::{EnvCredentialProvider, RequestSigner, StaticCredentialProvider};
@@ -215,6 +215,12 @@ impl Builder for TosBuilder {
215
215
  list_with_versions: true,
216
216
  list_with_deleted: true,
217
217
 
218
+ presign: true,
219
+ presign_stat: true,
220
+ presign_read: true,
221
+ presign_write: true,
222
+ presign_delete: true,
223
+
218
224
  stat: true,
219
225
  stat_with_if_match: true,
220
226
  stat_with_if_none_match: true,
@@ -264,6 +270,7 @@ impl Service for TosBackend {
264
270
  type Lister = TosListers;
265
271
  type Deleter = oio::BatchDeleter<TosDeleter>;
266
272
  type Copier = TosCopiers;
273
+ type Composer = ();
267
274
 
268
275
  fn info(&self) -> ServiceInfo {
269
276
  self.core.info.clone()
@@ -293,18 +300,18 @@ impl Service for TosBackend {
293
300
  match status {
294
301
  StatusCode::OK => {
295
302
  let headers = resp.headers();
296
- let mut meta = tos_parse_into_metadata(path, headers)?;
303
+ let mut meta = tos_parse_into_metadata(path, headers)?.into_builder();
297
304
 
298
305
  let user_meta = parse_prefixed_headers(headers, X_TOS_META_PREFIX);
299
306
  if !user_meta.is_empty() {
300
- meta = meta.with_user_metadata(user_meta);
307
+ meta.user_metadata(user_meta);
301
308
  }
302
309
 
303
310
  if let Some(v) = parse_header_to_str(headers, X_TOS_VERSION_ID)? {
304
- meta.set_version(v);
311
+ meta.version(v);
305
312
  }
306
313
 
307
- if let Some(is_dir) = parse_header_to_str(headers, X_TOS_DIRECTORY)?
314
+ if parse_header_to_str(headers, X_TOS_DIRECTORY)?
308
315
  .map(|v| {
309
316
  v.parse::<bool>().map_err(|e| {
310
317
  Error::new(ErrorKind::Unexpected, "header value is not valid integer")
@@ -312,17 +319,17 @@ impl Service for TosBackend {
312
319
  })
313
320
  })
314
321
  .transpose()?
322
+ == Some(true)
315
323
  {
316
- meta = meta.with_mode(if is_dir {
317
- EntryMode::DIR
318
- } else {
319
- EntryMode::FILE
320
- });
324
+ meta.set_dir();
321
325
  }
322
326
 
323
- Ok(RpStat::new(meta))
327
+ Ok(RpStat::new(meta.build()))
324
328
  }
325
- _ => Err(parse_error(resp)),
329
+ _ => Err(parse_error(
330
+ ErrorContext::new(ServiceOperation("HeadObject")),
331
+ resp,
332
+ )),
326
333
  }
327
334
  }
328
335
  fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -391,10 +398,9 @@ impl Service for TosBackend {
391
398
  from: &str,
392
399
  to: &str,
393
400
  args: OpCopy,
394
- opts: OpCopier,
395
401
  ) -> Result<Self::Copier> {
396
402
  let output: TosCopiers = {
397
- let copier = new_tos_copier(self.core.clone(), ctx, from, to, args, opts)?;
403
+ let copier = new_tos_copier(self.core.clone(), ctx, from, to, args)?;
398
404
  Ok(copier)
399
405
  }?;
400
406
 
@@ -416,13 +422,31 @@ impl Service for TosBackend {
416
422
 
417
423
  async fn presign(
418
424
  &self,
419
- _ctx: &OperationContext,
420
- _path: &str,
421
- _args: OpPresign,
425
+ ctx: &OperationContext,
426
+ path: &str,
427
+ args: OpPresign,
422
428
  ) -> Result<RpPresign> {
423
- Err(Error::new(
424
- ErrorKind::Unsupported,
425
- "operation is not supported",
426
- ))
429
+ let (expire, op) = args.into_parts();
430
+ let req = match op {
431
+ PresignOperation::Stat(v) => self.core.tos_head_object_request(path, v),
432
+ PresignOperation::Read(range, v) => self.core.tos_get_object_request(path, range, &v),
433
+ PresignOperation::Write(v) => {
434
+ self.core
435
+ .tos_put_object_request(path, None, &v, Buffer::new())
436
+ }
437
+ PresignOperation::Delete(v) => self.core.tos_delete_object_request(path, &v),
438
+ _ => Err(Error::new(
439
+ ErrorKind::Unsupported,
440
+ "operation is not supported",
441
+ )),
442
+ }?;
443
+ let req = self.core.sign_query(ctx, req, expire).await?;
444
+ let (parts, _) = req.into_parts();
445
+
446
+ Ok(RpPresign::new(PresignedRequest::new(
447
+ parts.method,
448
+ parts.uri,
449
+ parts.headers,
450
+ )))
427
451
  }
428
452
  }
@@ -34,7 +34,6 @@ pub fn new_tos_copier(
34
34
  from: &str,
35
35
  to: &str,
36
36
  args: OpCopy,
37
- opts: OpCopier,
38
37
  ) -> Result<TosCopiers> {
39
38
  let capability = core.capability;
40
39
  let max_part_size = capability.copy_multi_max_size.ok_or_else(|| {
@@ -44,7 +43,7 @@ pub fn new_tos_copier(
44
43
  )
45
44
  })?;
46
45
 
47
- let (copy_once_threshold, part_size) = match opts.chunk() {
46
+ let (copy_once_threshold, part_size) = match args.chunk() {
48
47
  Some(chunk) => {
49
48
  let min_part_size = capability.copy_multi_min_size.ok_or_else(|| {
50
49
  Error::new(
@@ -60,6 +59,8 @@ pub fn new_tos_copier(
60
59
  (part_size, part_size)
61
60
  }
62
61
  };
62
+ let source_content_length_hint = args.source_content_length_hint();
63
+ let concurrent = args.concurrent();
63
64
 
64
65
  Ok(oio::MultipartCopier::new(
65
66
  (ctx.executor().clone(), capability),
@@ -70,10 +71,10 @@ pub fn new_tos_copier(
70
71
  to: to.to_string(),
71
72
  args,
72
73
  },
73
- opts.source_content_length_hint(),
74
+ source_content_length_hint,
74
75
  copy_once_threshold,
75
76
  part_size,
76
- opts.concurrent(),
77
+ concurrent,
77
78
  ))
78
79
  }
79
80
 
@@ -97,7 +98,10 @@ impl oio::MultipartCopy for TosCopier {
97
98
  let headers = resp.headers();
98
99
  tos_parse_into_metadata(&self.from, headers)
99
100
  }
100
- _ => Err(parse_error(resp)),
101
+ _ => Err(parse_error(
102
+ ErrorContext::new(ServiceOperation("HeadObject")),
103
+ resp,
104
+ )),
101
105
  }
102
106
  }
103
107
 
@@ -119,11 +123,18 @@ impl oio::MultipartCopy for TosCopier {
119
123
  .set_temporary());
120
124
  }
121
125
 
122
- let mut meta = Metadata::new(EntryMode::from_path(&self.to));
123
- meta.set_etag(result.etag.trim_matches('"'));
124
- Ok(meta)
126
+ let mut meta = if self.to.ends_with('/') {
127
+ MetadataBuilder::dir()
128
+ } else {
129
+ MetadataBuilder::unknown()
130
+ };
131
+ meta.etag(result.etag.trim_matches('"'));
132
+ Ok(meta.build())
125
133
  }
126
- _ => Err(parse_error(resp)),
134
+ _ => Err(parse_error(
135
+ ErrorContext::new(ServiceOperation("CopyObject")),
136
+ resp,
137
+ )),
127
138
  }
128
139
  }
129
140
 
@@ -141,7 +152,10 @@ impl oio::MultipartCopy for TosCopier {
141
152
 
142
153
  Ok(result.upload_id)
143
154
  }
144
- _ => Err(parse_error(resp)),
155
+ _ => Err(parse_error(
156
+ ErrorContext::new(ServiceOperation("CreateMultipartUpload")),
157
+ resp,
158
+ )),
145
159
  }
146
160
  }
147
161
 
@@ -185,7 +199,10 @@ impl oio::MultipartCopy for TosCopier {
185
199
  size: Some(size),
186
200
  })
187
201
  }
188
- _ => Err(parse_error(resp)),
202
+ _ => Err(parse_error(
203
+ ErrorContext::new(ServiceOperation("UploadPartCopy")),
204
+ resp,
205
+ )),
189
206
  }
190
207
  }
191
208
 
@@ -216,17 +233,24 @@ impl oio::MultipartCopy for TosCopier {
216
233
  return Err(Error::new(ErrorKind::Unexpected, ret.message));
217
234
  }
218
235
 
219
- let mut meta = Metadata::new(EntryMode::from_path(&self.to));
236
+ let mut meta = if self.to.ends_with('/') {
237
+ MetadataBuilder::dir()
238
+ } else {
239
+ MetadataBuilder::unknown()
240
+ };
220
241
  if !ret.etag.is_empty() {
221
- meta.set_etag(ret.etag.trim_matches('"'));
242
+ meta.etag(ret.etag.trim_matches('"'));
222
243
  }
223
244
  if !ret.version_id.is_empty() {
224
- meta.set_version(&ret.version_id);
245
+ meta.version(&ret.version_id);
225
246
  }
226
247
 
227
- Ok(meta)
248
+ Ok(meta.build())
228
249
  }
229
- _ => Err(parse_error(resp)),
250
+ _ => Err(parse_error(
251
+ ErrorContext::new(ServiceOperation("CompleteMultipartUpload")),
252
+ resp,
253
+ )),
230
254
  }
231
255
  }
232
256
 
@@ -237,7 +261,10 @@ impl oio::MultipartCopy for TosCopier {
237
261
  .await?;
238
262
  match resp.status() {
239
263
  StatusCode::NO_CONTENT => Ok(()),
240
- _ => Err(parse_error(resp)),
264
+ _ => Err(parse_error(
265
+ ErrorContext::new(ServiceOperation("AbortMultipartUpload")),
266
+ resp,
267
+ )),
241
268
  }
242
269
  }
243
270
  }