opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -16,6 +16,7 @@
16
16
  // under the License.
17
17
 
18
18
  use std::fmt::Debug;
19
+ use std::time::Duration;
19
20
 
20
21
  use http::Request;
21
22
  use http::Response;
@@ -99,6 +100,27 @@ impl TosCore {
99
100
  )
100
101
  }
101
102
 
103
+ pub async fn sign_query<T>(
104
+ &self,
105
+ ctx: &OperationContext,
106
+ req: Request<T>,
107
+ duration: Duration,
108
+ ) -> Result<Request<T>> {
109
+ if self.skip_signature {
110
+ return Ok(req);
111
+ }
112
+
113
+ let (mut parts, body) = req.into_parts();
114
+ self.signer(ctx)
115
+ .sign(&mut parts, Some(duration))
116
+ .await
117
+ .map_err(|e| new_request_sign_error(e.into()))?;
118
+
119
+ parts.headers.remove(HOST);
120
+
121
+ Ok(Request::from_parts(parts, body))
122
+ }
123
+
102
124
  pub async fn send(
103
125
  &self,
104
126
  ctx: &OperationContext,
@@ -413,12 +435,11 @@ impl TosCore {
413
435
  self.send(ctx, req).await
414
436
  }
415
437
 
416
- pub async fn tos_delete_object(
438
+ pub fn tos_delete_object_request(
417
439
  &self,
418
- ctx: &OperationContext,
419
440
  path: &str,
420
441
  args: &OpDelete,
421
- ) -> Result<Response<Buffer>> {
442
+ ) -> Result<Request<Buffer>> {
422
443
  let p = build_abs_path(&self.root, path);
423
444
 
424
445
  let mut url = format!(
@@ -444,11 +465,19 @@ impl TosCore {
444
465
 
445
466
  let req = Request::delete(&url);
446
467
 
447
- let req = req
448
- .extension(Operation::Delete)
468
+ req.extension(Operation::Delete)
449
469
  .extension(ServiceOperation("DeleteObject"))
450
470
  .body(Buffer::new())
451
- .map_err(new_request_build_error)?;
471
+ .map_err(new_request_build_error)
472
+ }
473
+
474
+ pub async fn tos_delete_object(
475
+ &self,
476
+ ctx: &OperationContext,
477
+ path: &str,
478
+ args: &OpDelete,
479
+ ) -> Result<Response<Buffer>> {
480
+ let req = self.tos_delete_object_request(path, args)?;
452
481
 
453
482
  self.send(ctx, req).await
454
483
  }
@@ -1052,99 +1081,103 @@ pub struct CopyObjectOutput {
1052
1081
  pub etag: String,
1053
1082
  }
1054
1083
 
1055
- mod error {
1056
- use bytes::Buf;
1057
- use http::{Response, StatusCode};
1058
- use quick_xml::de;
1059
- use serde::Deserialize;
1060
-
1061
- use opendal_core::raw::*;
1062
- use opendal_core::*;
1063
-
1064
- /// TosError is the error returned by TOS service.
1065
- #[derive(Default, Debug, Deserialize, PartialEq, Eq)]
1066
- #[serde(default, rename_all = "PascalCase")]
1067
- pub(crate) struct TosError {
1068
- pub code: String,
1069
- pub message: String,
1070
- pub resource: String,
1071
- pub request_id: String,
1072
- }
1084
+ use bytes::Buf;
1085
+ use http::StatusCode;
1086
+ use quick_xml::de;
1073
1087
 
1074
- /// Parse error response into Error.
1075
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
1076
- let (parts, body) = resp.into_parts();
1077
- let bs = body.to_bytes();
1088
+ /// TosError is the error returned by TOS service.
1089
+ #[derive(Default, Debug, Deserialize, PartialEq, Eq)]
1090
+ #[serde(default, rename_all = "PascalCase")]
1091
+ pub(crate) struct TosError {
1092
+ pub code: String,
1093
+ pub message: String,
1094
+ pub resource: String,
1095
+ pub request_id: String,
1096
+ }
1078
1097
 
1079
- let (mut kind, mut retryable) = match parts.status {
1080
- StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
1081
- StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
1082
- StatusCode::NOT_MODIFIED | StatusCode::PRECONDITION_FAILED => {
1083
- (ErrorKind::ConditionNotMatch, false)
1084
- }
1085
- StatusCode::CONFLICT => (ErrorKind::ConditionNotMatch, true),
1086
- StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
1087
- code if code.is_server_error() => (ErrorKind::Unexpected, true),
1088
- _ => (ErrorKind::Unexpected, false),
1089
- };
1090
-
1091
- let (message, tos_err) = parse_tos_error(&bs)
1092
- .map(|tos_err| (format!("{tos_err:?}"), Some(tos_err)))
1093
- .unwrap_or_else(|| (String::from_utf8_lossy(&bs).into_owned(), None));
1094
-
1095
- if let Some(tos_err) = tos_err {
1096
- (kind, retryable) =
1097
- parse_tos_error_code(tos_err.code.as_str()).unwrap_or((kind, retryable));
1098
- }
1098
+ /// Context needed to classify an error from this service.
1099
+ #[derive(Clone, Copy, Debug)]
1100
+ pub(crate) struct ErrorContext {
1101
+ service_operation: ServiceOperation,
1102
+ }
1099
1103
 
1100
- let mut err = Error::new(kind, message);
1104
+ impl ErrorContext {
1105
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
1106
+ Self { service_operation }
1107
+ }
1108
+ }
1101
1109
 
1102
- err = with_error_response_context(err, parts);
1110
+ /// Parse an error response using its service request context.
1111
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
1112
+ let (parts, body) = resp.into_parts();
1113
+ let bs = body.to_bytes();
1103
1114
 
1104
- if retryable {
1105
- err = err.set_temporary();
1115
+ let (mut kind, mut retryable) = match parts.status {
1116
+ StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
1117
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
1118
+ StatusCode::NOT_MODIFIED | StatusCode::PRECONDITION_FAILED => {
1119
+ (ErrorKind::ConditionNotMatch, false)
1106
1120
  }
1107
-
1108
- err
1121
+ StatusCode::CONFLICT => (ErrorKind::ConditionNotMatch, true),
1122
+ StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
1123
+ code if code.is_server_error() => (ErrorKind::Unexpected, true),
1124
+ _ => (ErrorKind::Unexpected, false),
1125
+ };
1126
+
1127
+ let (message, tos_err) = parse_tos_error(&bs)
1128
+ .map(|tos_err| (format!("{tos_err:?}"), Some(tos_err)))
1129
+ .unwrap_or_else(|| (String::from_utf8_lossy(&bs).into_owned(), None));
1130
+
1131
+ if let Some(tos_err) = tos_err {
1132
+ (kind, retryable) =
1133
+ parse_tos_error_code(tos_err.code.as_str()).unwrap_or((kind, retryable));
1109
1134
  }
1110
1135
 
1111
- fn parse_tos_error(bs: &bytes::Bytes) -> Option<TosError> {
1112
- de::from_reader::<_, TosError>(bs.chunk().reader())
1113
- .or_else(|_| serde_json::from_slice::<TosError>(bs))
1114
- .ok()
1136
+ let mut err = Error::new(kind, message);
1137
+
1138
+ err = err.with_context("service_operation", ctx.service_operation.0);
1139
+ err = with_error_response_context(err, parts);
1140
+
1141
+ if retryable {
1142
+ err = err.set_temporary();
1115
1143
  }
1116
1144
 
1117
- /// Returns the `Error kind` of this code and whether the error is retryable.
1118
- /// TOS error codes: https://www.volcengine.com/docs/6349/74874
1119
- pub fn parse_tos_error_code(code: &str) -> Option<(ErrorKind, bool)> {
1120
- match code {
1121
- "NoSuchBucket" => Some((ErrorKind::ConfigInvalid, false)),
1122
- "NoSuchKey" => Some((ErrorKind::NotFound, false)),
1123
- "DuplicateObject" => Some((ErrorKind::ConditionNotMatch, false)),
1124
- "RequestTimeout" => Some((ErrorKind::Unexpected, true)),
1125
- "InternalError" => Some((ErrorKind::Unexpected, true)),
1126
- "OperationAborted" => Some((ErrorKind::Unexpected, true)),
1127
- "SlowDown" => Some((ErrorKind::RateLimited, true)),
1128
- "ServiceUnavailable" => Some((ErrorKind::Unexpected, true)),
1129
- "TooManyRequests" => Some((ErrorKind::RateLimited, true)),
1130
- "ExceedAccountQPSLimit" | "ExceedAccountRateLimit" => {
1131
- Some((ErrorKind::RateLimited, true))
1132
- }
1133
- "ExceedBucketQPSLimit" | "ExceedBucketRateLimit" => {
1134
- Some((ErrorKind::RateLimited, true))
1135
- }
1136
- _ => None,
1137
- }
1145
+ err
1146
+ }
1147
+
1148
+ fn parse_tos_error(bs: &bytes::Bytes) -> Option<TosError> {
1149
+ de::from_reader::<_, TosError>(bs.chunk().reader())
1150
+ .or_else(|_| serde_json::from_slice::<TosError>(bs))
1151
+ .ok()
1152
+ }
1153
+
1154
+ /// Returns the `Error kind` of this code and whether the error is retryable.
1155
+ /// TOS error codes: https://www.volcengine.com/docs/6349/74874
1156
+ pub fn parse_tos_error_code(code: &str) -> Option<(ErrorKind, bool)> {
1157
+ match code {
1158
+ "NoSuchBucket" => Some((ErrorKind::ConfigInvalid, false)),
1159
+ "NoSuchKey" => Some((ErrorKind::NotFound, false)),
1160
+ "DuplicateObject" => Some((ErrorKind::ConditionNotMatch, false)),
1161
+ "RequestTimeout" => Some((ErrorKind::Unexpected, true)),
1162
+ "InternalError" => Some((ErrorKind::Unexpected, true)),
1163
+ "OperationAborted" => Some((ErrorKind::Unexpected, true)),
1164
+ "SlowDown" => Some((ErrorKind::RateLimited, true)),
1165
+ "ServiceUnavailable" => Some((ErrorKind::Unexpected, true)),
1166
+ "TooManyRequests" => Some((ErrorKind::RateLimited, true)),
1167
+ "ExceedAccountQPSLimit" | "ExceedAccountRateLimit" => Some((ErrorKind::RateLimited, true)),
1168
+ "ExceedBucketQPSLimit" | "ExceedBucketRateLimit" => Some((ErrorKind::RateLimited, true)),
1169
+ _ => None,
1138
1170
  }
1171
+ }
1139
1172
 
1140
- #[cfg(test)]
1141
- mod tests {
1142
- use super::*;
1173
+ #[cfg(test)]
1174
+ mod tests {
1175
+ use super::*;
1143
1176
 
1144
- #[test]
1145
- fn test_parse_error() {
1146
- let bs = bytes::Bytes::from(
1147
- r#"
1177
+ #[test]
1178
+ fn test_parse_error() {
1179
+ let bs = bytes::Bytes::from(
1180
+ r#"
1148
1181
  <?xml version="1.0" encoding="UTF-8"?>
1149
1182
  <Error>
1150
1183
  <Code>NoSuchKey</Code>
@@ -1153,34 +1186,34 @@ mod error {
1153
1186
  <RequestId>4442587FB7D0A2F9</RequestId>
1154
1187
  </Error>
1155
1188
  "#,
1156
- );
1189
+ );
1157
1190
 
1158
- let out: TosError = de::from_reader(bs.reader()).expect("must success");
1159
- println!("{out:?}");
1191
+ let out: TosError = de::from_reader(bs.reader()).expect("must success");
1192
+ println!("{out:?}");
1160
1193
 
1161
- assert_eq!(out.code, "NoSuchKey");
1162
- assert_eq!(out.message, "The resource you requested does not exist");
1163
- assert_eq!(out.resource, "/mybucket/myfoto.jpg");
1164
- assert_eq!(out.request_id, "4442587FB7D0A2F9");
1165
- }
1194
+ assert_eq!(out.code, "NoSuchKey");
1195
+ assert_eq!(out.message, "The resource you requested does not exist");
1196
+ assert_eq!(out.resource, "/mybucket/myfoto.jpg");
1197
+ assert_eq!(out.request_id, "4442587FB7D0A2F9");
1198
+ }
1166
1199
 
1167
- #[test]
1168
- fn test_parse_json_error() {
1169
- let bs = bytes::Bytes::from(
1170
- r#"{"Code":"DuplicateObject","RequestId":"request-id","Message":"The object already exists."}"#,
1171
- );
1200
+ #[test]
1201
+ fn test_parse_json_error() {
1202
+ let bs = bytes::Bytes::from(
1203
+ r#"{"Code":"DuplicateObject","RequestId":"request-id","Message":"The object already exists."}"#,
1204
+ );
1172
1205
 
1173
- let out = parse_tos_error(&bs).expect("must success");
1206
+ let out = parse_tos_error(&bs).expect("must success");
1174
1207
 
1175
- assert_eq!(out.code, "DuplicateObject");
1176
- assert_eq!(out.message, "The object already exists.");
1177
- assert_eq!(out.request_id, "request-id");
1178
- }
1208
+ assert_eq!(out.code, "DuplicateObject");
1209
+ assert_eq!(out.message, "The object already exists.");
1210
+ assert_eq!(out.request_id, "request-id");
1211
+ }
1179
1212
 
1180
- #[test]
1181
- fn test_parse_error_from_unrelated_input() {
1182
- let bs = bytes::Bytes::from(
1183
- r#"
1213
+ #[test]
1214
+ fn test_parse_error_from_unrelated_input() {
1215
+ let bs = bytes::Bytes::from(
1216
+ r#"
1184
1217
  <?xml version="1.0" encoding="UTF-8"?>
1185
1218
  <CompleteMultipartUploadResult xmlns="http://tos.apache.org/doc/2024-01-01/">
1186
1219
  <Bucket>example-bucket</Bucket>
@@ -1188,16 +1221,13 @@ mod error {
1188
1221
  <UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId>
1189
1222
  </CompleteMultipartUploadResult>
1190
1223
  "#,
1191
- );
1224
+ );
1192
1225
 
1193
- let out: TosError = de::from_reader(bs.reader()).expect("must success");
1194
- assert_eq!(out, TosError::default());
1195
- }
1226
+ let out: TosError = de::from_reader(bs.reader()).expect("must success");
1227
+ assert_eq!(out, TosError::default());
1196
1228
  }
1197
1229
  }
1198
1230
 
1199
- pub(super) use error::*;
1200
-
1201
1231
  mod utils {
1202
1232
  use http::HeaderMap;
1203
1233
  use http::header::ETAG;
@@ -1216,13 +1246,13 @@ mod utils {
1216
1246
  path: &str,
1217
1247
  headers: &HeaderMap,
1218
1248
  ) -> opendal_core::Result<Metadata> {
1219
- let mut metadata = parse_into_metadata(path, headers)?;
1249
+ let mut metadata = parse_into_metadata(path, headers)?.into_builder();
1220
1250
 
1221
1251
  if let Some(etag) = tos_parse_etag(headers)? {
1222
- metadata.set_etag(etag);
1252
+ metadata.etag(etag);
1223
1253
  }
1224
1254
 
1225
- Ok(metadata)
1255
+ Ok(metadata.build())
1226
1256
  }
1227
1257
  }
1228
1258
 
@@ -50,7 +50,10 @@ impl oio::BatchDelete for TosDeleter {
50
50
  match status {
51
51
  StatusCode::NO_CONTENT => Ok(()),
52
52
  StatusCode::NOT_FOUND => Ok(()),
53
- _ => Err(crate::core::parse_error(resp)),
53
+ _ => Err(crate::core::parse_error(
54
+ crate::core::ErrorContext::new(ServiceOperation("DeleteObject")),
55
+ resp,
56
+ )),
54
57
  }
55
58
  }
56
59
 
@@ -59,7 +62,10 @@ impl oio::BatchDelete for TosDeleter {
59
62
 
60
63
  let status = resp.status();
61
64
  if status != StatusCode::OK {
62
- return Err(crate::core::parse_error(resp));
65
+ return Err(crate::core::parse_error(
66
+ crate::core::ErrorContext::new(ServiceOperation("DeleteMultipleObjects")),
67
+ resp,
68
+ ));
63
69
  }
64
70
 
65
71
  let bs = resp.into_body();
@@ -90,7 +96,7 @@ impl oio::BatchDelete for TosDeleter {
90
96
  let deleted = deleted.swap_remove(idx);
91
97
  let op = if op.version().is_some() {
92
98
  if let Some(version) = &deleted.version_id {
93
- op.with_version(version)
99
+ op.into_version(version)
94
100
  } else {
95
101
  op
96
102
  }
@@ -10,7 +10,7 @@ Depending on its configuration and the backing system, this service can expose:
10
10
  - [x] list
11
11
  - [x] copy
12
12
  - [ ] rename
13
- - [ ] presign
13
+ - [x] presign
14
14
 
15
15
  Inspect the effective capability set with [`opendal_core::Operator::info`] and
16
16
  [`opendal_core::OperatorInfo::capability`] after building an operator.
@@ -21,8 +21,7 @@ use bytes::Buf;
21
21
 
22
22
  use crate::core::parse_error;
23
23
  use crate::core::*;
24
- use opendal_core::EntryMode;
25
- use opendal_core::Metadata;
24
+ use opendal_core::MetadataBuilder;
26
25
  use opendal_core::OperationContext;
27
26
  use opendal_core::Result;
28
27
  use opendal_core::raw::*;
@@ -77,7 +76,10 @@ impl oio::PageList for TosLister {
77
76
  .await?;
78
77
 
79
78
  if resp.status() != http::StatusCode::OK {
80
- return Err(parse_error(resp));
79
+ return Err(parse_error(
80
+ ErrorContext::new(ServiceOperation("ListObjectsV2")),
81
+ resp,
82
+ ));
81
83
  }
82
84
 
83
85
  let bs = resp.into_body();
@@ -90,7 +92,7 @@ impl oio::PageList for TosLister {
90
92
  for prefix in output.common_prefixes {
91
93
  let de = oio::Entry::new(
92
94
  &build_rel_path(&self.core.root, &prefix.prefix),
93
- Metadata::new(EntryMode::DIR),
95
+ MetadataBuilder::dir().build(),
94
96
  );
95
97
 
96
98
  ctx.entries.push_back(de);
@@ -102,16 +104,19 @@ impl oio::PageList for TosLister {
102
104
  path = "/".to_string();
103
105
  }
104
106
 
105
- let mut meta = Metadata::new(EntryMode::from_path(&path));
106
- meta.set_is_current(true);
107
+ let mut meta = if path.ends_with('/') {
108
+ MetadataBuilder::dir()
109
+ } else {
110
+ MetadataBuilder::file(object.size)
111
+ };
112
+ meta.is_current(Some(true));
107
113
  if let Some(etag) = &object.etag {
108
- meta.set_etag(etag);
109
- meta.set_content_md5(etag.trim_matches('"'));
114
+ meta.etag(etag);
115
+ meta.content_md5(etag.trim_matches('"'));
110
116
  }
111
- meta.set_content_length(object.size);
112
- meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
117
+ meta.last_modified(object.last_modified.parse::<Timestamp>()?);
113
118
 
114
- let de = oio::Entry::with(path, meta);
119
+ let de = oio::Entry::with(path, meta.build());
115
120
  ctx.entries.push_back(de);
116
121
  }
117
122
 
@@ -172,7 +177,10 @@ impl oio::PageList for TosObjectVersionsLister {
172
177
  .await?;
173
178
 
174
179
  if resp.status() != http::StatusCode::OK {
175
- return Err(parse_error(resp));
180
+ return Err(parse_error(
181
+ ErrorContext::new(ServiceOperation("ListObjectVersions")),
182
+ resp,
183
+ ));
176
184
  }
177
185
 
178
186
  let bs = resp.into_body();
@@ -193,7 +201,7 @@ impl oio::PageList for TosObjectVersionsLister {
193
201
  for prefix in output.common_prefixes {
194
202
  let de = oio::Entry::new(
195
203
  &build_rel_path(&self.core.root, &prefix.prefix),
196
- Metadata::new(EntryMode::DIR),
204
+ MetadataBuilder::dir().build(),
197
205
  );
198
206
 
199
207
  ctx.entries.push_back(de);
@@ -209,18 +217,21 @@ impl oio::PageList for TosObjectVersionsLister {
209
217
  path = "/".to_string();
210
218
  }
211
219
 
212
- let mut meta = Metadata::new(EntryMode::from_path(&path));
213
- meta.set_version(&version_object.version_id);
214
- meta.set_is_current(version_object.is_latest);
215
- meta.set_content_length(version_object.size);
216
- meta.set_last_modified(version_object.last_modified.parse::<Timestamp>()?);
220
+ let mut meta = if path.ends_with('/') {
221
+ MetadataBuilder::dir()
222
+ } else {
223
+ MetadataBuilder::file(version_object.size)
224
+ };
225
+ meta.version(&version_object.version_id);
226
+ meta.is_current(Some(version_object.is_latest));
227
+ meta.last_modified(version_object.last_modified.parse::<Timestamp>()?);
217
228
 
218
229
  if let Some(etag) = version_object.etag {
219
- meta.set_etag(&etag);
220
- meta.set_content_md5(etag.trim_matches('"'));
230
+ meta.etag(&etag);
231
+ meta.content_md5(etag.trim_matches('"'));
221
232
  }
222
233
 
223
- let de = oio::Entry::with(path, meta);
234
+ let de = oio::Entry::with(path, meta.build());
224
235
  ctx.entries.push_back(de);
225
236
  }
226
237
 
@@ -231,13 +242,17 @@ impl oio::PageList for TosObjectVersionsLister {
231
242
  path = "/".to_string();
232
243
  }
233
244
 
234
- let mut meta = Metadata::new(EntryMode::FILE);
235
- meta.set_version(&delete_marker.version_id);
236
- meta.set_is_deleted(true);
237
- meta.set_is_current(delete_marker.is_latest);
238
- meta.set_last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
239
-
240
- let de = oio::Entry::with(path, meta);
245
+ let mut meta = if path.ends_with('/') {
246
+ MetadataBuilder::dir()
247
+ } else {
248
+ MetadataBuilder::file(0)
249
+ };
250
+ meta.version(&delete_marker.version_id);
251
+ meta.is_deleted(true);
252
+ meta.is_current(Some(delete_marker.is_latest));
253
+ meta.last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
254
+
255
+ let de = oio::Entry::with(path, meta.build());
241
256
  ctx.entries.push_back(de);
242
257
  }
243
258
  }
@@ -16,7 +16,7 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use crate::core::parse_error;
19
+ use crate::core::{ErrorContext, parse_error};
20
20
  use http::Response;
21
21
  use http::StatusCode;
22
22
  use opendal_core::BytesRange;
@@ -67,7 +67,10 @@ impl oio::StreamRead for TosReader {
67
67
  _ => {
68
68
  let (part, mut body) = resp.into_parts();
69
69
  let buf = body.to_buffer().await?;
70
- return Err(parse_error(Response::from_parts(part, buf)));
70
+ return Err(parse_error(
71
+ ErrorContext::new(ServiceOperation("GetObject")),
72
+ Response::from_parts(part, buf),
73
+ ));
71
74
  }
72
75
  };
73
76
 
@@ -47,19 +47,24 @@ impl TosWriter {
47
47
  }
48
48
 
49
49
  fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> {
50
- let mut meta = Metadata::new(EntryMode::from_path(path));
50
+ let mut meta = if path.ends_with('/') {
51
+ MetadataBuilder::dir()
52
+ } else {
53
+ MetadataBuilder::unknown()
54
+ };
51
55
  if let Some(etag) = tos_parse_etag(headers)? {
52
- meta.set_etag(etag);
56
+ meta.etag(etag);
53
57
  }
54
58
  if let Some(version) = parse_header_to_str(headers, X_TOS_VERSION_ID)? {
55
- meta.set_version(version);
59
+ meta.version(version);
56
60
  }
57
- if let Some(value) =
58
- parse_header_to_str(headers, X_TOS_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
61
+ if !path.ends_with('/')
62
+ && let Some(value) =
63
+ parse_header_to_str(headers, X_TOS_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
59
64
  {
60
- meta.set_content_length(value);
65
+ meta.set_file(value);
61
66
  }
62
- Ok(meta)
67
+ Ok(meta.build())
63
68
  }
64
69
  }
65
70
 
@@ -77,7 +82,10 @@ impl oio::MultipartWrite for TosWriter {
77
82
 
78
83
  match status {
79
84
  StatusCode::CREATED | StatusCode::OK => Ok(meta),
80
- _ => Err(parse_error(resp)),
85
+ _ => Err(parse_error(
86
+ ErrorContext::new(ServiceOperation("PutObject")),
87
+ resp,
88
+ )),
81
89
  }
82
90
  }
83
91
 
@@ -97,7 +105,10 @@ impl oio::MultipartWrite for TosWriter {
97
105
 
98
106
  Ok(result.upload_id)
99
107
  }
100
- _ => Err(parse_error(resp)),
108
+ _ => Err(parse_error(
109
+ ErrorContext::new(ServiceOperation("CreateMultipartUpload")),
110
+ resp,
111
+ )),
101
112
  }
102
113
  }
103
114
 
@@ -135,7 +146,10 @@ impl oio::MultipartWrite for TosWriter {
135
146
  size: None,
136
147
  })
137
148
  }
138
- _ => Err(parse_error(resp)),
149
+ _ => Err(parse_error(
150
+ ErrorContext::new(ServiceOperation("UploadPart")),
151
+ resp,
152
+ )),
139
153
  }
140
154
  }
141
155
 
@@ -158,7 +172,8 @@ impl oio::MultipartWrite for TosWriter {
158
172
  .await?;
159
173
 
160
174
  let status = resp.status();
161
- let mut meta = TosWriter::parse_header_into_meta(&self.path, resp.headers())?;
175
+ let mut meta =
176
+ TosWriter::parse_header_into_meta(&self.path, resp.headers())?.into_builder();
162
177
 
163
178
  match status {
164
179
  StatusCode::OK => {
@@ -171,15 +186,18 @@ impl oio::MultipartWrite for TosWriter {
171
186
  if !ret.etag.is_empty() {
172
187
  // CompleteMultipartUpload response wraps ETag in quotes:
173
188
  // https://www.volcengine.com/docs/6349/74868
174
- meta.set_etag(ret.etag.trim_matches('"'));
189
+ meta.etag(ret.etag.trim_matches('"'));
175
190
  }
176
191
  if !ret.version_id.is_empty() {
177
- meta.set_version(&ret.version_id);
192
+ meta.version(&ret.version_id);
178
193
  }
179
194
 
180
- Ok(meta)
195
+ Ok(meta.build())
181
196
  }
182
- _ => Err(parse_error(resp)),
197
+ _ => Err(parse_error(
198
+ ErrorContext::new(ServiceOperation("CompleteMultipartUpload")),
199
+ resp,
200
+ )),
183
201
  }
184
202
  }
185
203
 
@@ -191,7 +209,10 @@ impl oio::MultipartWrite for TosWriter {
191
209
 
192
210
  match resp.status() {
193
211
  StatusCode::NO_CONTENT => Ok(()),
194
- _ => Err(parse_error(resp)),
212
+ _ => Err(parse_error(
213
+ ErrorContext::new(ServiceOperation("AbortMultipartUpload")),
214
+ resp,
215
+ )),
195
216
  }
196
217
  }
197
218
  }
@@ -38,7 +38,7 @@ hmac = "0.13.0"
38
38
  http = { workspace = true }
39
39
  log = { workspace = true }
40
40
  md-5 = "0.11.0"
41
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
41
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
42
42
  quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
43
43
  serde = { workspace = true, features = ["derive"] }
44
44
  serde_json = { workspace = true }