opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -222,6 +222,7 @@ impl Service for D1Backend {
222
222
  type Lister = ();
223
223
  type Deleter = oio::OneShotDeleter<D1Deleter>;
224
224
  type Copier = ();
225
+ type Composer = ();
225
226
 
226
227
  fn info(&self) -> ServiceInfo {
227
228
  self.info.clone()
@@ -247,12 +248,13 @@ impl Service for D1Backend {
247
248
  let p = build_abs_path(&self.root, path);
248
249
 
249
250
  if p == build_abs_path(&self.root, "") {
250
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
251
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
251
252
  } else {
252
253
  match self.core.get_length(ctx, &p).await? {
253
- Some(length) => Ok(RpStat::new(
254
- Metadata::new(EntryMode::FILE).with_content_length(length as u64),
255
- )),
254
+ Some(length) => Ok(RpStat::new({
255
+ let metadata = MetadataBuilder::file(length as u64);
256
+ metadata.build()
257
+ })),
256
258
  None => Err(Error::new(ErrorKind::NotFound, "kv not found in d1")),
257
259
  }
258
260
  }
@@ -304,7 +306,6 @@ impl Service for D1Backend {
304
306
  _from: &str,
305
307
  _to: &str,
306
308
  _args: OpCopy,
307
- _opts: OpCopier,
308
309
  ) -> Result<Self::Copier> {
309
310
  Err(Error::new(
310
311
  ErrorKind::Unsupported,
@@ -103,7 +103,10 @@ impl D1Core {
103
103
  let d1_response = D1Response::parse(&bs)?;
104
104
  Ok(d1_response.get_result(&self.value_field))
105
105
  }
106
- _ => Err(parse_error(resp)),
106
+ _ => Err(parse_error(
107
+ ErrorContext::new(ServiceOperation("Query")),
108
+ resp,
109
+ )),
107
110
  }
108
111
  }
109
112
 
@@ -124,7 +127,10 @@ impl D1Core {
124
127
  let d1_response = D1Response::parse(&bs)?;
125
128
  d1_response.get_usize_result("content_length")
126
129
  }
127
- _ => Err(parse_error(resp)),
130
+ _ => Err(parse_error(
131
+ ErrorContext::new(ServiceOperation("Query")),
132
+ resp,
133
+ )),
128
134
  }
129
135
  }
130
136
 
@@ -133,9 +139,9 @@ impl D1Core {
133
139
  let key_field = &self.key_field;
134
140
  let value_field = &self.value_field;
135
141
  let query = format!(
136
- r#"INSERT INTO "{table}" ("{key_field}", "{value_field}") \
137
- VALUES ('?', ?) \
138
- ON CONFLICT ("{key_field}") \
142
+ r#"INSERT INTO "{table}" ("{key_field}", "{value_field}")
143
+ VALUES (?, ?)
144
+ ON CONFLICT ("{key_field}")
139
145
  DO UPDATE SET "{value_field}" = EXCLUDED."{value_field}""#,
140
146
  );
141
147
 
@@ -146,7 +152,10 @@ impl D1Core {
146
152
  let status = resp.status();
147
153
  match status {
148
154
  StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
149
- _ => Err(parse_error(resp)),
155
+ _ => Err(parse_error(
156
+ ErrorContext::new(ServiceOperation("Query")),
157
+ resp,
158
+ )),
150
159
  }
151
160
  }
152
161
 
@@ -162,73 +171,80 @@ impl D1Core {
162
171
  let status = resp.status();
163
172
  match status {
164
173
  StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
165
- _ => Err(parse_error(resp)),
174
+ _ => Err(parse_error(
175
+ ErrorContext::new(ServiceOperation("Query")),
176
+ resp,
177
+ )),
166
178
  }
167
179
  }
168
180
  }
169
181
 
170
- mod error {
171
- use bytes::Buf;
172
- use http::Response;
173
- use http::StatusCode;
174
- use serde_json::de;
175
-
176
- use crate::model::*;
177
- use opendal_core::raw::*;
178
- use opendal_core::*;
179
-
180
- /// Parse error response into Error.
181
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
182
- let (parts, body) = resp.into_parts();
183
- let bs = body.to_bytes();
184
-
185
- let (mut kind, mut retryable) = match parts.status {
186
- StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
187
- // Some services (like owncloud) return 403 while file locked.
188
- StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
189
- // Allowing retry for resource locked.
190
- StatusCode::LOCKED => (ErrorKind::Unexpected, true),
191
- StatusCode::INTERNAL_SERVER_ERROR
192
- | StatusCode::BAD_GATEWAY
193
- | StatusCode::SERVICE_UNAVAILABLE
194
- | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
195
- _ => (ErrorKind::Unexpected, false),
196
- };
197
-
198
- let (message, d1_err) = de::from_reader::<_, D1Response>(bs.clone().reader())
199
- .map(|d1_err| (format!("{d1_err:?}"), Some(d1_err)))
200
- .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
201
-
202
- if let Some(d1_err) = d1_err {
203
- (kind, retryable) = parse_d1_error_code(d1_err.errors).unwrap_or((kind, retryable));
204
- }
205
-
206
- let mut err = Error::new(kind, message);
182
+ use bytes::Buf;
183
+ use http::Response;
184
+ use serde_json::de;
207
185
 
208
- err = with_error_response_context(err, parts);
186
+ /// Context needed to classify an error from this service.
187
+ #[derive(Clone, Copy, Debug)]
188
+ pub(crate) struct ErrorContext {
189
+ service_operation: ServiceOperation,
190
+ }
209
191
 
210
- if retryable {
211
- err = err.set_temporary();
212
- }
192
+ impl ErrorContext {
193
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
194
+ Self { service_operation }
195
+ }
196
+ }
213
197
 
214
- err
198
+ /// Parse an error response using its service request context.
199
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
200
+ let (parts, body) = resp.into_parts();
201
+ let bs = body.to_bytes();
202
+
203
+ let (mut kind, mut retryable) = match parts.status {
204
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
205
+ // Some services (like owncloud) return 403 while file locked.
206
+ StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
207
+ // Allowing retry for resource locked.
208
+ StatusCode::LOCKED => (ErrorKind::Unexpected, true),
209
+ StatusCode::INTERNAL_SERVER_ERROR
210
+ | StatusCode::BAD_GATEWAY
211
+ | StatusCode::SERVICE_UNAVAILABLE
212
+ | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
213
+ _ => (ErrorKind::Unexpected, false),
214
+ };
215
+
216
+ let (message, d1_err) = de::from_reader::<_, D1Response>(bs.clone().reader())
217
+ .map(|d1_err| (format!("{d1_err:?}"), Some(d1_err)))
218
+ .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
219
+
220
+ if let Some(d1_err) = d1_err {
221
+ (kind, retryable) = parse_d1_error_code(d1_err.errors).unwrap_or((kind, retryable));
215
222
  }
216
223
 
217
- pub fn parse_d1_error_code(errors: Vec<D1Error>) -> Option<(ErrorKind, bool)> {
218
- if errors.is_empty() {
219
- return None;
220
- }
224
+ let mut err = Error::new(kind, message);
221
225
 
222
- match errors[0].code {
223
- // The request is malformed: failed to decode id.
224
- 7400 => Some((ErrorKind::Unexpected, false)),
225
- // no such column: Xxxx.
226
- 7500 => Some((ErrorKind::NotFound, false)),
227
- // Authentication error.
228
- 10000 => Some((ErrorKind::PermissionDenied, false)),
229
- _ => None,
230
- }
226
+ err = err.with_context("service_operation", ctx.service_operation.0);
227
+ err = with_error_response_context(err, parts);
228
+
229
+ if retryable {
230
+ err = err.set_temporary();
231
231
  }
232
+
233
+ err
232
234
  }
233
235
 
234
- pub(super) use error::*;
236
+ pub fn parse_d1_error_code(errors: Vec<D1Error>) -> Option<(ErrorKind, bool)> {
237
+ if errors.is_empty() {
238
+ return None;
239
+ }
240
+
241
+ match errors[0].code {
242
+ // The request is malformed: failed to decode id.
243
+ 7400 => Some((ErrorKind::Unexpected, false)),
244
+ // no such column: Xxxx.
245
+ 7500 => Some((ErrorKind::NotFound, false)),
246
+ // Authentication error.
247
+ 10000 => Some((ErrorKind::PermissionDenied, false)),
248
+ _ => None,
249
+ }
250
+ }
@@ -48,7 +48,10 @@ impl oio::StreamRead for D1Reader {
48
48
  }
49
49
  };
50
50
  let content = bs.slice(range.to_content_range(bs.len())?);
51
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
51
+ let metadata = {
52
+ let metadata = MetadataBuilder::file(bs.len() as u64);
53
+ metadata.build()
54
+ };
52
55
  Ok((
53
56
  RpRead::new(metadata),
54
57
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -50,8 +50,12 @@ impl oio::Write for D1Writer {
50
50
  let length = buf.len() as u64;
51
51
  self.core.set(&self.ctx, &self.path, buf).await?;
52
52
 
53
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
54
- Ok(meta)
53
+ let metadata = if self.path.ends_with('/') {
54
+ MetadataBuilder::dir()
55
+ } else {
56
+ MetadataBuilder::file(length)
57
+ };
58
+ Ok(metadata.build())
55
59
  }
56
60
 
57
61
  async fn abort(&mut self) -> Result<()> {
@@ -34,7 +34,7 @@ all-features = true
34
34
  [dependencies]
35
35
  dashmap = "6"
36
36
  log = { workspace = true }
37
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
38
38
  serde = { workspace = true, features = ["derive"] }
39
39
 
40
40
  [dev-dependencies]
@@ -116,6 +116,7 @@ impl Service for DashmapBackend {
116
116
  type Lister = oio::HierarchyLister<DashmapLister>;
117
117
  type Deleter = oio::OneShotDeleter<DashmapDeleter>;
118
118
  type Copier = ();
119
+ type Composer = ();
119
120
 
120
121
  fn info(&self) -> ServiceInfo {
121
122
  self.info.clone()
@@ -149,7 +150,7 @@ impl Service for DashmapBackend {
149
150
  if p.ends_with('/') {
150
151
  let has_children = self.core.cache.iter().any(|kv| kv.key().starts_with(&p));
151
152
  if has_children {
152
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
153
+ return Ok(RpStat::new(MetadataBuilder::dir().build()));
153
154
  }
154
155
  }
155
156
  Err(Error::new(ErrorKind::NotFound, "key not found in dashmap"))
@@ -204,7 +205,6 @@ impl Service for DashmapBackend {
204
205
  _from: &str,
205
206
  _to: &str,
206
207
  _args: OpCopy,
207
- _opts: OpCopier,
208
208
  ) -> Result<Self::Copier> {
209
209
  Err(Error::new(
210
210
  ErrorKind::Unsupported,
@@ -46,7 +46,10 @@ impl oio::StreamRead for DashmapReader {
46
46
  let buffer = value
47
47
  .content
48
48
  .slice(range.to_content_range(value.content.len())?);
49
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(total_size);
49
+ let metadata = {
50
+ let metadata = MetadataBuilder::file(total_size);
51
+ metadata.build()
52
+ };
50
53
  Ok((
51
54
  RpRead::new(metadata),
52
55
  Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,
@@ -50,24 +50,27 @@ impl oio::Write for DashmapWriter {
50
50
  async fn close(&mut self) -> Result<Metadata> {
51
51
  let content = self.buf.clone().collect();
52
52
 
53
- let entry_mode = EntryMode::from_path(&self.path);
54
- let mut meta = Metadata::new(entry_mode);
55
- meta.set_content_length(content.len() as u64);
56
- meta.set_last_modified(Timestamp::now());
53
+ let mut meta = if self.path.ends_with('/') {
54
+ MetadataBuilder::dir()
55
+ } else {
56
+ MetadataBuilder::file(content.len() as u64)
57
+ };
58
+ meta.last_modified(Timestamp::now());
57
59
 
58
60
  if let Some(v) = self.op.content_type() {
59
- meta.set_content_type(v);
61
+ meta.content_type(v);
60
62
  }
61
63
  if let Some(v) = self.op.content_disposition() {
62
- meta.set_content_disposition(v);
64
+ meta.content_disposition(v);
63
65
  }
64
66
  if let Some(v) = self.op.cache_control() {
65
- meta.set_cache_control(v);
67
+ meta.cache_control(v);
66
68
  }
67
69
  if let Some(v) = self.op.content_encoding() {
68
- meta.set_content_encoding(v);
70
+ meta.content_encoding(v);
69
71
  }
70
72
 
73
+ let meta = meta.build();
71
74
  self.core.set(
72
75
  &self.path,
73
76
  DashmapValue {
@@ -36,7 +36,7 @@ base64 = { workspace = true }
36
36
  bytes = { workspace = true }
37
37
  http = { workspace = true }
38
38
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
40
40
  serde = { workspace = true, features = ["derive"] }
41
41
  serde_json = { workspace = true }
42
42
 
@@ -24,8 +24,8 @@ use serde::Deserialize;
24
24
 
25
25
  use super::DBFS_SCHEME;
26
26
  use super::config::DbfsConfig;
27
- use super::core::DbfsCore;
28
27
  use super::core::parse_error;
28
+ use super::core::{DbfsCore, ErrorContext};
29
29
  use super::deleter::DbfsDeleter;
30
30
  use super::lister::DbfsLister;
31
31
  use super::writer::DbfsWriter;
@@ -144,6 +144,7 @@ impl Service for DbfsBackend {
144
144
  type Lister = oio::PageLister<DbfsLister>;
145
145
  type Deleter = oio::OneShotDeleter<DbfsDeleter>;
146
146
  type Copier = ();
147
+ type Composer = ();
147
148
 
148
149
  fn info(&self) -> ServiceInfo {
149
150
  ServiceInfo::new(DBFS_SCHEME, &self.core.root, "")
@@ -165,14 +166,17 @@ impl Service for DbfsBackend {
165
166
 
166
167
  match status {
167
168
  StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
168
- _ => Err(parse_error(resp)),
169
+ _ => Err(parse_error(
170
+ ErrorContext::new(ServiceOperation("Mkdirs")),
171
+ resp,
172
+ )),
169
173
  }
170
174
  }
171
175
 
172
176
  async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
173
177
  // Stat root always returns a DIR.
174
178
  if path == "/" {
175
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
179
+ return Ok(RpStat::new(MetadataBuilder::dir().build()));
176
180
  }
177
181
 
178
182
  let resp = self.core.dbfs_get_status(ctx, path).await?;
@@ -181,26 +185,26 @@ impl Service for DbfsBackend {
181
185
 
182
186
  match status {
183
187
  StatusCode::OK => {
184
- let mut meta = parse_into_metadata(path, resp.headers())?;
188
+ let mut meta = MetadataBuilder::unknown();
185
189
  let bs = resp.into_body();
186
190
  let decoded_response: DbfsStatus =
187
191
  serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
188
- meta.set_last_modified(Timestamp::from_millisecond(
192
+ meta.last_modified(Timestamp::from_millisecond(
189
193
  decoded_response.modification_time,
190
194
  )?);
191
195
  match decoded_response.is_dir {
192
- true => meta.set_mode(EntryMode::DIR),
193
- false => {
194
- meta.set_mode(EntryMode::FILE);
195
- meta.set_content_length(decoded_response.file_size as u64)
196
- }
196
+ true => meta.set_dir(),
197
+ false => meta.set_file(decoded_response.file_size as u64),
197
198
  };
198
- Ok(RpStat::new(meta))
199
+ Ok(RpStat::new(meta.build()))
199
200
  }
200
201
  StatusCode::NOT_FOUND if path.ends_with('/') => {
201
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
202
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
202
203
  }
203
- _ => Err(parse_error(resp)),
204
+ _ => Err(parse_error(
205
+ ErrorContext::new(ServiceOperation("GetStatus")),
206
+ resp,
207
+ )),
204
208
  }
205
209
  }
206
210
 
@@ -260,7 +264,10 @@ impl Service for DbfsBackend {
260
264
 
261
265
  match status {
262
266
  StatusCode::OK => Ok(RpRename::default()),
263
- _ => Err(parse_error(resp)),
267
+ _ => Err(parse_error(
268
+ ErrorContext::new(ServiceOperation("Move")),
269
+ resp,
270
+ )),
264
271
  }
265
272
  }
266
273
 
@@ -270,7 +277,6 @@ impl Service for DbfsBackend {
270
277
  _from: &str,
271
278
  _to: &str,
272
279
  _args: OpCopy,
273
- _opts: OpCopier,
274
280
  ) -> Result<Self::Copier> {
275
281
  Err(Error::new(
276
282
  ErrorKind::Unsupported,
@@ -24,6 +24,7 @@ use http::Request;
24
24
  use http::Response;
25
25
  use http::StatusCode;
26
26
  use http::header;
27
+ use serde::Deserialize;
27
28
  use serde_json::json;
28
29
 
29
30
  use opendal_core::raw::*;
@@ -62,7 +63,7 @@ impl DbfsCore {
62
63
  .to_string();
63
64
 
64
65
  let req_body = &json!({
65
- "path": percent_encode_path(&p),
66
+ "path": p,
66
67
  });
67
68
  let body = Buffer::from(Bytes::from(req_body.to_string()));
68
69
 
@@ -91,7 +92,7 @@ impl DbfsCore {
91
92
  .to_string();
92
93
 
93
94
  let request_body = &json!({
94
- "path": percent_encode_path(&p),
95
+ "path": p,
95
96
  // TODO: support recursive toggle, should we add a new field in OpDelete?
96
97
  "recursive": true,
97
98
  });
@@ -123,8 +124,8 @@ impl DbfsCore {
123
124
  req = req.header(header::AUTHORIZATION, auth_header_content);
124
125
 
125
126
  let req_body = &json!({
126
- "source_path": percent_encode_path(&source),
127
- "destination_path": percent_encode_path(&target),
127
+ "source_path": source,
128
+ "destination_path": target,
128
129
  });
129
130
 
130
131
  let body = Buffer::from(Bytes::from(req_body.to_string()));
@@ -172,7 +173,7 @@ impl DbfsCore {
172
173
  req = req.header(header::AUTHORIZATION, auth_header_content);
173
174
 
174
175
  let req_body = &json!({
175
- "path": path,
176
+ "path": build_rooted_abs_path(&self.root, path),
176
177
  "contents": contents,
177
178
  "overwrite": true,
178
179
  });
@@ -222,71 +223,75 @@ impl DbfsCore {
222
223
  StatusCode::NOT_FOUND => {
223
224
  self.dbfs_create_dir(ctx, path).await?;
224
225
  }
225
- _ => return Err(parse_error(resp)),
226
+ _ => {
227
+ return Err(parse_error(
228
+ ErrorContext::new(ServiceOperation("GetStatus")),
229
+ resp,
230
+ ));
231
+ }
226
232
  }
227
233
  Ok(())
228
234
  }
229
235
  }
230
236
 
231
- mod error {
232
- use std::fmt::Debug;
237
+ /// DbfsError is the error returned by DBFS service.
238
+ #[derive(Default, Deserialize)]
239
+ struct DbfsError {
240
+ error_code: String,
241
+ message: String,
242
+ }
233
243
 
234
- use http::Response;
235
- use http::StatusCode;
236
- use serde::Deserialize;
244
+ impl Debug for DbfsError {
245
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
246
+ f.debug_struct("DbfsError")
247
+ .field("error_code", &self.error_code)
248
+ // replace `\n` to ` ` for better reading.
249
+ .field("message", &self.message.replace('\n', " "))
250
+ .finish()
251
+ }
252
+ }
237
253
 
238
- use opendal_core::raw::*;
239
- use opendal_core::*;
254
+ /// Context needed to classify an error from this service.
255
+ #[derive(Clone, Copy, Debug)]
256
+ pub(crate) struct ErrorContext {
257
+ service_operation: ServiceOperation,
258
+ }
240
259
 
241
- /// DbfsError is the error returned by DBFS service.
242
- #[derive(Default, Deserialize)]
243
- struct DbfsError {
244
- error_code: String,
245
- message: String,
260
+ impl ErrorContext {
261
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
262
+ Self { service_operation }
246
263
  }
264
+ }
247
265
 
248
- impl Debug for DbfsError {
249
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
250
- f.debug_struct("DbfsError")
251
- .field("error_code", &self.error_code)
252
- // replace `\n` to ` ` for better reading.
253
- .field("message", &self.message.replace('\n', " "))
254
- .finish()
255
- }
266
+ /// Parse an error response using its service request context.
267
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
268
+ let (parts, body) = resp.into_parts();
269
+ let bs = body.to_bytes();
270
+
271
+ let (kind, retryable) = match parts.status {
272
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
273
+ StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
274
+ StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
275
+ StatusCode::INTERNAL_SERVER_ERROR
276
+ | StatusCode::BAD_GATEWAY
277
+ | StatusCode::SERVICE_UNAVAILABLE
278
+ | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
279
+ _ => (ErrorKind::Unexpected, false),
280
+ };
281
+
282
+ let message = match serde_json::from_slice::<DbfsError>(&bs) {
283
+ Ok(dbfs_error) => format!("{:?}", dbfs_error.message),
284
+ Err(_) => String::from_utf8_lossy(&bs).into_owned(),
285
+ };
286
+
287
+ let mut err = Error::new(kind, message);
288
+
289
+ err = err.with_context("service_operation", ctx.service_operation.0);
290
+ err = with_error_response_context(err, parts);
291
+
292
+ if retryable {
293
+ err = err.set_temporary();
256
294
  }
257
295
 
258
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
259
- let (parts, body) = resp.into_parts();
260
- let bs = body.to_bytes();
261
-
262
- let (kind, retryable) = match parts.status {
263
- StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
264
- StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
265
- (ErrorKind::PermissionDenied, false)
266
- }
267
- StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
268
- StatusCode::INTERNAL_SERVER_ERROR
269
- | StatusCode::BAD_GATEWAY
270
- | StatusCode::SERVICE_UNAVAILABLE
271
- | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
272
- _ => (ErrorKind::Unexpected, false),
273
- };
274
-
275
- let message = match serde_json::from_slice::<DbfsError>(&bs) {
276
- Ok(dbfs_error) => format!("{:?}", dbfs_error.message),
277
- Err(_) => String::from_utf8_lossy(&bs).into_owned(),
278
- };
279
-
280
- let mut err = Error::new(kind, message);
281
-
282
- err = with_error_response_context(err, parts);
283
-
284
- if retryable {
285
- err = err.set_temporary();
286
- }
287
-
288
- err
289
- }
296
+ err
290
297
  }
291
-
292
- pub(super) use error::*;
@@ -44,7 +44,10 @@ impl oio::OneShotDelete for DbfsDeleter {
44
44
  match status {
45
45
  // NOTE: Server will return 200 even if the path doesn't exist.
46
46
  StatusCode::OK => Ok(()),
47
- _ => Err(parse_error(resp)),
47
+ _ => Err(parse_error(
48
+ ErrorContext::new(ServiceOperation("Delete")),
49
+ resp,
50
+ )),
48
51
  }
49
52
  }
50
53
  }
@@ -21,8 +21,8 @@ use bytes::Buf;
21
21
  use http::StatusCode;
22
22
  use serde::Deserialize;
23
23
 
24
- use super::core::DbfsCore;
25
24
  use super::core::parse_error;
25
+ use super::core::{DbfsCore, ErrorContext};
26
26
  use opendal_core::raw::*;
27
27
  use opendal_core::*;
28
28
 
@@ -48,7 +48,7 @@ impl oio::PageList for DbfsLister {
48
48
  ctx.done = true;
49
49
  return Ok(());
50
50
  }
51
- let error = parse_error(response);
51
+ let error = parse_error(ErrorContext::new(ServiceOperation("List")), response);
52
52
  return Err(error);
53
53
  }
54
54
 
@@ -59,18 +59,18 @@ impl oio::PageList for DbfsLister {
59
59
  ctx.done = true;
60
60
 
61
61
  for status in decoded_response.files {
62
+ let path = build_rel_path(&self.core.root, &status.path);
62
63
  let entry: oio::Entry = match status.is_dir {
63
64
  true => {
64
- let normalized_path = format!("{}/", status.path);
65
- let mut meta = Metadata::new(EntryMode::DIR);
66
- meta.set_last_modified(Timestamp::from_millisecond(status.modification_time)?);
67
- oio::Entry::new(&normalized_path, meta)
65
+ let normalized_path = format!("{path}/");
66
+ let mut meta = MetadataBuilder::dir();
67
+ meta.last_modified(Timestamp::from_millisecond(status.modification_time)?);
68
+ oio::Entry::new(&normalized_path, meta.build())
68
69
  }
69
70
  false => {
70
- let mut meta = Metadata::new(EntryMode::FILE);
71
- meta.set_last_modified(Timestamp::from_millisecond(status.modification_time)?);
72
- meta.set_content_length(status.file_size as u64);
73
- oio::Entry::new(&status.path, meta)
71
+ let mut meta = MetadataBuilder::file(status.file_size as u64);
72
+ meta.last_modified(Timestamp::from_millisecond(status.modification_time)?);
73
+ oio::Entry::new(&path, meta.build())
74
74
  }
75
75
  };
76
76
  ctx.entries.push_back(entry);