opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -19,8 +19,8 @@ use std::sync::Arc;
19
19
 
20
20
  use http::StatusCode;
21
21
 
22
- use super::core::DbfsCore;
23
22
  use super::core::parse_error;
23
+ use super::core::{DbfsCore, ErrorContext};
24
24
  use opendal_core::raw::*;
25
25
  use opendal_core::*;
26
26
 
@@ -58,8 +58,11 @@ impl oio::OneShotWrite for DbfsWriter {
58
58
 
59
59
  let status = resp.status();
60
60
  match status {
61
- StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
62
- _ => Err(parse_error(resp)),
61
+ StatusCode::CREATED | StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
62
+ _ => Err(parse_error(
63
+ ErrorContext::new(ServiceOperation("Put")),
64
+ resp,
65
+ )),
63
66
  }
64
67
  }
65
68
  }
@@ -32,10 +32,10 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
+ asyncband = { workspace = true }
35
36
  bytes = { workspace = true }
36
37
  http = { workspace = true }
37
- mea = { workspace = true }
38
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
38
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
39
39
  serde = { workspace = true, features = ["derive"] }
40
40
  serde_json = { workspace = true }
41
41
 
@@ -30,7 +30,7 @@ use opendal_core::*;
30
30
 
31
31
  use std::fmt::Debug;
32
32
 
33
- use mea::mutex::Mutex;
33
+ use asyncband::mutex::Mutex;
34
34
 
35
35
  use super::DROPBOX_SCHEME;
36
36
  use super::config::DropboxConfig;
@@ -200,6 +200,7 @@ impl Service for DropboxBackend {
200
200
  type Lister = oio::PageLister<DropboxLister>;
201
201
  type Deleter = oio::OneShotDeleter<DropboxDeleter>;
202
202
  type Copier = oio::OneShotCopier;
203
+ type Composer = ();
203
204
 
204
205
  fn info(&self) -> ServiceInfo {
205
206
  self.core.info.clone()
@@ -250,27 +251,32 @@ impl Service for DropboxBackend {
250
251
  _ => EntryMode::Unknown,
251
252
  };
252
253
 
253
- let mut metadata = Metadata::new(entry_mode);
254
+ let mut metadata = match entry_mode {
255
+ EntryMode::FILE => {
256
+ MetadataBuilder::file(decoded_response.size.ok_or_else(|| {
257
+ Error::new(
258
+ ErrorKind::Unexpected,
259
+ format!("no size found for file {path}"),
260
+ )
261
+ })?)
262
+ }
263
+ EntryMode::DIR => MetadataBuilder::dir(),
264
+ EntryMode::Unknown => MetadataBuilder::unknown(),
265
+ };
254
266
  // Only set last_modified and size if entry_mode is FILE, because Dropbox API
255
267
  // returns last_modified and size only for files.
256
268
  // FYI: https://www.dropbox.com/developers/documentation/http/documentation#files-get_metadata
257
269
  if entry_mode == EntryMode::FILE {
258
270
  let date_utc_last_modified =
259
271
  decoded_response.client_modified.parse::<Timestamp>()?;
260
- metadata.set_last_modified(date_utc_last_modified);
261
-
262
- if let Some(size) = decoded_response.size {
263
- metadata.set_content_length(size);
264
- } else {
265
- return Err(Error::new(
266
- ErrorKind::Unexpected,
267
- format!("no size found for file {path}"),
268
- ));
269
- }
272
+ metadata.last_modified(date_utc_last_modified);
270
273
  }
271
- Ok(RpStat::new(metadata))
274
+ Ok(RpStat::new(metadata.build()))
272
275
  }
273
- _ => Err(parse_error(resp)),
276
+ _ => Err(parse_error(
277
+ ErrorContext::new(ServiceOperation("GetMetadata")),
278
+ resp,
279
+ )),
274
280
  }
275
281
  }
276
282
  fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -330,7 +336,6 @@ impl Service for DropboxBackend {
330
336
  from: &str,
331
337
  to: &str,
332
338
  _: OpCopy,
333
- _opts: OpCopier,
334
339
  ) -> Result<Self::Copier> {
335
340
  let core = self.core.clone();
336
341
  let ctx = ctx.clone();
@@ -342,14 +347,16 @@ impl Service for DropboxBackend {
342
347
  let status = resp.status();
343
348
 
344
349
  match status {
345
- StatusCode::OK => Ok(Metadata::default()),
346
- _ => {
347
- let err = parse_error(resp);
348
- match err.kind() {
349
- ErrorKind::NotFound => Ok(Metadata::default()),
350
- _ => Err(err),
351
- }
350
+ StatusCode::OK => {
351
+ let decoded_response: DropboxMetadataResponse =
352
+ serde_json::from_reader(resp.into_body().reader())
353
+ .map_err(new_json_deserialize_error)?;
354
+ DropboxWriter::parse_metadata(decoded_response)
352
355
  }
356
+ _ => Err(parse_error(
357
+ ErrorContext::new(ServiceOperation("CopyFile")),
358
+ resp,
359
+ )),
353
360
  }
354
361
  }))
355
362
  }
@@ -368,7 +375,7 @@ impl Service for DropboxBackend {
368
375
  match status {
369
376
  StatusCode::OK => Ok(RpRename::default()),
370
377
  _ => {
371
- let err = parse_error(resp);
378
+ let err = parse_error(ErrorContext::new(ServiceOperation("MoveFile")), resp);
372
379
  match err.kind() {
373
380
  ErrorKind::NotFound => Ok(RpRename::default()),
374
381
  _ => Err(err),
@@ -18,6 +18,7 @@
18
18
  use std::fmt::Debug;
19
19
  use std::sync::Arc;
20
20
 
21
+ use asyncband::mutex::Mutex;
21
22
  use bytes::Buf;
22
23
  use bytes::Bytes;
23
24
  use http::Request;
@@ -26,7 +27,6 @@ use http::StatusCode;
26
27
  use http::header;
27
28
  use http::header::CONTENT_LENGTH;
28
29
  use http::header::CONTENT_TYPE;
29
- use mea::mutex::Mutex;
30
30
  use serde::Deserialize;
31
31
  use serde::Serialize;
32
32
 
@@ -224,7 +224,7 @@ impl DropboxCore {
224
224
  match status {
225
225
  StatusCode::OK => Ok(RpCreateDir::default()),
226
226
  _ => {
227
- let err = parse_error(resp);
227
+ let err = parse_error(ErrorContext::new(ServiceOperation("CreateFolder")), resp);
228
228
  match err.kind() {
229
229
  ErrorKind::AlreadyExists => Ok(RpCreateDir::default()),
230
230
  _ => Err(err),
@@ -529,75 +529,77 @@ pub struct DropboxListResponse {
529
529
  pub has_more: bool,
530
530
  }
531
531
 
532
- mod error {
533
- use http::Response;
534
- use http::StatusCode;
535
- use serde::Deserialize;
532
+ #[derive(Default, Debug, Deserialize)]
533
+ #[serde(default)]
534
+ pub struct DropboxErrorResponse {
535
+ pub error_summary: String,
536
+ }
536
537
 
537
- use opendal_core::raw::*;
538
- use opendal_core::*;
538
+ /// Context needed to classify an error from this service.
539
+ #[derive(Clone, Copy, Debug)]
540
+ pub(crate) struct ErrorContext {
541
+ service_operation: ServiceOperation,
542
+ }
539
543
 
540
- #[derive(Default, Debug, Deserialize)]
541
- #[serde(default)]
542
- pub struct DropboxErrorResponse {
543
- pub error_summary: String,
544
+ impl ErrorContext {
545
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
546
+ Self { service_operation }
544
547
  }
548
+ }
545
549
 
546
- /// Parse error response into Error.
547
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
548
- let (parts, body) = resp.into_parts();
549
- let bs = body.to_bytes();
550
-
551
- let (mut kind, mut retryable) = match parts.status {
552
- StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
553
- StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
554
- StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
555
- StatusCode::INTERNAL_SERVER_ERROR
556
- | StatusCode::BAD_GATEWAY
557
- | StatusCode::SERVICE_UNAVAILABLE
558
- | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
559
- _ => (ErrorKind::Unexpected, false),
560
- };
561
-
562
- let (message, dropbox_err) = serde_json::from_slice::<DropboxErrorResponse>(&bs)
563
- .map(|dropbox_err| (format!("{dropbox_err:?}"), Some(dropbox_err)))
564
- .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
565
-
566
- if let Some(dropbox_err) = dropbox_err {
567
- (kind, retryable) = parse_dropbox_error_summary(&dropbox_err.error_summary)
568
- .unwrap_or((kind, retryable));
569
- }
570
-
571
- let mut err = Error::new(kind, message);
550
+ /// Parse an error response using its service request context.
551
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
552
+ let (parts, body) = resp.into_parts();
553
+ let bs = body.to_bytes();
554
+
555
+ let (mut kind, mut retryable) = match parts.status {
556
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
557
+ StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
558
+ StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
559
+ StatusCode::INTERNAL_SERVER_ERROR
560
+ | StatusCode::BAD_GATEWAY
561
+ | StatusCode::SERVICE_UNAVAILABLE
562
+ | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
563
+ _ => (ErrorKind::Unexpected, false),
564
+ };
565
+
566
+ let (message, dropbox_err) = serde_json::from_slice::<DropboxErrorResponse>(&bs)
567
+ .map(|dropbox_err| (format!("{dropbox_err:?}"), Some(dropbox_err)))
568
+ .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
569
+
570
+ if let Some(dropbox_err) = dropbox_err {
571
+ (kind, retryable) =
572
+ parse_dropbox_error_summary(&dropbox_err.error_summary).unwrap_or((kind, retryable));
573
+ }
572
574
 
573
- err = with_error_response_context(err, parts);
575
+ let mut err = Error::new(kind, message);
574
576
 
575
- if retryable {
576
- err = err.set_temporary();
577
- }
577
+ err = err.with_context("service_operation", ctx.service_operation.0);
578
+ err = with_error_response_context(err, parts);
578
579
 
579
- err
580
+ if retryable {
581
+ err = err.set_temporary();
580
582
  }
581
583
 
582
- /// We cannot get the error type from the response header when the status code is 409.
583
- /// Because Dropbox API v2 will put error summary in the response body,
584
- /// we need to parse it to get the correct error type and then error kind.
585
- ///
586
- /// See <https://www.dropbox.com/developers/documentation/http/documentation#error-handling>
587
- pub fn parse_dropbox_error_summary(summary: &str) -> Option<(ErrorKind, bool)> {
588
- if summary.starts_with("path/not_found")
589
- || summary.starts_with("path_lookup/not_found")
590
- || summary.starts_with("from_lookup/not_found")
591
- {
592
- Some((ErrorKind::NotFound, false))
593
- } else if summary.starts_with("path/conflict") {
594
- Some((ErrorKind::AlreadyExists, false))
595
- } else if summary.starts_with("too_many_write_operations") {
596
- Some((ErrorKind::RateLimited, true))
597
- } else {
598
- None
599
- }
600
- }
584
+ err
601
585
  }
602
586
 
603
- pub(super) use error::*;
587
+ /// We cannot get the error type from the response header when the status code is 409.
588
+ /// Because Dropbox API v2 will put error summary in the response body,
589
+ /// we need to parse it to get the correct error type and then error kind.
590
+ ///
591
+ /// See <https://www.dropbox.com/developers/documentation/http/documentation#error-handling>
592
+ pub fn parse_dropbox_error_summary(summary: &str) -> Option<(ErrorKind, bool)> {
593
+ if summary.starts_with("path/not_found")
594
+ || summary.starts_with("path_lookup/not_found")
595
+ || summary.starts_with("from_lookup/not_found")
596
+ {
597
+ Some((ErrorKind::NotFound, false))
598
+ } else if summary.starts_with("path/conflict") {
599
+ Some((ErrorKind::AlreadyExists, false))
600
+ } else if summary.starts_with("too_many_write_operations") {
601
+ Some((ErrorKind::RateLimited, true))
602
+ } else {
603
+ None
604
+ }
605
+ }
@@ -44,7 +44,7 @@ impl oio::OneShotDelete for DropboxDeleter {
44
44
  match status {
45
45
  StatusCode::OK => Ok(()),
46
46
  _ => {
47
- let err = parse_error(resp);
47
+ let err = parse_error(ErrorContext::new(ServiceOperation("DeleteFile")), resp);
48
48
  match err.kind() {
49
49
  ErrorKind::NotFound => Ok(()),
50
50
  _ => Err(err),
@@ -68,7 +68,14 @@ impl oio::PageList for DropboxLister {
68
68
  let status_code = response.status();
69
69
 
70
70
  if !status_code.is_success() {
71
- let error = parse_error(response);
71
+ let error = parse_error(
72
+ ErrorContext::new(if ctx.token.is_empty() {
73
+ ServiceOperation("ListFolder")
74
+ } else {
75
+ ServiceOperation("ListFolderContinue")
76
+ }),
77
+ response,
78
+ );
72
79
 
73
80
  let result = match error.kind() {
74
81
  ErrorKind::NotFound => Ok(()),
@@ -91,7 +98,13 @@ impl oio::PageList for DropboxLister {
91
98
  };
92
99
 
93
100
  let mut name = entry.name;
94
- let mut meta = Metadata::new(entry_mode);
101
+ let mut meta = match entry_mode {
102
+ EntryMode::FILE => entry
103
+ .size
104
+ .map_or_else(MetadataBuilder::unknown, MetadataBuilder::file),
105
+ EntryMode::DIR => MetadataBuilder::dir(),
106
+ EntryMode::Unknown => MetadataBuilder::unknown(),
107
+ };
95
108
 
96
109
  // Dropbox will return folder names that do not end with '/'.
97
110
  if entry_mode == EntryMode::DIR && !name.ends_with('/') {
@@ -101,14 +114,10 @@ impl oio::PageList for DropboxLister {
101
114
  // The behavior here aligns with Dropbox's stat function.
102
115
  if entry_mode == EntryMode::FILE {
103
116
  let date_utc_last_modified = entry.client_modified.parse::<Timestamp>()?;
104
- meta.set_last_modified(date_utc_last_modified);
105
-
106
- if let Some(size) = entry.size {
107
- meta.set_content_length(size);
108
- }
117
+ meta.last_modified(date_utc_last_modified);
109
118
  }
110
119
 
111
- ctx.entries.push_back(oio::Entry::with(name, meta));
120
+ ctx.entries.push_back(oio::Entry::with(name, meta.build()));
112
121
  }
113
122
 
114
123
  if decoded_response.has_more {
@@ -65,7 +65,10 @@ impl oio::StreamRead for DropboxReader {
65
65
  _ => {
66
66
  let (part, mut body) = resp.into_parts();
67
67
  let buf = body.to_buffer().await?;
68
- return Err(parse_error(Response::from_parts(part, buf)));
68
+ return Err(parse_error(
69
+ ErrorContext::new(ServiceOperation("DownloadFile")),
70
+ Response::from_parts(part, buf),
71
+ ));
69
72
  }
70
73
  };
71
74
 
@@ -21,7 +21,7 @@ use bytes::Buf;
21
21
  use http::StatusCode;
22
22
 
23
23
  use super::core::parse_error;
24
- use super::core::{DropboxCore, DropboxMetadataResponse};
24
+ use super::core::{DropboxCore, DropboxMetadataResponse, ErrorContext};
25
25
  use opendal_core::raw::*;
26
26
  use opendal_core::*;
27
27
 
@@ -42,30 +42,30 @@ impl DropboxWriter {
42
42
  }
43
43
  }
44
44
 
45
- fn parse_metadata(decoded_response: DropboxMetadataResponse) -> Result<Metadata> {
46
- let mut metadata = Metadata::default();
45
+ pub(crate) fn parse_metadata(decoded_response: DropboxMetadataResponse) -> Result<Metadata> {
46
+ let mut metadata = MetadataBuilder::unknown();
47
47
 
48
48
  if let Some(size) = decoded_response.size {
49
- metadata.set_content_length(size);
49
+ metadata.set_file(size);
50
50
  }
51
51
 
52
52
  if let Some(content_hash) = decoded_response.content_hash {
53
- metadata.set_etag(&content_hash);
53
+ metadata.etag(&content_hash);
54
54
  }
55
55
 
56
56
  if let Some(rev) = decoded_response.rev {
57
- metadata.set_version(&rev);
57
+ metadata.version(&rev);
58
58
  }
59
59
 
60
60
  if let Some(server_modified) = decoded_response.server_modified {
61
61
  let date_utc = server_modified.parse::<Timestamp>()?;
62
- metadata.set_last_modified(date_utc);
62
+ metadata.last_modified(date_utc);
63
63
  } else {
64
64
  let date_utc = decoded_response.client_modified.parse::<Timestamp>()?;
65
- metadata.set_last_modified(date_utc);
65
+ metadata.last_modified(date_utc);
66
66
  }
67
67
 
68
- Ok(metadata)
68
+ Ok(metadata.build())
69
69
  }
70
70
  }
71
71
 
@@ -84,7 +84,10 @@ impl oio::OneShotWrite for DropboxWriter {
84
84
  let metadata = DropboxWriter::parse_metadata(decoded_response)?;
85
85
  Ok(metadata)
86
86
  }
87
- _ => Err(parse_error(resp)),
87
+ _ => Err(parse_error(
88
+ ErrorContext::new(ServiceOperation("UploadFile")),
89
+ resp,
90
+ )),
88
91
  }
89
92
  }
90
93
  }
@@ -34,7 +34,7 @@ all-features = true
34
34
  [dependencies]
35
35
  etcd-client = { version = "0.19.0", features = ["tls"] }
36
36
  fastpool = "1.0.2"
37
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
38
38
  serde = { workspace = true, features = ["derive"] }
39
39
  tokio = { workspace = true, features = ["time"] }
40
40
 
@@ -209,6 +209,7 @@ impl Service for EtcdBackend {
209
209
  type Lister = oio::HierarchyLister<EtcdLazyLister>;
210
210
  type Deleter = oio::OneShotDeleter<EtcdDeleter>;
211
211
  type Copier = ();
212
+ type Composer = ();
212
213
 
213
214
  fn info(&self) -> ServiceInfo {
214
215
  self.info.clone()
@@ -246,9 +247,12 @@ impl Service for EtcdBackend {
246
247
  // First check if it's a direct key
247
248
  match self.core.get(&abs_path).await? {
248
249
  Some(buffer) => {
249
- let mut metadata = Metadata::new(EntryMode::from_path(&abs_path));
250
- metadata.set_content_length(buffer.len() as u64);
251
- Ok(RpStat::new(metadata))
250
+ let metadata = if abs_path.ends_with('/') {
251
+ MetadataBuilder::dir()
252
+ } else {
253
+ MetadataBuilder::file(buffer.len() as u64)
254
+ };
255
+ Ok(RpStat::new(metadata.build()))
252
256
  }
253
257
  None => {
254
258
  // Check if it's a directory by looking for keys with this prefix
@@ -262,7 +266,7 @@ impl Service for EtcdBackend {
262
266
  let has_children = self.core.has_prefix(&prefix).await?;
263
267
  if has_children {
264
268
  // Has children, it's a directory
265
- let metadata = Metadata::new(EntryMode::DIR);
269
+ let metadata = MetadataBuilder::dir().build();
266
270
  Ok(RpStat::new(metadata))
267
271
  } else {
268
272
  Err(Error::new(ErrorKind::NotFound, "path not found"))
@@ -324,7 +328,6 @@ impl Service for EtcdBackend {
324
328
  _from: &str,
325
329
  _to: &str,
326
330
  _args: OpCopy,
327
- _opts: OpCopier,
328
331
  ) -> Result<Self::Copier> {
329
332
  Err(Error::new(
330
333
  ErrorKind::Unsupported,
@@ -148,16 +148,10 @@ impl EtcdCore {
148
148
  }
149
149
  }
150
150
 
151
- mod error {
152
- use etcd_client::Error as EtcdError;
151
+ use etcd_client::Error as EtcdError;
153
152
 
154
- use opendal_core::{Error, ErrorKind};
155
-
156
- pub fn format_etcd_error(e: EtcdError) -> Error {
157
- Error::new(ErrorKind::Unexpected, e.to_string().as_str())
158
- .set_source(e)
159
- .set_temporary()
160
- }
153
+ pub fn format_etcd_error(e: EtcdError) -> Error {
154
+ Error::new(ErrorKind::Unexpected, e.to_string().as_str())
155
+ .set_source(e)
156
+ .set_temporary()
161
157
  }
162
-
163
- pub(super) use error::*;
@@ -69,7 +69,12 @@ impl oio::List for EtcdLister {
69
69
  if key.starts_with(&self.path) {
70
70
  let path = build_rel_path(&self.root, &key);
71
71
 
72
- let entry = Entry::new(&path, Metadata::new(EntryMode::from_path(&key)));
72
+ let metadata = if key.ends_with('/') {
73
+ MetadataBuilder::dir()
74
+ } else {
75
+ MetadataBuilder::unknown()
76
+ };
77
+ let entry = Entry::new(&path, metadata.build());
73
78
  return Ok(Some(entry));
74
79
  }
75
80
  }
@@ -44,7 +44,10 @@ impl oio::StreamRead for EtcdReader {
44
44
  Some(buffer) => {
45
45
  let total_size = buffer.len() as u64;
46
46
  let sliced_buffer = buffer.slice(range.to_content_range(buffer.len())?);
47
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(total_size);
47
+ let metadata = {
48
+ let metadata = MetadataBuilder::file(total_size);
49
+ metadata.build()
50
+ };
48
51
 
49
52
  Ok((
50
53
  RpRead::new(metadata),
@@ -48,10 +48,13 @@ impl oio::Write for EtcdWriter {
48
48
 
49
49
  self.core.set(&self.path, buf.clone()).await?;
50
50
 
51
- let mut metadata = Metadata::new(EntryMode::from_path(&self.path));
52
- metadata.set_content_length(buf.len() as u64);
51
+ let metadata = if self.path.ends_with('/') {
52
+ MetadataBuilder::dir()
53
+ } else {
54
+ MetadataBuilder::file(buf.len() as u64)
55
+ };
53
56
 
54
- Ok(metadata)
57
+ Ok(metadata.build())
55
58
  }
56
59
 
57
60
  async fn abort(&mut self) -> Result<()> {
@@ -36,7 +36,7 @@ foundationdb = { version = "0.11.0", features = [
36
36
  "embedded-fdb-include",
37
37
  "fdb-7_3",
38
38
  ] }
39
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
40
40
  serde = { workspace = true, features = ["derive"] }
41
41
 
42
42
  [dev-dependencies]
@@ -126,6 +126,7 @@ impl Service for FoundationdbBackend {
126
126
  type Lister = ();
127
127
  type Deleter = oio::OneShotDeleter<FoundationdbDeleter>;
128
128
  type Copier = ();
129
+ type Composer = ();
129
130
 
130
131
  fn info(&self) -> ServiceInfo {
131
132
  self.info.clone()
@@ -151,13 +152,14 @@ impl Service for FoundationdbBackend {
151
152
  let p = build_abs_path(&self.root, path);
152
153
 
153
154
  if p == build_abs_path(&self.root, "") {
154
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
155
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
155
156
  } else {
156
157
  let bs = self.core.get(&p).await?;
157
158
  match bs {
158
- Some(bs) => Ok(RpStat::new(
159
- Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
160
- )),
159
+ Some(bs) => Ok(RpStat::new({
160
+ let metadata = MetadataBuilder::file(bs.len() as u64);
161
+ metadata.build()
162
+ })),
161
163
  None => Err(Error::new(
162
164
  ErrorKind::NotFound,
163
165
  "kv not found in foundationdb",
@@ -210,7 +212,6 @@ impl Service for FoundationdbBackend {
210
212
  _from: &str,
211
213
  _to: &str,
212
214
  _args: OpCopy,
213
- _opts: OpCopier,
214
215
  ) -> Result<Self::Copier> {
215
216
  Err(Error::new(
216
217
  ErrorKind::Unsupported,
@@ -49,7 +49,10 @@ impl oio::StreamRead for FoundationdbReader {
49
49
  }
50
50
  };
51
51
  let content = bs.slice(range.to_content_range(bs.len())?);
52
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
52
+ let metadata = {
53
+ let metadata = MetadataBuilder::file(bs.len() as u64);
54
+ metadata.build()
55
+ };
53
56
  Ok((
54
57
  RpRead::new(metadata),
55
58
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -48,8 +48,12 @@ impl oio::Write for FoundationdbWriter {
48
48
  let length = buf.len() as u64;
49
49
  self.core.set(&self.path, buf).await?;
50
50
 
51
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
52
- Ok(meta)
51
+ let metadata = if self.path.ends_with('/') {
52
+ MetadataBuilder::dir()
53
+ } else {
54
+ MetadataBuilder::file(length)
55
+ };
56
+ Ok(metadata.build())
53
57
  }
54
58
 
55
59
  async fn abort(&mut self) -> Result<()> {
@@ -34,12 +34,12 @@ all-features = true
34
34
  [dependencies]
35
35
  foyer = { version = "0.22.3" }
36
36
  log = { workspace = true }
37
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
38
38
  serde = { workspace = true, features = ["derive"] }
39
39
 
40
40
  [dev-dependencies]
41
- opendal = { path = "../..", version = "0.58.1", features = ["services-memory"] }
42
- opendal-core = { path = "../../core", version = "0.58.1", features = [
41
+ opendal = { path = "../..", version = "0.59.0", features = ["services-memory"] }
42
+ opendal-core = { path = "../../core", version = "0.59.0", features = [
43
43
  "services-memory",
44
44
  ] }
45
45
  size = "0.5"