opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -18,9 +18,8 @@
18
18
  use std::sync::Arc;
19
19
 
20
20
  use super::core::GoosefsCore;
21
- use opendal_core::EntryMode;
22
21
  use opendal_core::ErrorKind;
23
- use opendal_core::Metadata;
22
+ use opendal_core::MetadataBuilder;
24
23
  use opendal_core::Result;
25
24
  use opendal_core::raw::oio::Entry;
26
25
  use opendal_core::raw::*;
@@ -65,7 +64,7 @@ impl oio::PageList for GoosefsLister {
65
64
  // synthesis for it.
66
65
  if !self.path.is_empty() && self.path.ends_with('/') {
67
66
  ctx.entries
68
- .push_back(Entry::new(&self.path, Metadata::new(EntryMode::DIR)));
67
+ .push_back(Entry::new(&self.path, MetadataBuilder::dir().build()));
69
68
  }
70
69
 
71
70
  for file_info in file_infos {
@@ -17,7 +17,7 @@
17
17
 
18
18
  use super::backend::*;
19
19
  use super::core::GoosefsCore;
20
- use super::core::parse_error;
20
+ use super::core::{ErrorContext, parse_error};
21
21
  use goosefs_sdk::io::GoosefsFileReader as SdkReader;
22
22
  use opendal_core::raw::*;
23
23
  use opendal_core::*;
@@ -103,7 +103,11 @@ impl oio::ReadStream for GoosefsReadStream {
103
103
  }
104
104
  let reader = self.inner.as_mut().expect("inner was just set");
105
105
 
106
- match reader.read_next_block().await.map_err(parse_error)? {
106
+ match reader
107
+ .read_next_block()
108
+ .await
109
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("ReadNextBlock")), err))?
110
+ {
107
111
  Some(block) => Ok(Buffer::from(block)),
108
112
  None => {
109
113
  self.done = true;
@@ -138,7 +142,7 @@ impl oio::StreamRead for GoosefsReader {
138
142
  Some(
139
143
  backend
140
144
  .core
141
- .file_info_to_metadata(&file_info)
145
+ .file_info_to_metadata(&file_info)?
142
146
  .content_length(),
143
147
  )
144
148
  } else {
@@ -24,7 +24,7 @@ use std::time::UNIX_EPOCH;
24
24
  use goosefs_sdk::io::GoosefsFileWriter as ClientWriter;
25
25
 
26
26
  use super::core::GoosefsCore;
27
- use super::core::parse_error;
27
+ use super::core::{ErrorContext, parse_error};
28
28
  use opendal_core::raw::*;
29
29
  use opendal_core::*;
30
30
 
@@ -226,7 +226,10 @@ impl oio::Write for GoosefsWriter {
226
226
  // Iterator yields one `Bytes` per segment; the SDK writer
227
227
  // takes `&[u8]`, so each chunk flows through without a copy.
228
228
  for chunk in bs {
229
- writer.write(&chunk).await.map_err(parse_error)?;
229
+ writer
230
+ .write(&chunk)
231
+ .await
232
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Write")), err))?;
230
233
  }
231
234
  Ok(())
232
235
  }
@@ -235,7 +238,11 @@ impl oio::Write for GoosefsWriter {
235
238
  if let Some(mut writer) = self.writer.take() {
236
239
  // Commit the temp inode on the master, then publish it
237
240
  // onto the caller's final path via `finalize_rename`.
238
- if let Err(e) = writer.close().await.map_err(parse_error) {
241
+ if let Err(e) = writer
242
+ .close()
243
+ .await
244
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Close")), err))
245
+ {
239
246
  // Close failed — the temp is in an indeterminate state
240
247
  // on the master; best-effort sweep to avoid leaks.
241
248
  if let Some(tmp) = self.tmp_path.take() {
@@ -244,12 +251,12 @@ impl oio::Write for GoosefsWriter {
244
251
  return Err(e);
245
252
  }
246
253
  // Capture file_id before rename; Master rename keeps the same inode id.
247
- let mut meta = Metadata::default();
254
+ let mut meta = MetadataBuilder::unknown();
248
255
  if let Some(fid) = writer.file_info().file_id {
249
- meta.set_etag(&fid.to_string());
256
+ meta.etag(fid.to_string());
250
257
  }
251
258
  self.finalize_rename().await?;
252
- return Ok(meta);
259
+ return Ok(meta.build());
253
260
  }
254
261
 
255
262
  // Zero-write path: the caller closed without ever calling
@@ -267,17 +274,21 @@ impl oio::Write for GoosefsWriter {
267
274
 
268
275
  let tmp = Self::make_tmp_path(&self.path);
269
276
  let mut w = self.core.create_writer(&tmp).await?;
270
- if let Err(e) = w.close().await.map_err(parse_error) {
277
+ if let Err(e) = w
278
+ .close()
279
+ .await
280
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Close")), err))
281
+ {
271
282
  let _ = self.core.delete(&tmp).await;
272
283
  return Err(e);
273
284
  }
274
- let mut meta = Metadata::default();
285
+ let mut meta = MetadataBuilder::unknown();
275
286
  if let Some(fid) = w.file_info().file_id {
276
- meta.set_etag(&fid.to_string());
287
+ meta.etag(fid.to_string());
277
288
  }
278
289
  self.tmp_path = Some(tmp);
279
290
  self.finalize_rename().await?;
280
- Ok(meta)
291
+ Ok(meta.build())
281
292
  }
282
293
 
283
294
  async fn abort(&mut self) -> Result<()> {
@@ -197,3 +197,40 @@ async fn concurrent_write_if_not_exists_exactly_one_wins() {
197
197
  let _ = op.delete(&p).await;
198
198
  }
199
199
  }
200
+
201
+ /// Nested write: CreateFile(recursive) on the temp sibling creates the parent,
202
+ /// so publish rename must succeed without an eager CreateDirectory.
203
+ #[tokio::test]
204
+ async fn write_nested_path_creates_parents() {
205
+ let Some(op) = maybe_operator() else {
206
+ eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR unset");
207
+ return;
208
+ };
209
+
210
+ let path = format!("{}/nested/file", unique("nested-write"));
211
+ op.write(&path, "nested-body")
212
+ .await
213
+ .expect("write into missing parents");
214
+ let body = op.read(&path).await.expect("read").to_bytes();
215
+ assert_eq!(&body[..], b"nested-body");
216
+ let _ = op.delete(&path).await;
217
+ }
218
+
219
+ /// Public rename still creates a destination parent that does not exist yet.
220
+ #[tokio::test]
221
+ async fn rename_creates_missing_destination_parent() {
222
+ let Some(op) = maybe_operator() else {
223
+ eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR unset");
224
+ return;
225
+ };
226
+
227
+ let src = unique("mkdir-src");
228
+ op.write(&src, "payload").await.expect("write src");
229
+ let dst = format!("{}/nested/file", unique("mkdir-dst"));
230
+ op.rename(&src, &dst)
231
+ .await
232
+ .expect("rename should create missing parent");
233
+ let body = op.read(&dst).await.expect("read dst").to_bytes();
234
+ assert_eq!(&body[..], b"payload");
235
+ let _ = op.delete(&dst).await;
236
+ }
@@ -32,10 +32,10 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
+ asyncband = { workspace = true }
35
36
  futures = { workspace = true }
36
- mea = { workspace = true }
37
37
  mongodb = "3.3.0"
38
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
38
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
39
39
  serde = { workspace = true, features = ["derive"] }
40
40
 
41
41
  [dev-dependencies]
@@ -17,7 +17,7 @@
17
17
 
18
18
  use std::sync::Arc;
19
19
 
20
- use mea::once::OnceCell;
20
+ use asyncband::once::OnceCell;
21
21
  use opendal_core::raw::*;
22
22
  use opendal_core::*;
23
23
 
@@ -93,7 +93,7 @@ impl GridfsBuilder {
93
93
  self
94
94
  }
95
95
 
96
- /// Set the chunk size of the MongoDB GridFs service used to break the user file into chunks.
96
+ /// Set the chunk size in bytes for MongoDB GridFs service. We break the user file into chunks by size.
97
97
  ///
98
98
  /// Default to `255 KiB` if not specified.
99
99
  pub fn chunk_size(mut self, chunk_size: u32) -> Self {
@@ -104,6 +104,9 @@ impl GridfsBuilder {
104
104
  }
105
105
  }
106
106
 
107
+ /// Default GridFS chunk size in bytes.
108
+ const DEFAULT_CHUNK_SIZE_BYTES: u32 = 255 * 1024; // 255 KiB
109
+
107
110
  impl Builder for GridfsBuilder {
108
111
  type Config = GridfsConfig;
109
112
 
@@ -128,7 +131,7 @@ impl Builder for GridfsBuilder {
128
131
  Some(v) => v.clone(),
129
132
  None => "fs".to_string(),
130
133
  };
131
- let chunk_size = self.config.chunk_size.unwrap_or(255);
134
+ let chunk_size = self.config.chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE_BYTES);
132
135
 
133
136
  let root = normalize_root(
134
137
  self.config
@@ -196,6 +199,7 @@ impl Service for GridfsBackend {
196
199
  type Lister = ();
197
200
  type Deleter = oio::OneShotDeleter<GridfsDeleter>;
198
201
  type Copier = ();
202
+ type Composer = ();
199
203
 
200
204
  fn info(&self) -> ServiceInfo {
201
205
  self.info.clone()
@@ -221,12 +225,13 @@ impl Service for GridfsBackend {
221
225
  let p = build_abs_path(&self.root, path);
222
226
 
223
227
  if p == build_abs_path(&self.root, "") {
224
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
228
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
225
229
  } else {
226
230
  match self.core.get_length(&p).await? {
227
- Some(len) => Ok(RpStat::new(
228
- Metadata::new(EntryMode::FILE).with_content_length(len as u64),
229
- )),
231
+ Some(len) => Ok(RpStat::new({
232
+ let metadata = MetadataBuilder::file(len as u64);
233
+ metadata.build()
234
+ })),
230
235
  None => Err(Error::new(ErrorKind::NotFound, "kv not found in gridfs")),
231
236
  }
232
237
  }
@@ -276,7 +281,6 @@ impl Service for GridfsBackend {
276
281
  _from: &str,
277
282
  _to: &str,
278
283
  _args: OpCopy,
279
- _opts: OpCopier,
280
284
  ) -> Result<Self::Copier> {
281
285
  Err(Error::new(
282
286
  ErrorKind::Unsupported,
@@ -17,9 +17,9 @@
17
17
 
18
18
  use std::fmt::Debug;
19
19
 
20
+ use asyncband::once::OnceCell;
20
21
  use futures::AsyncReadExt;
21
22
  use futures::AsyncWriteExt;
22
- use mea::once::OnceCell;
23
23
  use mongodb::bson::doc;
24
24
  use mongodb::gridfs::{FilesCollectionDocument, GridFsBucket};
25
25
  use mongodb::options::ClientOptions;
@@ -46,7 +46,10 @@ impl oio::StreamRead for GridfsReader {
46
46
  }
47
47
  };
48
48
  let content = bs.slice(range.to_content_range(bs.len())?);
49
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
49
+ let metadata = {
50
+ let metadata = MetadataBuilder::file(bs.len() as u64);
51
+ metadata.build()
52
+ };
50
53
  Ok((
51
54
  RpRead::new(metadata),
52
55
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -49,8 +49,12 @@ impl oio::Write for GridfsWriter {
49
49
  let length = buf.len() as u64;
50
50
  self.core.set(&self.path, buf).await?;
51
51
 
52
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
53
- Ok(meta)
52
+ let metadata = if self.path.ends_with('/') {
53
+ MetadataBuilder::dir()
54
+ } else {
55
+ MetadataBuilder::file(length)
56
+ };
57
+ Ok(metadata.build())
54
58
  }
55
59
 
56
60
  async fn abort(&mut self) -> Result<()> {
@@ -32,7 +32,7 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
35
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
36
36
 
37
37
  bytes = { workspace = true }
38
38
  futures = { workspace = true }
@@ -196,6 +196,7 @@ impl Service for HdfsBackend {
196
196
  type Lister = Option<HdfsLister>;
197
197
  type Deleter = oio::OneShotDeleter<HdfsDeleter>;
198
198
  type Copier = ();
199
+ type Composer = ();
199
200
 
200
201
  fn info(&self) -> ServiceInfo {
201
202
  self.core.info.clone()
@@ -260,7 +261,6 @@ impl Service for HdfsBackend {
260
261
  _from: &str,
261
262
  _to: &str,
262
263
  _args: OpCopy,
263
- _opts: OpCopier,
264
264
  ) -> Result<Self::Copier> {
265
265
  Err(Error::new(
266
266
  ErrorKind::Unsupported,
@@ -72,11 +72,14 @@ impl HdfsCore {
72
72
  } else {
73
73
  EntryMode::Unknown
74
74
  };
75
- let mut m = Metadata::new(mode);
76
- m.set_content_length(meta.len());
77
- m.set_last_modified(Timestamp::try_from(meta.modified())?);
75
+ let mut m = match mode {
76
+ EntryMode::FILE => MetadataBuilder::file(meta.len()),
77
+ EntryMode::DIR => MetadataBuilder::dir(),
78
+ EntryMode::Unknown => MetadataBuilder::unknown(),
79
+ };
80
+ m.last_modified(Timestamp::try_from(meta.modified())?);
78
81
 
79
- Ok(m)
82
+ Ok(m.build())
80
83
  }
81
84
 
82
85
  pub async fn hdfs_open(&self, path: &str) -> Result<hdrs::File> {
@@ -15,8 +15,7 @@
15
15
  // specific language governing permissions and limitations
16
16
  // under the License.
17
17
 
18
- use opendal_core::EntryMode;
19
- use opendal_core::Metadata;
18
+ use opendal_core::MetadataBuilder;
20
19
  use opendal_core::Result;
21
20
  use opendal_core::raw::*;
22
21
 
@@ -43,7 +42,7 @@ impl HdfsLister {
43
42
  impl oio::List for HdfsLister {
44
43
  async fn next(&mut self) -> Result<Option<oio::Entry>> {
45
44
  if let Some(path) = self.current_path.take() {
46
- return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
45
+ return Ok(Some(oio::Entry::new(&path, MetadataBuilder::dir().build())));
47
46
  }
48
47
 
49
48
  let de = match self.rd.next() {
@@ -54,15 +53,14 @@ impl oio::List for HdfsLister {
54
53
  let path = build_rel_path(&self.root, de.path());
55
54
 
56
55
  let entry = if de.is_file() {
57
- let meta = Metadata::new(EntryMode::FILE)
58
- .with_content_length(de.len())
59
- .with_last_modified(Timestamp::try_from(de.modified())?);
60
- oio::Entry::new(&path, meta)
56
+ let mut meta = MetadataBuilder::file(de.len());
57
+ meta.last_modified(Timestamp::try_from(de.modified())?);
58
+ oio::Entry::new(&path, meta.build())
61
59
  } else if de.is_dir() {
62
60
  // Make sure we are returning the correct path.
63
- oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
61
+ oio::Entry::new(&format!("{path}/"), MetadataBuilder::dir().build())
64
62
  } else {
65
- oio::Entry::new(&path, Metadata::new(EntryMode::Unknown))
63
+ oio::Entry::new(&path, MetadataBuilder::unknown().build())
66
64
  };
67
65
 
68
66
  Ok(Some(entry))
@@ -92,7 +92,11 @@ impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
92
92
  .map_err(new_std_io_error)?
93
93
  }
94
94
 
95
- Ok(Metadata::default().with_content_length(self.size))
95
+ Ok({
96
+ let mut metadata = MetadataBuilder::unknown();
97
+ metadata.set_file(self.size);
98
+ metadata.build()
99
+ })
96
100
  }
97
101
 
98
102
  async fn abort(&mut self) -> Result<()> {
@@ -36,5 +36,5 @@ bytes = { workspace = true }
36
36
  futures = { workspace = true }
37
37
  hdfs-native = { version = "0.14" }
38
38
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
40
40
  serde = { workspace = true, features = ["derive"] }
@@ -167,6 +167,7 @@ impl Service for HdfsNativeBackend {
167
167
  type Lister = HdfsNativeLazyLister;
168
168
  type Deleter = oio::OneShotDeleter<HdfsNativeDeleter>;
169
169
  type Copier = ();
170
+ type Composer = ();
170
171
 
171
172
  fn info(&self) -> ServiceInfo {
172
173
  self.core.info.clone()
@@ -221,7 +222,6 @@ impl Service for HdfsNativeBackend {
221
222
  _from: &str,
222
223
  _to: &str,
223
224
  _args: OpCopy,
224
- _opts: OpCopier,
225
225
  ) -> Result<Self::Copier> {
226
226
  Err(Error::new(
227
227
  ErrorKind::Unsupported,
@@ -68,14 +68,16 @@ impl HdfsNativeCore {
68
68
  EntryMode::FILE
69
69
  };
70
70
 
71
- let mut metadata = Metadata::new(mode);
72
- metadata
73
- .set_last_modified(Timestamp::from_millisecond(
74
- status.modification_time as i64,
75
- )?)
76
- .set_content_length(status.length as u64);
77
-
78
- Ok(metadata)
71
+ let mut metadata = if mode == EntryMode::FILE {
72
+ MetadataBuilder::file(status.length as u64)
73
+ } else {
74
+ MetadataBuilder::dir()
75
+ };
76
+ metadata.last_modified(Timestamp::from_millisecond(
77
+ status.modification_time as i64,
78
+ )?);
79
+
80
+ Ok(metadata.build())
79
81
  }
80
82
 
81
83
  pub async fn hdfs_open(&self, path: &str) -> Result<hdfs_native::file::FileReader> {
@@ -200,49 +202,41 @@ impl HdfsNativeCore {
200
202
  }
201
203
  }
202
204
 
203
- mod error {
204
- use hdfs_native::HdfsError;
205
-
206
- use opendal_core::*;
207
-
208
- /// Parse hdfs-native error into opendal::Error.
209
- pub fn parse_hdfs_error(hdfs_error: HdfsError) -> Error {
210
- let (kind, retryable, msg) = match &hdfs_error {
211
- HdfsError::IOError(err) => (ErrorKind::Unexpected, false, err.to_string()),
212
- HdfsError::DataTransferError(msg) => (ErrorKind::Unexpected, false, msg.clone()),
213
- HdfsError::ChecksumError => (
214
- ErrorKind::Unexpected,
215
- false,
216
- "checksums didn't match".to_string(),
217
- ),
218
- HdfsError::UrlParseError(err) => (ErrorKind::Unexpected, false, err.to_string()),
219
- HdfsError::AlreadyExists(msg) => (ErrorKind::AlreadyExists, false, msg.clone()),
220
- HdfsError::OperationFailed(msg) => (ErrorKind::Unexpected, false, msg.clone()),
221
- HdfsError::RPCError(msg0, msg1) => {
222
- if msg0.contains("java.io.FileNotFoundException") {
223
- (ErrorKind::NotFound, false, msg1.clone())
224
- } else {
225
- (ErrorKind::Unexpected, false, msg1.clone())
226
- }
205
+ /// Parse hdfs-native error into opendal::Error.
206
+ pub fn parse_hdfs_error(hdfs_error: HdfsError) -> Error {
207
+ let (kind, retryable, msg) = match &hdfs_error {
208
+ HdfsError::IOError(err) => (ErrorKind::Unexpected, false, err.to_string()),
209
+ HdfsError::DataTransferError(msg) => (ErrorKind::Unexpected, false, msg.clone()),
210
+ HdfsError::ChecksumError => (
211
+ ErrorKind::Unexpected,
212
+ false,
213
+ "checksums didn't match".to_string(),
214
+ ),
215
+ HdfsError::UrlParseError(err) => (ErrorKind::Unexpected, false, err.to_string()),
216
+ HdfsError::AlreadyExists(msg) => (ErrorKind::AlreadyExists, false, msg.clone()),
217
+ HdfsError::OperationFailed(msg) => (ErrorKind::Unexpected, false, msg.clone()),
218
+ HdfsError::RPCError(msg0, msg1) => {
219
+ if msg0.contains("java.io.FileNotFoundException") {
220
+ (ErrorKind::NotFound, false, msg1.clone())
221
+ } else {
222
+ (ErrorKind::Unexpected, false, msg1.clone())
227
223
  }
228
- HdfsError::FileNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
229
- HdfsError::BlocksNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
230
- HdfsError::IsADirectoryError(msg) => (ErrorKind::IsADirectory, false, msg.clone()),
231
- _ => (
232
- ErrorKind::Unexpected,
233
- false,
234
- "unexpected error from hdfs".to_string(),
235
- ),
236
- };
237
-
238
- let mut err = Error::new(kind, msg).set_source(hdfs_error);
239
-
240
- if retryable {
241
- err = err.set_temporary();
242
224
  }
243
-
244
- err
225
+ HdfsError::FileNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
226
+ HdfsError::BlocksNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
227
+ HdfsError::IsADirectoryError(msg) => (ErrorKind::IsADirectory, false, msg.clone()),
228
+ _ => (
229
+ ErrorKind::Unexpected,
230
+ false,
231
+ "unexpected error from hdfs".to_string(),
232
+ ),
233
+ };
234
+
235
+ let mut err = Error::new(kind, msg).set_source(hdfs_error);
236
+
237
+ if retryable {
238
+ err = err.set_temporary();
245
239
  }
246
- }
247
240
 
248
- pub(super) use error::*;
241
+ err
242
+ }
@@ -17,8 +17,7 @@
17
17
 
18
18
  use hdfs_native::client::ListStatusIterator;
19
19
 
20
- use opendal_core::EntryMode;
21
- use opendal_core::Metadata;
20
+ use opendal_core::MetadataBuilder;
22
21
  use opendal_core::Result;
23
22
  use opendal_core::raw::*;
24
23
 
@@ -54,7 +53,7 @@ impl oio::List for HdfsNativeLister {
54
53
  }
55
54
 
56
55
  if let Some(path) = self.current_path.take() {
57
- return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
56
+ return Ok(Some(oio::Entry::new(&path, MetadataBuilder::dir().build())));
58
57
  }
59
58
 
60
59
  match self.iter.next().await {
@@ -62,14 +61,13 @@ impl oio::List for HdfsNativeLister {
62
61
  let path = build_rel_path(&self.root, &status.path);
63
62
 
64
63
  let entry = if status.isdir {
65
- oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
64
+ oio::Entry::new(&format!("{path}/"), MetadataBuilder::dir().build())
66
65
  } else {
67
- let meta = Metadata::new(EntryMode::FILE)
68
- .with_content_length(status.length as u64)
69
- .with_last_modified(Timestamp::from_millisecond(
70
- status.modification_time as i64,
71
- )?);
72
- oio::Entry::new(&path, meta)
66
+ let mut meta = MetadataBuilder::file(status.length as u64);
67
+ meta.last_modified(Timestamp::from_millisecond(
68
+ status.modification_time as i64,
69
+ )?);
70
+ oio::Entry::new(&path, meta.build())
73
71
  };
74
72
 
75
73
  Ok(Some(entry))
@@ -49,7 +49,11 @@ impl oio::Write for HdfsNativeWriter {
49
49
  async fn close(&mut self) -> Result<Metadata> {
50
50
  self.f.close().await.map_err(parse_hdfs_error)?;
51
51
 
52
- Ok(Metadata::default().with_content_length(self.size))
52
+ Ok({
53
+ let mut metadata = MetadataBuilder::unknown();
54
+ metadata.set_file(self.size);
55
+ metadata.build()
56
+ })
53
57
  }
54
58
 
55
59
  async fn abort(&mut self) -> Result<()> {
@@ -32,11 +32,12 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
+ asyncband = { workspace = true }
35
36
  bytes = { workspace = true }
36
- hf-xet = "1.5.0"
37
+ hf-xet = "1.6.0"
37
38
  http = { workspace = true }
38
39
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
40
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
40
41
  percent-encoding = "2"
41
42
  serde = { workspace = true, features = ["derive"] }
42
43
  serde_json = { workspace = true }
@@ -44,6 +45,6 @@ serde_json = { workspace = true }
44
45
  [dev-dependencies]
45
46
  base64 = { workspace = true }
46
47
  futures = { workspace = true }
47
- opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.58.1" }
48
+ opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.0" }
48
49
  serde_json = { workspace = true }
49
50
  tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
@@ -248,6 +248,7 @@ impl Builder for HfBuilder {
248
248
  stat: true,
249
249
  read: true,
250
250
  write: token.is_some(),
251
+ write_can_multi: token.is_some(),
251
252
  delete: token.is_some(),
252
253
  delete_max_size: Some(100),
253
254
  list: true,
@@ -285,6 +286,7 @@ impl Service for HfBackend {
285
286
  type Lister = oio::PageLister<HfLister>;
286
287
  type Deleter = oio::BatchDeleter<HfDeleter>;
287
288
  type Copier = ();
289
+ type Composer = ();
288
290
 
289
291
  fn info(&self) -> ServiceInfo {
290
292
  self.core.info.clone()
@@ -309,12 +311,12 @@ impl Service for HfBackend {
309
311
  async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
310
312
  // Stat root always returns a DIR.
311
313
  if path == "/" {
312
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
314
+ return Ok(RpStat::new(MetadataBuilder::dir().build()));
313
315
  }
314
316
 
315
317
  // Buckets have no git directory entries; treat any trailing-slash path as a virtual dir.
316
318
  if self.core.repo.is_bucket() && path.ends_with('/') {
317
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
319
+ return Ok(RpStat::new(MetadataBuilder::dir().build()));
318
320
  }
319
321
 
320
322
  let info = self.core.path_info(ctx, path).await?;
@@ -371,7 +373,6 @@ impl Service for HfBackend {
371
373
  _from: &str,
372
374
  _to: &str,
373
375
  _args: OpCopy,
374
- _opts: OpCopier,
375
376
  ) -> Result<Self::Copier> {
376
377
  Err(Error::new(
377
378
  ErrorKind::Unsupported,