opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -20,11 +20,11 @@ use std::sync::Arc;
20
20
  use bytes::Buf;
21
21
  use http::StatusCode;
22
22
 
23
- use super::core::B2Core;
24
23
  use super::core::StartLargeFileResponse;
25
24
  use super::core::UploadPartResponse;
26
25
  use super::core::UploadResponse;
27
26
  use super::core::parse_error;
27
+ use super::core::{B2Core, ErrorContext};
28
28
  use opendal_core::raw::*;
29
29
  use opendal_core::*;
30
30
 
@@ -49,19 +49,21 @@ impl B2Writer {
49
49
  }
50
50
 
51
51
  pub fn parse_body_into_meta(path: &str, resp: UploadResponse) -> Metadata {
52
- let mut meta = Metadata::new(EntryMode::from_path(path));
52
+ let mut meta = if path.ends_with('/') {
53
+ MetadataBuilder::dir()
54
+ } else {
55
+ MetadataBuilder::file(resp.content_length)
56
+ };
53
57
 
54
58
  if let Some(md5) = resp.content_md5 {
55
- meta.set_content_md5(&md5);
59
+ meta.content_md5(&md5);
56
60
  }
57
61
 
58
62
  if let Some(content_type) = resp.content_type {
59
- meta.set_content_type(&content_type);
63
+ meta.content_type(&content_type);
60
64
  }
61
65
 
62
- meta.set_content_length(resp.content_length);
63
-
64
- meta
66
+ meta.build()
65
67
  }
66
68
  }
67
69
 
@@ -85,7 +87,10 @@ impl oio::MultipartWrite for B2Writer {
85
87
 
86
88
  Ok(meta)
87
89
  }
88
- _ => Err(parse_error(resp)),
90
+ _ => Err(parse_error(
91
+ ErrorContext::new(ServiceOperation("UploadFile")),
92
+ resp,
93
+ )),
89
94
  }
90
95
  }
91
96
 
@@ -106,7 +111,10 @@ impl oio::MultipartWrite for B2Writer {
106
111
 
107
112
  Ok(result.file_id)
108
113
  }
109
- _ => Err(parse_error(resp)),
114
+ _ => Err(parse_error(
115
+ ErrorContext::new(ServiceOperation("StartLargeFile")),
116
+ resp,
117
+ )),
110
118
  }
111
119
  }
112
120
 
@@ -141,7 +149,10 @@ impl oio::MultipartWrite for B2Writer {
141
149
  size: None,
142
150
  })
143
151
  }
144
- _ => Err(parse_error(resp)),
152
+ _ => Err(parse_error(
153
+ ErrorContext::new(ServiceOperation("UploadPart")),
154
+ resp,
155
+ )),
145
156
  }
146
157
  }
147
158
 
@@ -180,7 +191,10 @@ impl oio::MultipartWrite for B2Writer {
180
191
 
181
192
  Ok(meta)
182
193
  }
183
- _ => Err(parse_error(resp)),
194
+ _ => Err(parse_error(
195
+ ErrorContext::new(ServiceOperation("FinishLargeFile")),
196
+ resp,
197
+ )),
184
198
  }
185
199
  }
186
200
 
@@ -189,7 +203,10 @@ impl oio::MultipartWrite for B2Writer {
189
203
  match resp.status() {
190
204
  // b2 returns code 200 if abort succeeds.
191
205
  StatusCode::OK => Ok(()),
192
- _ => Err(parse_error(resp)),
206
+ _ => Err(parse_error(
207
+ ErrorContext::new(ServiceOperation("CancelLargeFile")),
208
+ resp,
209
+ )),
193
210
  }
194
211
  }
195
212
  }
@@ -37,7 +37,7 @@ cacache = { version = "13.0", default-features = false, features = [
37
37
  "tokio-runtime",
38
38
  "mmap",
39
39
  ] }
40
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
40
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
41
41
  serde = { workspace = true, features = ["derive"] }
42
42
 
43
43
  [dev-dependencies]
@@ -87,6 +87,7 @@ impl Service for CacacheBackend {
87
87
  type Lister = ();
88
88
  type Deleter = oio::OneShotDeleter<CacacheDeleter>;
89
89
  type Copier = ();
90
+ type Composer = ();
90
91
 
91
92
  fn info(&self) -> ServiceInfo {
92
93
  self.info.clone()
@@ -113,14 +114,13 @@ impl Service for CacacheBackend {
113
114
 
114
115
  match metadata {
115
116
  Some(meta) => {
116
- let mut md = Metadata::new(EntryMode::FILE);
117
- md.set_content_length(meta.size as u64);
117
+ let mut md = MetadataBuilder::file(meta.size as u64);
118
118
  // Convert u128 milliseconds to Timestamp
119
119
  let millis = meta.time as i64;
120
120
  if let Ok(dt) = Timestamp::from_millisecond(millis) {
121
- md.set_last_modified(dt);
121
+ md.last_modified(dt);
122
122
  }
123
- Ok(RpStat::new(md))
123
+ Ok(RpStat::new(md.build()))
124
124
  }
125
125
  None => Err(Error::new(ErrorKind::NotFound, "entry not found")),
126
126
  }
@@ -167,7 +167,6 @@ impl Service for CacacheBackend {
167
167
  _from: &str,
168
168
  _to: &str,
169
169
  _args: OpCopy,
170
- _opts: OpCopier,
171
170
  ) -> Result<Self::Copier> {
172
171
  Err(Error::new(
173
172
  ErrorKind::Unsupported,
@@ -44,7 +44,10 @@ impl oio::StreamRead for CacacheReader {
44
44
  Some(bytes) => {
45
45
  let content_length = bytes.len() as u64;
46
46
  let buffer = Buffer::from(bytes.slice(range.to_content_range(bytes.len())?));
47
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(content_length);
47
+ let metadata = {
48
+ let metadata = MetadataBuilder::file(content_length);
49
+ metadata.build()
50
+ };
48
51
  Ok((
49
52
  RpRead::new(metadata),
50
53
  Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,
@@ -50,8 +50,12 @@ impl oio::Write for CacacheWriter {
50
50
 
51
51
  self.core.set(&self.path, buf.to_bytes()).await?;
52
52
 
53
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
54
- Ok(meta)
53
+ let metadata = if self.path.ends_with('/') {
54
+ MetadataBuilder::dir()
55
+ } else {
56
+ MetadataBuilder::file(length)
57
+ };
58
+ Ok(metadata.build())
55
59
  }
56
60
 
57
61
  async fn abort(&mut self) -> Result<()> {
@@ -34,6 +34,6 @@ all-features = true
34
34
  [dependencies]
35
35
  bytes = { workspace = true }
36
36
  http = { workspace = true }
37
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
38
38
  serde = { workspace = true, features = ["derive"] }
39
39
  serde_json = { workspace = true }
@@ -25,8 +25,8 @@ use opendal_core::*;
25
25
 
26
26
  use super::CLOUDFLARE_KV_SCHEME;
27
27
  use super::config::CloudflareKvConfig;
28
- use super::core::CloudflareKvCore;
29
28
  use super::core::parse_error;
29
+ use super::core::{CloudflareKvCore, ErrorContext};
30
30
  use super::deleter::CloudflareKvDeleter;
31
31
  use super::lister::CloudflareKvLister;
32
32
  use super::model::*;
@@ -37,6 +37,7 @@ use super::writer::CloudflareWriter;
37
37
  #[derive(Default)]
38
38
  pub struct CloudflareKvBuilder {
39
39
  pub(super) config: CloudflareKvConfig,
40
+ pub(super) default_ttl: Option<Duration>,
40
41
  }
41
42
 
42
43
  impl Debug for CloudflareKvBuilder {
@@ -76,7 +77,7 @@ impl CloudflareKvBuilder {
76
77
  ///
77
78
  /// If set, we will specify `EX` for write operations.
78
79
  pub fn default_ttl(mut self, ttl: Duration) -> Self {
79
- self.config.default_ttl = Some(ttl);
80
+ self.default_ttl = Some(ttl);
80
81
  self
81
82
  }
82
83
 
@@ -96,6 +97,14 @@ impl Builder for CloudflareKvBuilder {
96
97
  type Config = CloudflareKvConfig;
97
98
 
98
99
  fn build(self) -> Result<impl Service> {
100
+ let default_ttl = match self.default_ttl {
101
+ Some(ttl) => Some(ttl),
102
+ None => self
103
+ .config
104
+ .default_ttl
105
+ .map(signed_duration_to_duration)
106
+ .transpose()?,
107
+ };
99
108
  let api_token = match &self.config.api_token {
100
109
  Some(api_token) => format_authorization_by_bearer(api_token)?,
101
110
  None => {
@@ -121,7 +130,7 @@ impl Builder for CloudflareKvBuilder {
121
130
  };
122
131
 
123
132
  // Validate default TTL is at least 60 seconds if specified
124
- if let Some(ttl) = self.config.default_ttl
133
+ if let Some(ttl) = default_ttl
125
134
  && ttl < Duration::from_secs(60)
126
135
  {
127
136
  return Err(Error::new(
@@ -143,7 +152,7 @@ impl Builder for CloudflareKvBuilder {
143
152
  api_token,
144
153
  account_id,
145
154
  namespace_id,
146
- expiration_ttl: self.config.default_ttl,
155
+ expiration_ttl: default_ttl,
147
156
  info: ServiceInfo::new(CLOUDFLARE_KV_SCHEME, &root, ""),
148
157
  capability: Capability {
149
158
  create_dir: true,
@@ -189,6 +198,7 @@ impl Service for CloudflareKvBackend {
189
198
  type Lister = oio::PageLister<CloudflareKvLister>;
190
199
  type Deleter = oio::BatchDeleter<CloudflareKvDeleter>;
191
200
  type Copier = ();
201
+ type Composer = ();
192
202
 
193
203
  fn info(&self) -> ServiceInfo {
194
204
  self.core.info.clone()
@@ -266,7 +276,7 @@ impl Service for CloudflareKvBackend {
266
276
  if let Some(entries) = list_result.result
267
277
  && !entries.is_empty()
268
278
  {
269
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
279
+ return Ok(RpStat::new(MetadataBuilder::dir().build()));
270
280
  }
271
281
 
272
282
  // Empty or no results means not found
@@ -278,7 +288,10 @@ impl Service for CloudflareKvBackend {
278
288
  }
279
289
 
280
290
  // For all other error cases, parse the error response
281
- return Err(parse_error(resp));
291
+ return Err(parse_error(
292
+ ErrorContext::new(ServiceOperation("GetMetadata")),
293
+ resp,
294
+ ));
282
295
  }
283
296
 
284
297
  let resp_body = resp.into_body();
@@ -354,16 +367,15 @@ impl Service for CloudflareKvBackend {
354
367
  ));
355
368
  }
356
369
 
357
- let meta = Metadata::new(if metadata.is_dir {
358
- EntryMode::DIR
370
+ let mut meta = if metadata.is_dir {
371
+ MetadataBuilder::dir()
359
372
  } else {
360
- EntryMode::FILE
361
- })
362
- .with_etag(metadata.etag)
363
- .with_content_length(metadata.content_length as u64)
364
- .with_last_modified(metadata.last_modified.parse::<Timestamp>()?);
373
+ MetadataBuilder::file(metadata.content_length as u64)
374
+ };
375
+ meta.etag(metadata.etag)
376
+ .last_modified(metadata.last_modified.parse::<Timestamp>()?);
365
377
 
366
- Ok(RpStat::new(meta))
378
+ Ok(RpStat::new(meta.build()))
367
379
  }
368
380
  fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
369
381
  let output: oio::StreamReader<CloudflareKvReader> = {
@@ -438,7 +450,6 @@ impl Service for CloudflareKvBackend {
438
450
  _from: &str,
439
451
  _to: &str,
440
452
  _args: OpCopy,
441
- _opts: OpCopier,
442
453
  ) -> Result<Self::Copier> {
443
454
  Err(Error::new(
444
455
  ErrorKind::Unsupported,
@@ -34,7 +34,7 @@ pub struct CloudflareKvConfig {
34
34
  /// The namespace ID. Used as URI path parameter.
35
35
  pub namespace_id: Option<String>,
36
36
  /// The default ttl for write operations.
37
- pub default_ttl: Option<Duration>,
37
+ pub default_ttl: Option<SignedDuration>,
38
38
 
39
39
  /// Root within this backend.
40
40
  pub root: Option<String>,
@@ -94,7 +94,10 @@ impl opendal_core::Configurator for CloudflareKvConfig {
94
94
  }
95
95
 
96
96
  fn into_builder(self) -> Self::Builder {
97
- CloudflareKvBuilder { config: self }
97
+ CloudflareKvBuilder {
98
+ config: self,
99
+ default_ttl: None,
100
+ }
98
101
  }
99
102
  }
100
103
 
@@ -125,4 +128,12 @@ mod tests {
125
128
 
126
129
  assert!(CloudflareKvConfig::from_uri(&uri).is_err());
127
130
  }
131
+
132
+ #[test]
133
+ fn from_iter_parses_default_ttl() {
134
+ let cfg = CloudflareKvConfig::from_iter([("default_ttl".to_string(), "PT1M".to_string())])
135
+ .unwrap();
136
+
137
+ assert_eq!(cfg.default_ttl, Some(SignedDuration::from_mins(1)));
138
+ }
128
139
  }
@@ -174,69 +174,75 @@ impl CloudflareKvCore {
174
174
  }
175
175
  }
176
176
 
177
- mod error {
178
- use bytes::Buf;
179
- use http::Response;
180
- use http::StatusCode;
181
- use opendal_core::raw::*;
182
- use opendal_core::*;
183
- use serde_json::de;
184
-
185
- use crate::model::CfKvError;
186
- use crate::model::CfKvResponse;
187
-
188
- /// Parse error response into Error.
189
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
190
- let (parts, body) = resp.into_parts();
191
- let bs = body.to_bytes();
192
-
193
- let (mut kind, mut retryable) = match parts.status {
194
- StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
195
- // Some services (like owncloud) return 403 while file locked.
196
- StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
197
- // Allowing retry for resource locked.
198
- StatusCode::LOCKED => (ErrorKind::Unexpected, true),
199
- StatusCode::INTERNAL_SERVER_ERROR
200
- | StatusCode::BAD_GATEWAY
201
- | StatusCode::SERVICE_UNAVAILABLE
202
- | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
203
- _ => (ErrorKind::Unexpected, false),
204
- };
205
-
206
- let (message, err) = de::from_reader::<_, CfKvResponse>(bs.clone().reader())
207
- .map(|err| (format!("{err:?}"), Some(err)))
208
- .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
209
-
210
- if let Some(err) = err {
211
- (kind, retryable) = parse_cfkv_error_code(err.errors).unwrap_or((kind, retryable));
212
- }
177
+ use bytes::Buf;
178
+ use http::StatusCode;
179
+ use serde_json::de;
213
180
 
214
- let mut err = Error::new(kind, message);
181
+ use crate::model::CfKvError;
182
+ use crate::model::CfKvResponse;
215
183
 
216
- err = with_error_response_context(err, parts);
184
+ /// Context needed to classify an error from this service.
185
+ #[derive(Clone, Copy, Debug)]
186
+ pub(crate) struct ErrorContext {
187
+ service_operation: ServiceOperation,
188
+ }
217
189
 
218
- if retryable {
219
- err = err.set_temporary();
220
- }
190
+ impl ErrorContext {
191
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
192
+ Self { service_operation }
193
+ }
194
+ }
221
195
 
222
- err
196
+ /// Parse an error response using its service request context.
197
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
198
+ let (parts, body) = resp.into_parts();
199
+ let bs = body.to_bytes();
200
+
201
+ let (mut kind, mut retryable) = match parts.status {
202
+ StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
203
+ // Some services (like owncloud) return 403 while file locked.
204
+ StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
205
+ // Allowing retry for resource locked.
206
+ StatusCode::LOCKED => (ErrorKind::Unexpected, true),
207
+ StatusCode::INTERNAL_SERVER_ERROR
208
+ | StatusCode::BAD_GATEWAY
209
+ | StatusCode::SERVICE_UNAVAILABLE
210
+ | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
211
+ _ => (ErrorKind::Unexpected, false),
212
+ };
213
+
214
+ let (message, err) = de::from_reader::<_, CfKvResponse>(bs.clone().reader())
215
+ .map(|err| (format!("{err:?}"), Some(err)))
216
+ .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
217
+
218
+ if let Some(err) = err {
219
+ (kind, retryable) = parse_cfkv_error_code(err.errors).unwrap_or((kind, retryable));
223
220
  }
224
221
 
225
- pub(crate) fn parse_cfkv_error_code(errors: Vec<CfKvError>) -> Option<(ErrorKind, bool)> {
226
- if errors.is_empty() {
227
- return None;
228
- }
222
+ let mut err = Error::new(kind, message);
229
223
 
230
- match errors[0].code {
231
- // The request is malformed: failed to decode id.
232
- 7400 => Some((ErrorKind::Unexpected, false)),
233
- // no such column: Xxxx.
234
- 7500 => Some((ErrorKind::NotFound, false)),
235
- // Authentication error.
236
- 10000 => Some((ErrorKind::PermissionDenied, false)),
237
- _ => None,
238
- }
224
+ err = err.with_context("service_operation", ctx.service_operation.0);
225
+ err = with_error_response_context(err, parts);
226
+
227
+ if retryable {
228
+ err = err.set_temporary();
239
229
  }
230
+
231
+ err
240
232
  }
241
233
 
242
- pub(super) use error::*;
234
+ pub(crate) fn parse_cfkv_error_code(errors: Vec<CfKvError>) -> Option<(ErrorKind, bool)> {
235
+ if errors.is_empty() {
236
+ return None;
237
+ }
238
+
239
+ match errors[0].code {
240
+ // The request is malformed: failed to decode id.
241
+ 7400 => Some((ErrorKind::Unexpected, false)),
242
+ // no such column: Xxxx.
243
+ 7500 => Some((ErrorKind::NotFound, false)),
244
+ // Authentication error.
245
+ 10000 => Some((ErrorKind::PermissionDenied, false)),
246
+ _ => None,
247
+ }
248
+ }
@@ -49,7 +49,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
49
49
  let status = resp.status();
50
50
 
51
51
  if status != StatusCode::OK {
52
- return Err(parse_error(resp.clone()));
52
+ return Err(parse_error(
53
+ ErrorContext::new(ServiceOperation("BulkDelete")),
54
+ resp.clone(),
55
+ ));
53
56
  }
54
57
 
55
58
  let bs = resp.clone().into_body();
@@ -57,7 +60,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
57
60
  serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
58
61
 
59
62
  if !res.success {
60
- return Err(parse_error(resp.clone()));
63
+ return Err(parse_error(
64
+ ErrorContext::new(ServiceOperation("BulkDelete")),
65
+ resp.clone(),
66
+ ));
61
67
  }
62
68
 
63
69
  Ok(())
@@ -77,7 +83,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
77
83
  let status = resp.status();
78
84
 
79
85
  if status != StatusCode::OK {
80
- return Err(parse_error(resp));
86
+ return Err(parse_error(
87
+ ErrorContext::new(ServiceOperation("BulkDelete")),
88
+ resp,
89
+ ));
81
90
  }
82
91
 
83
92
  let bs = resp.into_body();
@@ -21,8 +21,8 @@ use bytes::Buf;
21
21
  use opendal_core::raw::*;
22
22
  use opendal_core::*;
23
23
 
24
- use super::core::CloudflareKvCore;
25
24
  use super::core::parse_error;
25
+ use super::core::{CloudflareKvCore, ErrorContext};
26
26
  use super::model::{CfKvListKey, CfKvListResponse};
27
27
 
28
28
  pub struct CloudflareKvLister {
@@ -34,6 +34,22 @@ pub struct CloudflareKvLister {
34
34
  recursive: bool,
35
35
  }
36
36
 
37
+ /// Strip the service root from a root-prefixed key.
38
+ ///
39
+ /// The keys returned by the KV API carry the service root as a prefix, so only that prefix may be
40
+ /// removed. `str::replace` removes *every* occurrence anywhere in the key, which silently mangles
41
+ /// any key that repeats the root as an inner path segment.
42
+ ///
43
+ /// The sibling object-store listers reach for `build_rel_path` here, but that helper
44
+ /// `debug_assert!`s that the path really does start with the root -- a guarantee this service does
45
+ /// not enforce on the values the API hands back -- so this stays total and leaves a
46
+ /// non-root-prefixed key untouched.
47
+ fn relative_to_root(root: &str, name: &str) -> String {
48
+ name.strip_prefix(root.trim_start_matches('/'))
49
+ .unwrap_or(name)
50
+ .to_string()
51
+ }
52
+
37
53
  impl CloudflareKvLister {
38
54
  pub fn new(
39
55
  core: Arc<CloudflareKvCore>,
@@ -60,7 +76,7 @@ impl CloudflareKvLister {
60
76
  name += "/";
61
77
  }
62
78
 
63
- let mut name = name.replace(root.trim_start_matches('/'), "");
79
+ let mut name = relative_to_root(root, &name);
64
80
 
65
81
  // If it is the root directory, it needs to be processed as /
66
82
  if name.is_empty() {
@@ -68,14 +84,15 @@ impl CloudflareKvLister {
68
84
  }
69
85
 
70
86
  let entry_metadata = if name.ends_with('/') {
71
- Metadata::new(EntryMode::DIR)
72
- .with_etag(build_tmp_path_of(&name))
73
- .with_content_length(0)
87
+ let mut metadata = MetadataBuilder::dir();
88
+ metadata.etag(build_tmp_path_of(&name));
89
+ metadata.build()
74
90
  } else {
75
- Metadata::new(EntryMode::FILE)
76
- .with_etag(metadata.etag)
77
- .with_content_length(metadata.content_length as u64)
78
- .with_last_modified(metadata.last_modified.parse::<Timestamp>()?)
91
+ let mut result = MetadataBuilder::file(metadata.content_length as u64);
92
+ result
93
+ .etag(&metadata.etag)
94
+ .last_modified(metadata.last_modified.parse::<Timestamp>()?);
95
+ result.build()
79
96
  };
80
97
 
81
98
  Ok(oio::Entry::new(&name, entry_metadata))
@@ -91,13 +108,12 @@ impl CloudflareKvLister {
91
108
  let entry = self.build_entry_for_item(item, root)?;
92
109
  ctx.entries.push_back(entry);
93
110
  } else if !result.is_empty() {
94
- let path_name = self.path.replace(root.trim_start_matches('/'), "");
95
- let entry = oio::Entry::new(
96
- &format!("{path_name}/"),
97
- Metadata::new(EntryMode::DIR)
98
- .with_etag(build_tmp_path_of(&path_name))
99
- .with_content_length(0),
100
- );
111
+ let path_name = relative_to_root(root, &self.path);
112
+ let entry = oio::Entry::new(&format!("{path_name}/"), {
113
+ let mut metadata = MetadataBuilder::dir();
114
+ metadata.etag(build_tmp_path_of(&path_name));
115
+ metadata.build()
116
+ });
101
117
  ctx.entries.push_back(entry);
102
118
  }
103
119
  ctx.done = true;
@@ -114,7 +130,10 @@ impl oio::PageList for CloudflareKvLister {
114
130
  .await?;
115
131
 
116
132
  if resp.status() != http::StatusCode::OK {
117
- return Err(parse_error(resp));
133
+ return Err(parse_error(
134
+ ErrorContext::new(ServiceOperation("ListKeys")),
135
+ resp,
136
+ ));
118
137
  }
119
138
 
120
139
  let bs = resp.into_body();
@@ -171,3 +190,33 @@ impl oio::PageList for CloudflareKvLister {
171
190
  Ok(())
172
191
  }
173
192
  }
193
+
194
+ #[cfg(test)]
195
+ mod tests {
196
+ use super::*;
197
+
198
+ #[test]
199
+ fn relative_to_root_strips_only_the_prefix() {
200
+ // The root repeated as an inner segment must survive; only the leading copy goes.
201
+ assert_eq!(
202
+ relative_to_root("/data/", "data/backup/data/file.txt"),
203
+ "backup/data/file.txt"
204
+ );
205
+ assert_eq!(relative_to_root("/data/", "data/file.txt"), "file.txt");
206
+ }
207
+
208
+ #[test]
209
+ fn relative_to_root_does_not_match_mid_key() {
210
+ // "replace" would turn this into "xfile.txt" by deleting a substring that is not a prefix.
211
+ assert_eq!(
212
+ relative_to_root("/data/", "xdata/file.txt"),
213
+ "xdata/file.txt"
214
+ );
215
+ }
216
+
217
+ #[test]
218
+ fn relative_to_root_handles_the_root_itself_and_a_bare_root() {
219
+ assert_eq!(relative_to_root("/data/", "data/"), "");
220
+ assert_eq!(relative_to_root("/", "file.txt"), "file.txt");
221
+ }
222
+ }
@@ -16,7 +16,7 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use super::core::parse_error;
19
+ use super::core::{ErrorContext, parse_error};
20
20
  use super::model::*;
21
21
  use bytes::Buf;
22
22
  use http::StatusCode;
@@ -58,20 +58,22 @@ impl oio::StreamRead for CloudflareKvReader {
58
58
  let status = resp.status();
59
59
 
60
60
  if status != StatusCode::OK {
61
- return Err(parse_error(resp));
61
+ return Err(parse_error(
62
+ ErrorContext::new(ServiceOperation("GetValue")),
63
+ resp,
64
+ ));
62
65
  }
63
66
 
64
67
  let resp_body = resp.into_body();
65
68
 
66
- if args.if_match().is_some()
67
- || args.if_none_match().is_some()
68
- || args.if_modified_since().is_some()
69
- || args.if_unmodified_since().is_some()
70
- {
69
+ if args.is_conditional() {
71
70
  let meta_resp = backend.core.metadata(&self.ctx, &path).await?;
72
71
 
73
72
  if meta_resp.status() != StatusCode::OK {
74
- return Err(parse_error(meta_resp));
73
+ return Err(parse_error(
74
+ ErrorContext::new(ServiceOperation("GetMetadata")),
75
+ meta_resp,
76
+ ));
75
77
  }
76
78
 
77
79
  let cf_response: CfKvStatResponse =
@@ -133,7 +135,10 @@ impl oio::StreamRead for CloudflareKvReader {
133
135
 
134
136
  let total_size = resp_body.len() as u64;
135
137
  let buffer = resp_body.slice(range.to_content_range(resp_body.len())?);
136
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(total_size);
138
+ let metadata = {
139
+ let metadata = MetadataBuilder::file(total_size);
140
+ metadata.build()
141
+ };
137
142
  Ok((
138
143
  RpRead::new(metadata),
139
144
  Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,