opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -39,6 +39,7 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
39
39
  test_stat_not_exist,
40
40
  test_stat_with_if_match,
41
41
  test_stat_with_if_none_match,
42
+ test_stat_with_version_conditions,
42
43
  test_stat_with_if_modified_since,
43
44
  test_stat_with_if_unmodified_since,
44
45
  test_stat_with_override_cache_control,
@@ -200,6 +201,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
200
201
  .await;
201
202
  assert!(result.is_ok());
202
203
 
204
+ let missing = TEST_FIXTURE.new_file_path();
205
+ let err = op
206
+ .stat_with(&missing)
207
+ .if_match(meta.etag().expect("etag must exist"))
208
+ .await
209
+ .expect_err("missing target must fail");
210
+ assert_eq!(err.kind(), ErrorKind::NotFound);
211
+
203
212
  Ok(())
204
213
  }
205
214
 
@@ -233,6 +242,71 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
233
242
  assert_eq!(res.mode(), meta.mode());
234
243
  assert_eq!(res.content_length(), meta.content_length());
235
244
 
245
+ let missing = TEST_FIXTURE.new_file_path();
246
+ let err = op
247
+ .stat_with(&missing)
248
+ .if_none_match(meta.etag().expect("etag must exist"))
249
+ .await
250
+ .expect_err("missing target must fail");
251
+ assert_eq!(err.kind(), ErrorKind::NotFound);
252
+
253
+ Ok(())
254
+ }
255
+
256
+ /// Version preconditions should compare against the current live object version.
257
+ pub async fn test_stat_with_version_conditions(op: Operator) -> Result<()> {
258
+ let cap = op.info().capability();
259
+ if !cap.stat_with_if_version_match || !cap.stat_with_if_version_not_match {
260
+ return Ok(());
261
+ }
262
+
263
+ let path = TEST_FIXTURE.new_file_path();
264
+ let (first, _) = gen_bytes(cap);
265
+ let (second, _) = gen_bytes(cap);
266
+ assert_ne!(first, second);
267
+
268
+ op.write(&path, first).await?;
269
+ let stale = op
270
+ .stat(&path)
271
+ .await?
272
+ .version()
273
+ .expect("version must exist")
274
+ .to_string();
275
+ op.write(&path, second).await?;
276
+ let current = op
277
+ .stat(&path)
278
+ .await?
279
+ .version()
280
+ .expect("version must exist")
281
+ .to_string();
282
+
283
+ op.stat_with(&path).if_version_match(&current).await?;
284
+ let err = op
285
+ .stat_with(&path)
286
+ .if_version_match(&stale)
287
+ .await
288
+ .expect_err("stale version match must fail");
289
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
290
+
291
+ op.stat_with(&path).if_version_not_match(&stale).await?;
292
+ let err = op
293
+ .stat_with(&path)
294
+ .if_version_not_match(&current)
295
+ .await
296
+ .expect_err("equal version non-match must fail");
297
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
298
+
299
+ let missing = TEST_FIXTURE.new_file_path();
300
+ for result in [
301
+ op.stat_with(&missing).if_version_match(&current).await,
302
+ op.stat_with(&missing).if_version_not_match(&current).await,
303
+ ] {
304
+ assert_eq!(
305
+ result.expect_err("missing target must fail").kind(),
306
+ ErrorKind::NotFound
307
+ );
308
+ }
309
+
236
310
  Ok(())
237
311
  }
238
312
 
@@ -263,6 +337,14 @@ pub async fn test_stat_with_if_modified_since(op: Operator) -> Result<()> {
263
337
  assert!(res.is_err());
264
338
  assert_eq!(res.err().unwrap().kind(), ErrorKind::ConditionNotMatch);
265
339
 
340
+ let missing = TEST_FIXTURE.new_file_path();
341
+ let err = op
342
+ .stat_with(&missing)
343
+ .if_modified_since(since)
344
+ .await
345
+ .expect_err("missing target must fail");
346
+ assert_eq!(err.kind(), ErrorKind::NotFound);
347
+
266
348
  Ok(())
267
349
  }
268
350
 
@@ -293,6 +375,14 @@ pub async fn test_stat_with_if_unmodified_since(op: Operator) -> Result<()> {
293
375
  let res = op.stat_with(&path).if_unmodified_since(since).await?;
294
376
  assert_eq!(res.last_modified(), meta.last_modified());
295
377
 
378
+ let missing = TEST_FIXTURE.new_file_path();
379
+ let err = op
380
+ .stat_with(&missing)
381
+ .if_unmodified_since(since)
382
+ .await
383
+ .expect_err("missing target must fail");
384
+ assert_eq!(err.kind(), ErrorKind::NotFound);
385
+
296
386
  Ok(())
297
387
  }
298
388
 
@@ -45,6 +45,8 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
45
45
  test_write_with_if_none_match,
46
46
  test_write_with_if_not_exists,
47
47
  test_write_with_if_match,
48
+ test_write_with_version_conditions,
49
+ test_write_if_not_changed,
48
50
  test_write_with_user_metadata,
49
51
  test_write_returns_metadata,
50
52
  test_writer_write,
@@ -57,7 +59,12 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
57
59
  test_writer_futures_copy,
58
60
  test_writer_futures_copy_with_concurrent,
59
61
  test_writer_return_metadata,
60
- test_writer_write_non_contiguous_data
62
+ test_writer_copy_from_interleaved,
63
+ test_writer_write_non_contiguous_data,
64
+ test_writer_write_with_if_not_exists,
65
+ test_writer_write_with_if_none_match,
66
+ test_writer_write_with_if_match,
67
+ test_writer_write_with_version_conditions
61
68
  ))
62
69
  }
63
70
 
@@ -242,7 +249,10 @@ pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> {
242
249
  let resp_meta = meta.user_metadata().expect("meta data must exist");
243
250
 
244
251
  assert_eq!(
245
- *resp_meta,
252
+ resp_meta
253
+ .into_iter()
254
+ .map(|(key, value)| (key.to_owned(), value.to_owned()))
255
+ .collect::<HashMap<_, _>>(),
246
256
  target_user_metadata.into_iter().collect::<HashMap<_, _>>()
247
257
  );
248
258
 
@@ -349,6 +359,59 @@ pub async fn test_writer_write(op: Operator) -> Result<()> {
349
359
  Ok(())
350
360
  }
351
361
 
362
+ /// Assemble a destination from local bytes and source ranges in call order.
363
+ pub async fn test_writer_copy_from_interleaved(op: Operator) -> Result<()> {
364
+ if !op.info().capability().write_can_multi {
365
+ return Ok(());
366
+ }
367
+
368
+ let source_path = TEST_FIXTURE.new_file_path();
369
+ let target_path = TEST_FIXTURE.new_file_path();
370
+ let source = gen_fixed_bytes(18 * 1024 * 1024);
371
+ op.write(&source_path, source.clone()).await?;
372
+ let source_meta = op.stat(&source_path).await?;
373
+ let source_if_match =
374
+ if op.info().capability().read_with_if_match && op.info().capability().stat_with_if_match {
375
+ source_meta.etag().map(str::to_string)
376
+ } else {
377
+ None
378
+ };
379
+
380
+ let mut writer = op.writer(&target_path).await?;
381
+ writer.write("header").await?;
382
+ writer.copy_from(&source_path, 1024_u64..2048).await?;
383
+ writer
384
+ .copy_from_options(
385
+ &source_path,
386
+ options::ReadOptions {
387
+ range: (2048_u64..14 * 1024 * 1024).into(),
388
+ if_match: source_if_match.clone(),
389
+ ..Default::default()
390
+ },
391
+ )
392
+ .await?;
393
+ writer.write("footer").await?;
394
+ writer
395
+ .copy_from_options(
396
+ &source_path,
397
+ options::ReadOptions {
398
+ range: (15 * 1024 * 1024_u64..).into(),
399
+ if_match: source_if_match,
400
+ ..Default::default()
401
+ },
402
+ )
403
+ .await?;
404
+ writer.close().await?;
405
+
406
+ let mut expected = Vec::new();
407
+ expected.extend_from_slice(b"header");
408
+ expected.extend_from_slice(&source[1024..14 * 1024 * 1024]);
409
+ expected.extend_from_slice(b"footer");
410
+ expected.extend_from_slice(&source[15 * 1024 * 1024..]);
411
+ assert_eq!(op.read(&target_path).await?.to_bytes(), expected);
412
+ Ok(())
413
+ }
414
+
352
415
  /// Append data into writer
353
416
  pub async fn test_writer_write_with_concurrent(op: Operator) -> Result<()> {
354
417
  if !(op.info().capability().write_can_multi) {
@@ -810,6 +873,271 @@ pub async fn test_write_with_if_match(op: Operator) -> Result<()> {
810
873
  Ok(())
811
874
  }
812
875
 
876
+ /// Version preconditions should compare against the current live object version.
877
+ pub async fn test_write_with_version_conditions(op: Operator) -> Result<()> {
878
+ let cap = op.info().capability();
879
+ if !cap.write_with_if_version_match || !cap.write_with_if_version_not_match {
880
+ return Ok(());
881
+ }
882
+
883
+ let path = TEST_FIXTURE.new_file_path();
884
+ let (initial, _) = gen_bytes(cap);
885
+ let (replacement, _) = gen_bytes(cap);
886
+ assert_ne!(initial, replacement);
887
+
888
+ op.write(&path, initial).await?;
889
+ let first_version = op
890
+ .stat(&path)
891
+ .await?
892
+ .version()
893
+ .expect("version must exist")
894
+ .to_string();
895
+
896
+ op.write_with(&path, replacement.clone())
897
+ .if_version_match(&first_version)
898
+ .await?;
899
+
900
+ let current_version = op
901
+ .stat(&path)
902
+ .await?
903
+ .version()
904
+ .expect("version must exist")
905
+ .to_string();
906
+ let err = op
907
+ .write_with(&path, replacement.clone())
908
+ .if_version_match(&first_version)
909
+ .await
910
+ .expect_err("stale version match must fail");
911
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
912
+
913
+ let err = op
914
+ .write_with(&path, replacement.clone())
915
+ .if_version_not_match(&current_version)
916
+ .await
917
+ .expect_err("equal version non-match must fail");
918
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
919
+ op.write_with(&path, replacement)
920
+ .if_version_not_match(&first_version)
921
+ .await?;
922
+
923
+ let missing = TEST_FIXTURE.new_file_path();
924
+ for result in [
925
+ op.write_with(&missing, Vec::<u8>::new())
926
+ .if_version_match(&current_version)
927
+ .await,
928
+ op.write_with(&missing, Vec::<u8>::new())
929
+ .if_version_not_match(&current_version)
930
+ .await,
931
+ ] {
932
+ assert_eq!(
933
+ result.expect_err("missing target must fail").kind(),
934
+ ErrorKind::ConditionNotMatch
935
+ );
936
+ }
937
+
938
+ Ok(())
939
+ }
940
+
941
+ /// `if_not_changed` should accept the observed state once and reject it after replacement.
942
+ pub async fn test_write_if_not_changed(op: Operator) -> Result<()> {
943
+ let cap = op.info().capability();
944
+ if !cap.write_with_if_version_match && !cap.write_with_if_match {
945
+ return Ok(());
946
+ }
947
+
948
+ let path = TEST_FIXTURE.new_file_path();
949
+ let (initial, _) = gen_bytes(cap);
950
+ let (replacement, _) = gen_bytes(cap);
951
+ assert_ne!(initial, replacement);
952
+
953
+ op.write(&path, initial).await?;
954
+ let expected = op.stat(&path).await?;
955
+
956
+ let mut conflicting = options::WriteOptions {
957
+ if_not_changed: Some(expected.clone()),
958
+ ..Default::default()
959
+ };
960
+ let mut matching = conflicting.clone();
961
+ if cap.write_with_if_version_match {
962
+ let version = expected.version().expect("version must exist");
963
+ conflicting.if_version_match = Some(format!("different-{version}"));
964
+ matching.if_version_match = Some(version.to_string());
965
+ } else {
966
+ let etag = expected.etag().expect("etag must exist");
967
+ conflicting.if_match = Some(format!("different-{etag}"));
968
+ matching.if_match = Some(etag.to_string());
969
+ }
970
+
971
+ let err = op
972
+ .write_options(&path, replacement.clone(), conflicting)
973
+ .await
974
+ .expect_err("conflicting explicit condition must fail");
975
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
976
+
977
+ op.write_options(&path, replacement.clone(), matching)
978
+ .await?;
979
+ let err = op
980
+ .write_with(&path, replacement)
981
+ .if_not_changed(&expected)
982
+ .await
983
+ .expect_err("stale metadata must fail");
984
+ assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
985
+
986
+ Ok(())
987
+ }
988
+
989
+ /// Write an existing file through a chunked writer with if_not_exists should get a
990
+ /// ConditionNotMatch error.
991
+ pub async fn test_writer_write_with_if_not_exists(op: Operator) -> Result<()> {
992
+ let cap = op.info().capability();
993
+ if !cap.write_with_if_not_exists || !cap.write_can_multi {
994
+ return Ok(());
995
+ }
996
+
997
+ let path = TEST_FIXTURE.new_file_path();
998
+ let content = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
999
+
1000
+ op.write(&path, content.clone())
1001
+ .await
1002
+ .expect("write must succeed");
1003
+
1004
+ // Some services reject the precondition when the writer is created or on an early
1005
+ // write rather than at commit time
1006
+ let res: opendal::Result<()> = async {
1007
+ let mut w = op.writer_with(&path).if_not_exists(true).await?;
1008
+ w.write(content.clone()).await?;
1009
+ w.write(content.clone()).await?;
1010
+ w.close().await?;
1011
+ Ok(())
1012
+ }
1013
+ .await;
1014
+ assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
1015
+
1016
+ Ok(())
1017
+ }
1018
+
1019
+ /// Write an existing file through a chunked writer with its own etag as if_none_match
1020
+ /// should get a ConditionNotMatch error.
1021
+ pub async fn test_writer_write_with_if_none_match(op: Operator) -> Result<()> {
1022
+ let cap = op.info().capability();
1023
+ if !cap.write_with_if_none_match || !cap.write_can_multi {
1024
+ return Ok(());
1025
+ }
1026
+
1027
+ let path = TEST_FIXTURE.new_file_path();
1028
+ let content = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
1029
+
1030
+ op.write(&path, content.clone())
1031
+ .await
1032
+ .expect("write must succeed");
1033
+
1034
+ let meta = op.stat(&path).await?;
1035
+ let etag = meta.etag().expect("etag must exist");
1036
+
1037
+ let res: opendal::Result<()> = async {
1038
+ let mut w = op.writer_with(&path).if_none_match(etag).await?;
1039
+ w.write(content.clone()).await?;
1040
+ w.write(content.clone()).await?;
1041
+ w.close().await?;
1042
+ Ok(())
1043
+ }
1044
+ .await;
1045
+ assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
1046
+
1047
+ Ok(())
1048
+ }
1049
+
1050
+ /// Write a file through a chunked writer with if_match should succeed with the file's own
1051
+ /// etag and get a ConditionNotMatch error with a stale one.
1052
+ pub async fn test_writer_write_with_if_match(op: Operator) -> Result<()> {
1053
+ let cap = op.info().capability();
1054
+ if !cap.write_with_if_match || !cap.write_can_multi {
1055
+ return Ok(());
1056
+ }
1057
+
1058
+ let path_a = TEST_FIXTURE.new_file_path();
1059
+ let content_a = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
1060
+ let (path_b, content_b, _) = TEST_FIXTURE.new_file(op.clone());
1061
+
1062
+ op.write(&path_a, content_a.clone()).await?;
1063
+ op.write(&path_b, content_b.clone()).await?;
1064
+
1065
+ let etag_a = op
1066
+ .stat(&path_a)
1067
+ .await?
1068
+ .etag()
1069
+ .expect("etag must exist")
1070
+ .to_string();
1071
+ let etag_b = op
1072
+ .stat(&path_b)
1073
+ .await?
1074
+ .etag()
1075
+ .expect("etag must exist")
1076
+ .to_string();
1077
+
1078
+ let mut w = op.writer_with(&path_a).if_match(&etag_a).await?;
1079
+ w.write(content_a.clone()).await?;
1080
+ w.write(content_a.clone()).await?;
1081
+ w.close().await.expect("close with own etag must succeed");
1082
+
1083
+ // Should fail: writing to path_a with path_b's etag.
1084
+ let res: opendal::Result<()> = async {
1085
+ let mut w = op.writer_with(&path_a).if_match(&etag_b).await?;
1086
+ w.write(content_a.clone()).await?;
1087
+ w.write(content_a.clone()).await?;
1088
+ w.close().await?;
1089
+ Ok(())
1090
+ }
1091
+ .await;
1092
+ assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
1093
+
1094
+ Ok(())
1095
+ }
1096
+
1097
+ /// Chunked writers should preserve version preconditions through final commit.
1098
+ pub async fn test_writer_write_with_version_conditions(op: Operator) -> Result<()> {
1099
+ let cap = op.info().capability();
1100
+ if !cap.write_can_multi
1101
+ || !cap.write_with_if_version_match
1102
+ || !cap.write_with_if_version_not_match
1103
+ {
1104
+ return Ok(());
1105
+ }
1106
+
1107
+ let path = TEST_FIXTURE.new_file_path();
1108
+ let body = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
1109
+ op.write(&path, body.clone()).await?;
1110
+ let expected = op
1111
+ .stat(&path)
1112
+ .await?
1113
+ .version()
1114
+ .expect("version must exist")
1115
+ .to_string();
1116
+
1117
+ let mut writer = op.writer_with(&path).if_version_match(&expected).await?;
1118
+ writer.write(body.clone()).await?;
1119
+ writer.write(body.clone()).await?;
1120
+ writer.close().await?;
1121
+
1122
+ let current = op
1123
+ .stat(&path)
1124
+ .await?
1125
+ .version()
1126
+ .expect("version must exist")
1127
+ .to_string();
1128
+ let result: opendal::Result<()> = async {
1129
+ let mut writer = op.writer_with(&path).if_version_not_match(&current).await?;
1130
+ writer.write(body.clone()).await?;
1131
+ writer.write(body).await?;
1132
+ writer.close().await?;
1133
+ Ok(())
1134
+ }
1135
+ .await;
1136
+ assert_eq!(result.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
1137
+
1138
+ Ok(())
1139
+ }
1140
+
813
1141
  pub async fn test_writer_write_non_contiguous_data(op: Operator) -> Result<()> {
814
1142
  let path = TEST_FIXTURE.new_file_path();
815
1143
  let size = 1024 * 1024; // write file with 1 MiB
@@ -21,6 +21,7 @@ mod utils;
21
21
 
22
22
  pub use utils::*;
23
23
 
24
+ mod async_compose;
24
25
  mod async_copy;
25
26
  mod async_create_dir;
26
27
  mod async_delete;
@@ -28,6 +29,7 @@ mod async_list;
28
29
  mod async_presign;
29
30
  mod async_read;
30
31
  mod async_rename;
32
+ mod async_restore;
31
33
  mod async_stat;
32
34
  mod async_write;
33
35
 
@@ -55,12 +57,14 @@ fn main() -> anyhow::Result<()> {
55
57
  let mut tests = Vec::new();
56
58
 
57
59
  async_copy::tests(&op, &mut tests);
60
+ async_compose::tests(&op, &mut tests);
58
61
  async_create_dir::tests(&op, &mut tests);
59
62
  async_delete::tests(&op, &mut tests);
60
63
  async_list::tests(&op, &mut tests);
61
64
  async_presign::tests(&op, &mut tests);
62
65
  async_read::tests(&op, &mut tests);
63
66
  async_rename::tests(&op, &mut tests);
67
+ async_restore::tests(&op, &mut tests);
64
68
  async_stat::tests(&op, &mut tests);
65
69
  async_write::tests(&op, &mut tests);
66
70
 
data/core/upgrade.md CHANGED
@@ -1,3 +1,116 @@
1
+ # Upgrade to v0.59
2
+
3
+ ## Public API
4
+
5
+ ### Construct `Metadata` with `MetadataBuilder`
6
+
7
+ `Metadata::new`, `Default`, direct metadata setters, and `with_*` methods have been removed. Construct metadata with `MetadataBuilder::file(content_length)`, `MetadataBuilder::dir()`, or `MetadataBuilder::unknown()`, set optional fields on the builder, then call `build()`:
8
+
9
+ ```diff
10
+ -let mut metadata = Metadata::new(EntryMode::FILE);
11
+ -metadata.set_content_length(size);
12
+ -metadata.set_content_type("text/plain");
13
+ +let mut builder = MetadataBuilder::file(size);
14
+ +builder.content_type("text/plain");
15
+ +let metadata = builder.build();
16
+ ```
17
+
18
+ Consume existing metadata with `into_builder()` when creating a modified value. Use `set_file(content_length)`, `set_dir()`, or `set_unknown()` on the builder to change its mode.
19
+
20
+ `Metadata::user_metadata()` now returns a borrowed `UserMetadata` view instead of `&HashMap<String, String>`. The view supports `get`, `len`, `is_empty`, and `IntoIterator`; collect owned string pairs at boundaries that require a map.
21
+
22
+ Every file `Metadata` contains an explicit full-object content length by construction. Raw services use `Unknown` for incomplete metadata and promote it with `set_file(content_length)` only when the full length is authoritative.
23
+
24
+ Compact metadata and finalized raw-operation value blocks are limited to `u16::MAX` encoded bytes, including indexes. `MetadataBuilder::build`, raw option freezing, and raw listed-entry construction panic when a block exceeds this bound.
25
+
26
+ ### Delete inputs use `DeleteOptions`
27
+
28
+ `DeleteInput` has been removed. Pass `(String, DeleteOptions)` to `Deleter`, delete iterators, streams, and sinks when an entry needs options. Plain paths and `Entry` values remain supported through `IntoDeleteInput`. Custom `IntoDeleteInput` implementations must now return `(String, DeleteOptions)` from `into_delete_input`. `(String, OpDelete)` is no longer accepted by the public delete APIs because raw arguments are already capability-resolved.
29
+
30
+ ```diff
31
+ -deleter.delete(DeleteInput {
32
+ - path,
33
+ - version,
34
+ - recursive: false,
35
+ -});
36
+ +deleter.delete((
37
+ + path,
38
+ + DeleteOptions {
39
+ + version,
40
+ + ..Default::default()
41
+ + },
42
+ +));
43
+ ```
44
+
45
+ ### Existing option and capability literals require defaults
46
+
47
+ OpenDAL added fields to `Capability`, `ReadOptions`, `ReaderOptions`, `StatOptions`, `WriteOptions`, `DeleteOptions`, and `CopyOptions`. Code that constructs any of these types with an exhaustive struct literal must initialize the new fields. Prefer update syntax so later additions remain source-compatible:
48
+
49
+ ```rust
50
+ use opendal_core::Capability;
51
+ use opendal_core::options::DeleteOptions;
52
+
53
+ let options = DeleteOptions {
54
+ recursive: true,
55
+ ..Default::default()
56
+ };
57
+
58
+ let capability = Capability {
59
+ read: true,
60
+ ..Default::default()
61
+ };
62
+ ```
63
+
64
+ ### Conditional errors distinguish conflicts
65
+
66
+ Failed conditions supplied through operation options return `ErrorKind::ConditionNotMatch`. Conflicts reported independently by a service return the new `ErrorKind::Conflict`; inspect `Error::is_temporary()` to decide whether the receiving OpenDAL layer can replay the same operation. Unsupported conditions return `ErrorKind::Unsupported`, and a missing target observed by `stat` or `read` remains `ErrorKind::NotFound` even when the request was conditional. Update error recovery code that previously treated every service conflict as `ConditionNotMatch` or `Unexpected`.
67
+
68
+ ## Raw API
69
+
70
+ ### `OpCopier` merged into `OpCopy`
71
+
72
+ `OpCopier` has been removed. `Service::copy` and `ServiceDyn::copy_dyn` now receive a single `OpCopy` argument, and raw services read `chunk`, `concurrent`, and `source_content_length_hint` from `OpCopy`.
73
+
74
+ ### Raw operation arguments are frozen from public options
75
+
76
+ Public `with_*` mutation methods have been removed from `OpRead`, `OpStat`, `OpWrite`, `OpDelete`, `OpCopy`, and `OpList`. Construct non-empty read, stat, list, and restore arguments by converting the corresponding public `*Options`. Use the capability-aware `from_options(&capability, options)` constructors for `OpWrite`, `OpDelete`, `OpCopy`, and `OpCompose`; `new` and `Default` remain available for empty arguments.
77
+
78
+ The direct `From<WriteOptions>`, `From<DeleteOptions>`, and `From<CopyOptions>` conversions have been removed. The capability-aware constructors lower `if_not_changed` before freezing, so raw arguments contain only service-ready primitive conditions. `OpWrite::from_options` still returns `(OpWrite, OpWriter)`, while copy execution settings now live directly in `OpCopy`. `OpWrite::user_metadata()` returns the same borrowed `UserMetadata` view as `Metadata`.
79
+
80
+ ### Listed entries contain finalized metadata
81
+
82
+ `raw::oio::Entry::set_mode` and `metadata_mut` have been removed. Build the final `Metadata` first and pass it to `raw::oio::Entry::new` or `with`. To modify an existing raw entry, consume it with `into_parts`, rebuild its metadata, and construct a replacement entry:
83
+
84
+ ```rust
85
+ use opendal_core::MetadataBuilder;
86
+ use opendal_core::raw;
87
+
88
+ let entry = raw::oio::Entry::new("path", MetadataBuilder::file(0).build());
89
+ let (path, metadata) = entry.into_parts();
90
+ let mut builder = metadata.into_builder();
91
+ builder.content_type("text/plain");
92
+ let entry = raw::oio::Entry::new(&path, builder.build());
93
+ ```
94
+
95
+ The metadata mode must continue to match the path: directory paths end in `/`, while file paths do not.
96
+
97
+ ### Do not use numeric `Operation` discriminants
98
+
99
+ The new `Compose` and `Restore` variants changed the implicit numeric discriminants of later `Operation` variants. Code that casts `Operation` with `as usize` must migrate to `Operation::into_static()` or `Display` for stable operation names. `Operation` has no numeric representation contract.
100
+
101
+ ## Services
102
+
103
+ ### Duration config fields use `SignedDuration`
104
+
105
+ The public `default_ttl` fields on `CloudflareKvConfig`, `MemcachedConfig`, and `RedisConfig` now use `Option<raw::SignedDuration>` instead of `Option<std::time::Duration>`. This allows configuration maps and serialized configuration to parse duration strings. Direct config construction can use the re-exported duration type:
106
+
107
+ ```diff
108
+ -default_ttl: Some(std::time::Duration::from_secs(30)),
109
+ +default_ttl: Some(opendal::raw::SignedDuration::from_secs(30)),
110
+ ```
111
+
112
+ The corresponding builder methods continue to accept `std::time::Duration`.
113
+
1
114
  # Upgrade to v0.58
2
115
 
3
116
  ## Public API
@@ -388,7 +501,7 @@ The following new APIs have been added:
388
501
  - [`Deleter::flush`]
389
502
  - [`Deleter::close`]
390
503
  - [`Deleter::into_sink`]
391
- - [`DeleteInput`]
504
+ - `DeleteInput`
392
505
  - [`IntoDeleteInput`]
393
506
  - [`FuturesDeleteSink`]
394
507
 
data/src/capability.rs CHANGED
@@ -18,7 +18,6 @@
18
18
  use magnus::Error;
19
19
  use magnus::RModule;
20
20
  use magnus::method;
21
- use magnus::prelude::*;
22
21
 
23
22
  use crate::*;
24
23
 
data/src/io.rs CHANGED
@@ -36,7 +36,6 @@ use magnus::RString;
36
36
  use magnus::Value;
37
37
  use magnus::io::FMode;
38
38
  use magnus::method;
39
- use magnus::prelude::*;
40
39
  use magnus::scan_args::scan_args;
41
40
 
42
41
  use crate::*;
data/src/lister.rs CHANGED
@@ -31,7 +31,6 @@ use magnus::Error;
31
31
  use magnus::RModule;
32
32
  use magnus::Ruby;
33
33
  use magnus::method;
34
- use magnus::prelude::*;
35
34
 
36
35
  use crate::metadata::Metadata;
37
36
  use crate::*;
data/src/metadata.rs CHANGED
@@ -25,7 +25,6 @@
25
25
  use magnus::Error;
26
26
  use magnus::RModule;
27
27
  use magnus::method;
28
- use magnus::prelude::*;
29
28
 
30
29
  use crate::*;
31
30
 
data/src/operator_info.rs CHANGED
@@ -23,7 +23,6 @@
23
23
  use magnus::Error;
24
24
  use magnus::RModule;
25
25
  use magnus::method;
26
- use magnus::prelude::*;
27
26
 
28
27
  use crate::capability::Capability;
29
28
  use crate::*;