opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -179,6 +179,7 @@ impl Service for PcloudBackend {
179
179
  type Lister = oio::PageLister<PcloudLister>;
180
180
  type Deleter = oio::OneShotDeleter<PcloudDeleter>;
181
181
  type Copier = oio::OneShotCopier;
182
+ type Composer = ();
182
183
 
183
184
  fn info(&self) -> ServiceInfo {
184
185
  self.core.info.clone()
@@ -223,7 +224,10 @@ impl Service for PcloudBackend {
223
224
 
224
225
  Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")))
225
226
  }
226
- _ => Err(parse_error(resp)),
227
+ _ => Err(parse_error(
228
+ ErrorContext::new(ServiceOperation("Stat")),
229
+ resp,
230
+ )),
227
231
  }
228
232
  }
229
233
  fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -276,15 +280,29 @@ impl Service for PcloudBackend {
276
280
  ctx: &OperationContext,
277
281
  from: &str,
278
282
  to: &str,
279
- _args: OpCopy,
280
- _opts: OpCopier,
283
+ args: OpCopy,
281
284
  ) -> Result<Self::Copier> {
285
+ let backend = self.clone();
282
286
  let core = self.core.clone();
283
287
  let ctx = ctx.clone();
284
288
  let from = from.to_string();
285
289
  let to = to.to_string();
290
+ let source_content_length_hint = args.source_content_length_hint();
286
291
 
287
292
  Ok(oio::OneShotCopier::new(async move {
293
+ let source_size = if from.ends_with('/') {
294
+ None
295
+ } else {
296
+ Some(match source_content_length_hint {
297
+ Some(size) => size,
298
+ None => backend
299
+ .stat(&ctx, &from, OpStat::default())
300
+ .await?
301
+ .into_metadata()
302
+ .content_length(),
303
+ })
304
+ };
305
+
288
306
  core.ensure_dir_exists(&ctx, &to).await?;
289
307
 
290
308
  let resp = if from.ends_with('/') {
@@ -306,10 +324,19 @@ impl Service for PcloudBackend {
306
324
  } else if result != 0 {
307
325
  Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")))
308
326
  } else {
309
- Ok(Metadata::default())
327
+ let metadata =
328
+ source_size.map_or_else(MetadataBuilder::dir, MetadataBuilder::file);
329
+ Ok(metadata.build())
310
330
  }
311
331
  }
312
- _ => Err(parse_error(resp)),
332
+ _ => Err(parse_error(
333
+ ErrorContext::new(if from.ends_with('/') {
334
+ ServiceOperation("CopyFolder")
335
+ } else {
336
+ ServiceOperation("CopyFile")
337
+ }),
338
+ resp,
339
+ )),
313
340
  }
314
341
  }))
315
342
  }
@@ -346,7 +373,14 @@ impl Service for PcloudBackend {
346
373
 
347
374
  Ok(RpRename::default())
348
375
  }
349
- _ => Err(parse_error(resp)),
376
+ _ => Err(parse_error(
377
+ ErrorContext::new(if from.ends_with('/') {
378
+ ServiceOperation("RenameFolder")
379
+ } else {
380
+ ServiceOperation("RenameFile")
381
+ }),
382
+ resp,
383
+ )),
350
384
  }
351
385
  }
352
386
 
@@ -107,7 +107,10 @@ impl PcloudCore {
107
107
  }
108
108
  Err(Error::new(ErrorKind::Unexpected, "hosts is empty"))
109
109
  }
110
- _ => Err(parse_error(resp)),
110
+ _ => Err(parse_error(
111
+ ErrorContext::new(ServiceOperation("GetFileLink")),
112
+ resp,
113
+ )),
111
114
  }
112
115
  }
113
116
 
@@ -158,7 +161,12 @@ impl PcloudCore {
158
161
  return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")));
159
162
  }
160
163
  }
161
- _ => return Err(parse_error(resp)),
164
+ _ => {
165
+ return Err(parse_error(
166
+ ErrorContext::new(ServiceOperation("CreateFolderIfNotExists")),
167
+ resp,
168
+ ));
169
+ }
162
170
  }
163
171
  }
164
172
  Ok(())
@@ -429,6 +437,8 @@ impl PcloudCore {
429
437
 
430
438
  let path = path.trim_end_matches('/');
431
439
 
440
+ let path = if path.is_empty() { "/" } else { path };
441
+
432
442
  let url = format!(
433
443
  "{}/listfolder?path={}&username={}&password={}",
434
444
  self.endpoint,
@@ -452,34 +462,36 @@ impl PcloudCore {
452
462
 
453
463
  pub(super) fn parse_stat_metadata(content: StatMetadata) -> Result<Metadata> {
454
464
  let mut md = if content.isfolder {
455
- Metadata::new(EntryMode::DIR)
465
+ MetadataBuilder::dir()
456
466
  } else {
457
- Metadata::new(EntryMode::FILE)
467
+ MetadataBuilder::file(content.size.ok_or_else(|| {
468
+ Error::new(
469
+ ErrorKind::Unexpected,
470
+ "pcloud stat response does not contain file size",
471
+ )
472
+ })?)
458
473
  };
459
474
 
460
- if let Some(size) = content.size {
461
- md.set_content_length(size);
462
- }
463
-
464
- md.set_last_modified(Timestamp::parse_rfc2822(&content.modified)?);
475
+ md.last_modified(Timestamp::parse_rfc2822(&content.modified)?);
465
476
 
466
- Ok(md)
477
+ Ok(md.build())
467
478
  }
468
479
 
469
480
  pub(super) fn parse_list_metadata(content: ListMetadata) -> Result<Metadata> {
470
481
  let mut md = if content.isfolder {
471
- Metadata::new(EntryMode::DIR)
482
+ MetadataBuilder::dir()
472
483
  } else {
473
- Metadata::new(EntryMode::FILE)
484
+ MetadataBuilder::file(content.size.ok_or_else(|| {
485
+ Error::new(
486
+ ErrorKind::Unexpected,
487
+ "pcloud list response does not contain file size",
488
+ )
489
+ })?)
474
490
  };
475
491
 
476
- if let Some(size) = content.size {
477
- md.set_content_length(size);
478
- }
479
-
480
- md.set_last_modified(Timestamp::parse_rfc2822(&content.modified)?);
492
+ md.last_modified(Timestamp::parse_rfc2822(&content.modified)?);
481
493
 
482
- Ok(md)
494
+ Ok(md.build())
483
495
  }
484
496
 
485
497
  #[derive(Debug, Deserialize)]
@@ -517,53 +529,58 @@ pub struct ListMetadata {
517
529
  pub contents: Option<Vec<ListMetadata>>,
518
530
  }
519
531
 
520
- mod error {
521
- use std::fmt::Debug;
522
-
523
- use http::Response;
524
- use opendal_core::raw::*;
525
- use opendal_core::*;
526
- use serde::Deserialize;
532
+ /// PcloudError is the error returned by Pcloud service.
533
+ #[derive(Default, Deserialize)]
534
+ pub(crate) struct PcloudError {
535
+ pub result: u32,
536
+ pub error: Option<String>,
537
+ }
527
538
 
528
- /// PcloudError is the error returned by Pcloud service.
529
- #[derive(Default, Deserialize)]
530
- pub(crate) struct PcloudError {
531
- pub result: u32,
532
- pub error: Option<String>,
539
+ impl Debug for PcloudError {
540
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
541
+ f.debug_struct("PcloudError")
542
+ .field("result", &self.result)
543
+ .field("error", &self.error)
544
+ .finish_non_exhaustive()
533
545
  }
546
+ }
534
547
 
535
- impl Debug for PcloudError {
536
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
537
- f.debug_struct("PcloudError")
538
- .field("result", &self.result)
539
- .field("error", &self.error)
540
- .finish_non_exhaustive()
541
- }
548
+ /// Context needed to classify an error from this service.
549
+ #[derive(Clone, Copy, Debug)]
550
+ pub(crate) struct ErrorContext {
551
+ service_operation: ServiceOperation,
552
+ }
553
+
554
+ impl ErrorContext {
555
+ pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
556
+ Self { service_operation }
542
557
  }
558
+ }
543
559
 
544
- /// Parse error response into Error.
545
- pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
546
- let (parts, body) = resp.into_parts();
547
- let bs = body.to_bytes();
548
- let message = String::from_utf8_lossy(&bs).into_owned();
560
+ /// Parse an error response using its service request context.
561
+ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
562
+ let (parts, body) = resp.into_parts();
563
+ let bs = body.to_bytes();
564
+ let message = String::from_utf8_lossy(&bs).into_owned();
549
565
 
550
- let mut err = Error::new(ErrorKind::Unexpected, message);
566
+ let mut err = Error::new(ErrorKind::Unexpected, message);
551
567
 
552
- err = with_error_response_context(err, parts);
568
+ err = err.with_context("service_operation", ctx.service_operation.0);
569
+ err = with_error_response_context(err, parts);
553
570
 
554
- err
555
- }
571
+ err
572
+ }
556
573
 
557
- #[cfg(test)]
558
- mod test {
559
- use http::StatusCode;
574
+ #[cfg(test)]
575
+ mod tests {
576
+ use http::StatusCode;
560
577
 
561
- use super::*;
578
+ use super::*;
562
579
 
563
- #[tokio::test]
564
- async fn test_parse_error() {
565
- let err_res = vec![(
566
- r#"<html>
580
+ #[tokio::test]
581
+ async fn test_parse_error() {
582
+ let err_res = vec![(
583
+ r#"<html>
567
584
 
568
585
  <head>
569
586
  <title>Invalid link</title>
@@ -572,21 +589,18 @@ mod error {
572
589
  <body>This link was generated for another IP address. Try previous step again.</body>
573
590
 
574
591
  </html> "#,
575
- ErrorKind::Unexpected,
576
- StatusCode::GONE,
577
- )];
592
+ ErrorKind::Unexpected,
593
+ StatusCode::GONE,
594
+ )];
578
595
 
579
- for res in err_res {
580
- let bs = bytes::Bytes::from(res.0);
581
- let body = Buffer::from(bs);
582
- let resp = Response::builder().status(res.2).body(body).unwrap();
596
+ for res in err_res {
597
+ let bs = bytes::Bytes::from(res.0);
598
+ let body = Buffer::from(bs);
599
+ let resp = Response::builder().status(res.2).body(body).unwrap();
583
600
 
584
- let err = parse_error(resp);
601
+ let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
585
602
 
586
- assert_eq!(err.kind(), res.1);
587
- }
603
+ assert_eq!(err.kind(), res.1);
588
604
  }
589
605
  }
590
606
  }
591
-
592
- pub(super) use error::*;
@@ -61,7 +61,14 @@ impl oio::OneShotDelete for PcloudDeleter {
61
61
 
62
62
  Ok(())
63
63
  }
64
- _ => Err(parse_error(resp)),
64
+ _ => Err(parse_error(
65
+ ErrorContext::new(if path.ends_with('/') {
66
+ ServiceOperation("DeleteFolder")
67
+ } else {
68
+ ServiceOperation("DeleteFile")
69
+ }),
70
+ resp,
71
+ )),
65
72
  }
66
73
  }
67
74
  }
@@ -90,7 +90,10 @@ impl oio::PageList for PcloudLister {
90
90
  String::from_utf8_lossy(&bs.to_bytes()),
91
91
  ))
92
92
  }
93
- _ => Err(parse_error(resp)),
93
+ _ => Err(parse_error(
94
+ ErrorContext::new(ServiceOperation("ListFolder")),
95
+ resp,
96
+ )),
94
97
  }
95
98
  }
96
99
  }
@@ -16,7 +16,7 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use super::core::parse_error;
19
+ use super::core::{ErrorContext, parse_error};
20
20
  use http::Response;
21
21
  use http::StatusCode;
22
22
  use opendal_core::raw::*;
@@ -62,7 +62,10 @@ impl oio::StreamRead for PcloudReader {
62
62
  _ => {
63
63
  let (part, mut body) = resp.into_parts();
64
64
  let buf = body.to_buffer().await?;
65
- return Err(parse_error(Response::from_parts(part, buf)));
65
+ return Err(parse_error(
66
+ ErrorContext::new(ServiceOperation("DownloadFile")),
67
+ Response::from_parts(part, buf),
68
+ ));
66
69
  }
67
70
  };
68
71
 
@@ -22,9 +22,9 @@ use http::StatusCode;
22
22
  use opendal_core::raw::*;
23
23
  use opendal_core::*;
24
24
 
25
- use super::core::PcloudCore;
26
25
  use super::core::PcloudError;
27
26
  use super::core::parse_error;
27
+ use super::core::{ErrorContext, PcloudCore};
28
28
 
29
29
  pub type PcloudWriters = oio::OneShotWriter<PcloudWriter>;
30
30
 
@@ -59,9 +59,12 @@ impl oio::OneShotWrite for PcloudWriter {
59
59
  return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")));
60
60
  }
61
61
 
62
- Ok(Metadata::default())
62
+ Ok(MetadataBuilder::unknown().build())
63
63
  }
64
- _ => Err(parse_error(resp)),
64
+ _ => Err(parse_error(
65
+ ErrorContext::new(ServiceOperation("UploadFile")),
66
+ resp,
67
+ )),
65
68
  }
66
69
  }
67
70
  }
@@ -32,7 +32,7 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
35
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
36
36
  persy = "1.7.1"
37
37
  serde = { workspace = true, features = ["derive"] }
38
38
 
@@ -154,6 +154,7 @@ impl Service for PersyBackend {
154
154
  type Lister = ();
155
155
  type Deleter = oio::OneShotDeleter<PersyDeleter>;
156
156
  type Copier = ();
157
+ type Composer = ();
157
158
 
158
159
  fn info(&self) -> ServiceInfo {
159
160
  self.info.clone()
@@ -179,13 +180,14 @@ impl Service for PersyBackend {
179
180
  let p = build_abs_path(&self.root, path);
180
181
 
181
182
  if p == build_abs_path(&self.root, "") {
182
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
183
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
183
184
  } else {
184
185
  let bs = self.core.get(&p)?;
185
186
  match bs {
186
- Some(bs) => Ok(RpStat::new(
187
- Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
188
- )),
187
+ Some(bs) => Ok(RpStat::new({
188
+ let metadata = MetadataBuilder::file(bs.len() as u64);
189
+ metadata.build()
190
+ })),
189
191
  None => Err(Error::new(ErrorKind::NotFound, "kv not found in persy")),
190
192
  }
191
193
  }
@@ -235,7 +237,6 @@ impl Service for PersyBackend {
235
237
  _from: &str,
236
238
  _to: &str,
237
239
  _args: OpCopy,
238
- _opts: OpCopier,
239
240
  ) -> Result<Self::Copier> {
240
241
  Err(Error::new(
241
242
  ErrorKind::Unsupported,
@@ -17,8 +17,21 @@
17
17
 
18
18
  use std::fmt::Debug;
19
19
 
20
+ use opendal_core::raw::ServiceOperation;
20
21
  use opendal_core::*;
21
22
 
23
+ /// Context needed to classify an error from this service.
24
+ #[derive(Clone, Copy, Debug)]
25
+ struct ErrorContext {
26
+ service_operation: ServiceOperation,
27
+ }
28
+
29
+ impl ErrorContext {
30
+ const fn new(service_operation: ServiceOperation) -> Self {
31
+ Self { service_operation }
32
+ }
33
+ }
34
+
22
35
  #[derive(Clone)]
23
36
  pub struct PersyCore {
24
37
  pub datafile: String,
@@ -42,9 +55,11 @@ impl PersyCore {
42
55
  let mut read_id = self
43
56
  .persy
44
57
  .get::<String, persy::PersyId>(&self.index, &path.to_string())
45
- .map_err(parse_error)?;
58
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("GetIndex")), err))?;
46
59
  if let Some(id) = read_id.next() {
47
- let value = self.persy.read(&self.segment, &id).map_err(parse_error)?;
60
+ let value = self.persy.read(&self.segment, &id).map_err(|err| {
61
+ parse_error(ErrorContext::new(ServiceOperation("ReadRecord")), err)
62
+ })?;
48
63
  return Ok(value.map(Buffer::from));
49
64
  }
50
65
 
@@ -52,15 +67,27 @@ impl PersyCore {
52
67
  }
53
68
 
54
69
  pub fn set(&self, path: &str, value: Buffer) -> Result<()> {
55
- let mut tx = self.persy.begin().map_err(parse_error)?;
70
+ let mut tx = self.persy.begin().map_err(|err| {
71
+ parse_error(ErrorContext::new(ServiceOperation("BeginTransaction")), err)
72
+ })?;
56
73
  let id = tx
57
74
  .insert(&self.segment, &value.to_vec())
58
- .map_err(parse_error)?;
75
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("InsertRecord")), err))?;
59
76
 
60
77
  tx.put::<String, persy::PersyId>(&self.index, path.to_string(), id)
61
- .map_err(parse_error)?;
62
- let prepared = tx.prepare().map_err(parse_error)?;
63
- prepared.commit().map_err(parse_error)?;
78
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("PutIndex")), err))?;
79
+ let prepared = tx.prepare().map_err(|err| {
80
+ parse_error(
81
+ ErrorContext::new(ServiceOperation("PrepareTransaction")),
82
+ err,
83
+ )
84
+ })?;
85
+ prepared.commit().map_err(|err| {
86
+ parse_error(
87
+ ErrorContext::new(ServiceOperation("CommitTransaction")),
88
+ err,
89
+ )
90
+ })?;
64
91
 
65
92
  Ok(())
66
93
  }
@@ -69,30 +96,48 @@ impl PersyCore {
69
96
  let mut delete_id = self
70
97
  .persy
71
98
  .get::<String, persy::PersyId>(&self.index, &path.to_string())
72
- .map_err(parse_error)?;
99
+ .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("GetIndex")), err))?;
73
100
  if let Some(id) = delete_id.next() {
74
101
  // Begin a transaction.
75
- let mut tx = self.persy.begin().map_err(parse_error)?;
102
+ let mut tx = self.persy.begin().map_err(|err| {
103
+ parse_error(ErrorContext::new(ServiceOperation("BeginTransaction")), err)
104
+ })?;
76
105
  // Delete the record.
77
- tx.delete(&self.segment, &id).map_err(parse_error)?;
106
+ tx.delete(&self.segment, &id).map_err(|err| {
107
+ parse_error(ErrorContext::new(ServiceOperation("DeleteRecord")), err)
108
+ })?;
78
109
  // Remove the index.
79
110
  tx.remove::<String, persy::PersyId>(&self.index, path.to_string(), Some(id))
80
- .map_err(parse_error)?;
111
+ .map_err(|err| {
112
+ parse_error(ErrorContext::new(ServiceOperation("RemoveIndex")), err)
113
+ })?;
81
114
  // Commit the tx.
82
- let prepared = tx.prepare().map_err(parse_error)?;
83
- prepared.commit().map_err(parse_error)?;
115
+ let prepared = tx.prepare().map_err(|err| {
116
+ parse_error(
117
+ ErrorContext::new(ServiceOperation("PrepareTransaction")),
118
+ err,
119
+ )
120
+ })?;
121
+ prepared.commit().map_err(|err| {
122
+ parse_error(
123
+ ErrorContext::new(ServiceOperation("CommitTransaction")),
124
+ err,
125
+ )
126
+ })?;
84
127
  }
85
128
 
86
129
  Ok(())
87
130
  }
88
131
  }
89
132
 
90
- fn parse_error<T: Into<persy::PersyError>>(err: persy::PE<T>) -> Error {
133
+ fn parse_error<T: Into<persy::PersyError>>(ctx: ErrorContext, err: persy::PE<T>) -> Error {
91
134
  let err: persy::PersyError = err.persy_error();
92
135
  let kind = match err {
93
136
  persy::PersyError::RecordNotFound(_) => ErrorKind::NotFound,
94
137
  _ => ErrorKind::Unexpected,
95
138
  };
96
139
 
97
- Error::new(kind, "error from persy").set_source(err)
140
+ Error::new(kind, "error from persy")
141
+ .with_context("service_operation", ctx.service_operation.0)
142
+ .set_source(err)
98
143
  }
@@ -46,7 +46,10 @@ impl oio::StreamRead for PersyReader {
46
46
  }
47
47
  };
48
48
  let content = bs.slice(range.to_content_range(bs.len())?);
49
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
49
+ let metadata = {
50
+ let metadata = MetadataBuilder::file(bs.len() as u64);
51
+ metadata.build()
52
+ };
50
53
  Ok((
51
54
  RpRead::new(metadata),
52
55
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -48,8 +48,12 @@ impl oio::Write for PersyWriter {
48
48
  let length = buf.len() as u64;
49
49
  self.core.set(&self.path, buf)?;
50
50
 
51
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
52
- Ok(meta)
51
+ let metadata = if self.path.ends_with('/') {
52
+ MetadataBuilder::dir()
53
+ } else {
54
+ MetadataBuilder::file(length)
55
+ };
56
+ Ok(metadata.build())
53
57
  }
54
58
 
55
59
  async fn abort(&mut self) -> Result<()> {
@@ -32,8 +32,8 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- mea = { workspace = true }
36
- opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
35
+ asyncband = { workspace = true }
36
+ opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
37
37
  serde = { workspace = true, features = ["derive"] }
38
38
  sqlx = { version = "0.8.0", features = ["runtime-tokio-rustls", "postgres"] }
39
39
 
@@ -17,7 +17,7 @@
17
17
 
18
18
  use std::sync::Arc;
19
19
 
20
- use mea::once::OnceCell;
20
+ use asyncband::once::OnceCell;
21
21
  use sqlx::postgres::PgConnectOptions;
22
22
 
23
23
  use super::POSTGRESQL_SCHEME;
@@ -187,6 +187,7 @@ impl Service for PostgresqlBackend {
187
187
  type Lister = ();
188
188
  type Deleter = oio::OneShotDeleter<PostgresqlDeleter>;
189
189
  type Copier = ();
190
+ type Composer = ();
190
191
 
191
192
  fn info(&self) -> ServiceInfo {
192
193
  self.info.clone()
@@ -212,12 +213,13 @@ impl Service for PostgresqlBackend {
212
213
  let p = build_abs_path(&self.root, path);
213
214
 
214
215
  if p == build_abs_path(&self.root, "") {
215
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
216
+ Ok(RpStat::new(MetadataBuilder::dir().build()))
216
217
  } else {
217
218
  match self.core.get_length(&p).await? {
218
- Some(length) => Ok(RpStat::new(
219
- Metadata::new(EntryMode::FILE).with_content_length(length as u64),
220
- )),
219
+ Some(length) => Ok(RpStat::new({
220
+ let metadata = MetadataBuilder::file(length as u64);
221
+ metadata.build()
222
+ })),
221
223
  None => Err(Error::new(
222
224
  ErrorKind::NotFound,
223
225
  "kv not found in postgresql",
@@ -270,7 +272,6 @@ impl Service for PostgresqlBackend {
270
272
  _from: &str,
271
273
  _to: &str,
272
274
  _args: OpCopy,
273
- _opts: OpCopier,
274
275
  ) -> Result<Self::Copier> {
275
276
  Err(Error::new(
276
277
  ErrorKind::Unsupported,
@@ -15,7 +15,7 @@
15
15
  // specific language governing permissions and limitations
16
16
  // under the License.
17
17
 
18
- use mea::once::OnceCell;
18
+ use asyncband::once::OnceCell;
19
19
  use sqlx::PgPool;
20
20
  use sqlx::postgres::PgConnectOptions;
21
21
 
@@ -49,7 +49,10 @@ impl oio::StreamRead for PostgresqlReader {
49
49
  }
50
50
  };
51
51
  let content = bs.slice(range.to_content_range(bs.len())?);
52
- let metadata = Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64);
52
+ let metadata = {
53
+ let metadata = MetadataBuilder::file(bs.len() as u64);
54
+ metadata.build()
55
+ };
53
56
  Ok((
54
57
  RpRead::new(metadata),
55
58
  Box::new(content) as Box<dyn oio::ReadStreamDyn>,
@@ -48,8 +48,12 @@ impl oio::Write for PostgresqlWriter {
48
48
  let length = buf.len() as u64;
49
49
  self.core.set(&self.path, buf).await?;
50
50
 
51
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
52
- Ok(meta)
51
+ let metadata = if self.path.ends_with('/') {
52
+ MetadataBuilder::dir()
53
+ } else {
54
+ MetadataBuilder::file(length)
55
+ };
56
+ Ok(metadata.build())
53
57
  }
54
58
 
55
59
  async fn abort(&mut self) -> Result<()> {