opendal 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (566) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +89 -56
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +55 -41
  5. data/LICENSE +201 -0
  6. data/LICENSE-MPL-2.0.txt +373 -0
  7. data/NOTICE +5 -0
  8. data/core/CHANGELOG.md +175 -0
  9. data/core/Cargo.lock +225 -298
  10. data/core/Cargo.toml +100 -94
  11. data/core/DEPENDENCIES.rust.tsv +44 -73
  12. data/core/benches/metadata/README.md +21 -0
  13. data/core/benches/metadata/main.rs +541 -0
  14. data/core/benches/vs_fs/Cargo.toml +1 -1
  15. data/core/benches/vs_s3/Cargo.toml +1 -1
  16. data/core/core/Cargo.toml +7 -2
  17. data/core/core/src/blocking/delete.rs +5 -1
  18. data/core/core/src/blocking/operator.rs +12 -0
  19. data/core/core/src/blocking/read/std_reader.rs +25 -0
  20. data/core/core/src/docs/concepts.rs +70 -94
  21. data/core/core/src/docs/internals/accessor.rs +62 -293
  22. data/core/core/src/docs/internals/layer.rs +43 -59
  23. data/core/core/src/docs/internals/mod.rs +21 -23
  24. data/core/core/src/docs/mod.rs +10 -6
  25. data/core/core/src/docs/performance/concurrent_write.md +92 -55
  26. data/core/core/src/docs/performance/http_optimization.md +120 -104
  27. data/core/core/src/docs/performance/mod.rs +8 -6
  28. data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
  29. data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
  30. data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
  31. data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
  32. data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
  33. data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
  34. data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
  35. data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
  36. data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
  37. data/core/core/src/docs/rfcs/README.md +14 -3
  38. data/core/core/src/docs/rfcs/mod.rs +46 -22
  39. data/core/core/src/docs/specs/README.md +34 -0
  40. data/core/core/src/docs/specs/conditional-operations.md +156 -0
  41. data/core/core/src/docs/specs/metadata.md +63 -0
  42. data/core/core/src/docs/specs/mod.rs +26 -0
  43. data/core/core/src/docs/upgrade.md +114 -1
  44. data/core/core/src/layers/capability_override.rs +15 -2
  45. data/core/core/src/layers/complete.rs +347 -43
  46. data/core/core/src/layers/correctness_check.rs +718 -51
  47. data/core/core/src/layers/error_context.rs +76 -2
  48. data/core/core/src/layers/simulate.rs +91 -28
  49. data/core/core/src/lib.rs +1 -0
  50. data/core/core/src/raw/accessor.rs +91 -15
  51. data/core/core/src/raw/enum_utils.rs +7 -0
  52. data/core/core/src/raw/http_util/header.rs +22 -17
  53. data/core/core/src/raw/oio/compose/api.rs +92 -0
  54. data/core/core/src/raw/oio/compose/mod.rs +19 -0
  55. data/core/core/src/raw/oio/copy/api.rs +3 -3
  56. data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
  57. data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
  58. data/core/core/src/raw/oio/entry.rs +48 -22
  59. data/core/core/src/raw/oio/list/flat_list.rs +1 -1
  60. data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
  61. data/core/core/src/raw/oio/mod.rs +3 -0
  62. data/core/core/src/raw/oio/read/position_read.rs +1 -1
  63. data/core/core/src/raw/oio/write/api.rs +40 -0
  64. data/core/core/src/raw/oio/write/append_write.rs +4 -4
  65. data/core/core/src/raw/oio/write/block_write.rs +2 -2
  66. data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
  67. data/core/core/src/raw/oio/write/position_write.rs +1 -1
  68. data/core/core/src/raw/operation.rs +6 -0
  69. data/core/core/src/raw/ops.rs +1191 -456
  70. data/core/core/src/raw/rps.rs +11 -0
  71. data/core/core/src/raw/time.rs +14 -3
  72. data/core/core/src/services/memory/backend.rs +5 -10
  73. data/core/core/src/services/memory/lister.rs +4 -4
  74. data/core/core/src/services/memory/writer.rs +7 -6
  75. data/core/core/src/types/buffer.rs +102 -12
  76. data/core/core/src/types/bytes_range.rs +20 -1
  77. data/core/core/src/types/capability.rs +65 -0
  78. data/core/core/src/types/compact.rs +329 -0
  79. data/core/core/src/types/compose/composer.rs +183 -0
  80. data/core/core/src/types/compose/input.rs +57 -0
  81. data/core/core/src/types/compose/mod.rs +23 -0
  82. data/core/core/src/types/context/read.rs +16 -11
  83. data/core/core/src/types/context/write.rs +150 -2
  84. data/core/core/src/types/copy.rs +5 -3
  85. data/core/core/src/types/delete/deleter.rs +20 -12
  86. data/core/core/src/types/delete/input.rs +22 -64
  87. data/core/core/src/types/delete/mod.rs +0 -1
  88. data/core/core/src/types/entry.rs +8 -14
  89. data/core/core/src/types/error.rs +28 -11
  90. data/core/core/src/types/metadata.rs +709 -322
  91. data/core/core/src/types/mod.rs +9 -1
  92. data/core/core/src/types/operator/operator.rs +278 -29
  93. data/core/core/src/types/operator/operator_futures.rs +466 -105
  94. data/core/core/src/types/options.rs +631 -158
  95. data/core/core/src/types/read/futures_async_reader.rs +663 -50
  96. data/core/core/src/types/read/reader.rs +23 -5
  97. data/core/core/src/types/write/futures_async_writer.rs +27 -0
  98. data/core/core/src/types/write/writer.rs +47 -5
  99. data/core/http-transports/reqwest/Cargo.toml +1 -1
  100. data/core/layers/async-backtrace/Cargo.toml +2 -2
  101. data/core/layers/async-backtrace/src/lib.rs +21 -2
  102. data/core/layers/await-tree/Cargo.toml +2 -2
  103. data/core/layers/await-tree/src/lib.rs +25 -2
  104. data/core/layers/capability-check/Cargo.toml +2 -2
  105. data/core/layers/capability-check/src/lib.rs +98 -10
  106. data/core/layers/chaos/Cargo.toml +2 -2
  107. data/core/layers/chaos/src/lib.rs +18 -3
  108. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  109. data/core/layers/concurrent-limit/src/lib.rs +53 -13
  110. data/core/layers/dtrace/Cargo.toml +2 -2
  111. data/core/layers/dtrace/src/lib.rs +29 -2
  112. data/core/layers/fastmetrics/Cargo.toml +3 -3
  113. data/core/layers/fastrace/Cargo.toml +2 -2
  114. data/core/layers/fastrace/src/lib.rs +28 -2
  115. data/core/layers/foyer/Cargo.toml +2 -2
  116. data/core/layers/foyer/src/full.rs +16 -19
  117. data/core/layers/foyer/src/lib.rs +67 -14
  118. data/core/layers/hotpath/Cargo.toml +2 -2
  119. data/core/layers/hotpath/src/lib.rs +20 -2
  120. data/core/layers/immutable-index/Cargo.toml +2 -2
  121. data/core/layers/immutable-index/src/lib.rs +48 -24
  122. data/core/layers/logging/Cargo.toml +2 -2
  123. data/core/layers/logging/src/lib.rs +46 -2
  124. data/core/layers/metrics/Cargo.toml +3 -3
  125. data/core/layers/mime-guess/Cargo.toml +2 -2
  126. data/core/layers/mime-guess/src/lib.rs +31 -4
  127. data/core/layers/observe-metrics-common/Cargo.toml +2 -2
  128. data/core/layers/observe-metrics-common/src/lib.rs +55 -2
  129. data/core/layers/otelmetrics/Cargo.toml +3 -3
  130. data/core/layers/oteltrace/Cargo.toml +2 -2
  131. data/core/layers/oteltrace/src/lib.rs +27 -2
  132. data/core/layers/prometheus/Cargo.toml +3 -3
  133. data/core/layers/prometheus-client/Cargo.toml +3 -3
  134. data/core/layers/retry/Cargo.toml +4 -4
  135. data/core/layers/retry/src/lib.rs +76 -19
  136. data/core/layers/route/Cargo.toml +3 -3
  137. data/core/layers/route/src/lib.rs +34 -7
  138. data/core/layers/tail-cut/Cargo.toml +2 -2
  139. data/core/layers/tail-cut/src/lib.rs +28 -2
  140. data/core/layers/throttle/Cargo.toml +2 -2
  141. data/core/layers/throttle/src/lib.rs +19 -2
  142. data/core/layers/timeout/Cargo.toml +3 -3
  143. data/core/layers/timeout/src/lib.rs +38 -13
  144. data/core/layers/tracing/Cargo.toml +2 -2
  145. data/core/layers/tracing/src/lib.rs +24 -3
  146. data/core/services/aliyun-drive/Cargo.toml +2 -2
  147. data/core/services/aliyun-drive/src/backend.rs +34 -24
  148. data/core/services/aliyun-drive/src/core.rs +143 -54
  149. data/core/services/aliyun-drive/src/lister.rs +19 -18
  150. data/core/services/aliyun-drive/src/reader.rs +4 -1
  151. data/core/services/aliyun-drive/src/writer.rs +3 -3
  152. data/core/services/alluxio/Cargo.toml +1 -1
  153. data/core/services/alluxio/src/backend.rs +1 -1
  154. data/core/services/alluxio/src/core.rs +113 -94
  155. data/core/services/alluxio/src/reader.rs +7 -3
  156. data/core/services/alluxio/src/writer.rs +2 -2
  157. data/core/services/azblob/Cargo.toml +2 -2
  158. data/core/services/azblob/src/backend.rs +17 -9
  159. data/core/services/azblob/src/copier.rs +41 -15
  160. data/core/services/azblob/src/core.rs +372 -135
  161. data/core/services/azblob/src/deleter.rs +41 -18
  162. data/core/services/azblob/src/lister.rs +16 -10
  163. data/core/services/azblob/src/reader.rs +4 -1
  164. data/core/services/azblob/src/writer.rs +48 -15
  165. data/core/services/azdls/Cargo.toml +3 -3
  166. data/core/services/azdls/src/backend.rs +45 -15
  167. data/core/services/azdls/src/config.rs +4 -1
  168. data/core/services/azdls/src/core.rs +276 -87
  169. data/core/services/azdls/src/deleter.rs +13 -4
  170. data/core/services/azdls/src/lister.rs +18 -12
  171. data/core/services/azdls/src/reader.rs +6 -2
  172. data/core/services/azdls/src/writer.rs +44 -23
  173. data/core/services/azfile/Cargo.toml +2 -2
  174. data/core/services/azfile/src/backend.rs +24 -8
  175. data/core/services/azfile/src/core.rs +195 -77
  176. data/core/services/azfile/src/deleter.rs +8 -1
  177. data/core/services/azfile/src/lister.rs +20 -14
  178. data/core/services/azfile/src/reader.rs +5 -2
  179. data/core/services/azfile/src/writer.rs +27 -15
  180. data/core/services/azure-common/Cargo.toml +1 -1
  181. data/core/services/b2/Cargo.toml +2 -2
  182. data/core/services/b2/src/backend.rs +30 -11
  183. data/core/services/b2/src/core.rs +134 -120
  184. data/core/services/b2/src/deleter.rs +1 -1
  185. data/core/services/b2/src/lister.rs +5 -2
  186. data/core/services/b2/src/reader.rs +5 -2
  187. data/core/services/b2/src/writer.rs +29 -12
  188. data/core/services/cacache/Cargo.toml +1 -1
  189. data/core/services/cacache/src/backend.rs +4 -5
  190. data/core/services/cacache/src/reader.rs +4 -1
  191. data/core/services/cacache/src/writer.rs +6 -2
  192. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  193. data/core/services/cloudflare-kv/src/backend.rs +26 -15
  194. data/core/services/cloudflare-kv/src/config.rs +13 -2
  195. data/core/services/cloudflare-kv/src/core.rs +62 -56
  196. data/core/services/cloudflare-kv/src/deleter.rs +12 -3
  197. data/core/services/cloudflare-kv/src/lister.rs +66 -17
  198. data/core/services/cloudflare-kv/src/reader.rs +14 -9
  199. data/core/services/cloudflare-kv/src/writer.rs +10 -7
  200. data/core/services/compfs/Cargo.toml +1 -1
  201. data/core/services/compfs/src/backend.rs +11 -7
  202. data/core/services/compfs/src/lister.rs +7 -7
  203. data/core/services/compfs/src/writer.rs +1 -1
  204. data/core/services/cos/Cargo.toml +1 -1
  205. data/core/services/cos/src/backend.rs +129 -16
  206. data/core/services/cos/src/core.rs +198 -80
  207. data/core/services/cos/src/deleter.rs +4 -1
  208. data/core/services/cos/src/lister.rs +42 -27
  209. data/core/services/cos/src/reader.rs +6 -2
  210. data/core/services/cos/src/writer.rs +45 -21
  211. data/core/services/d1/Cargo.toml +1 -1
  212. data/core/services/d1/src/backend.rs +6 -5
  213. data/core/services/d1/src/core.rs +79 -63
  214. data/core/services/d1/src/reader.rs +4 -1
  215. data/core/services/d1/src/writer.rs +6 -2
  216. data/core/services/dashmap/Cargo.toml +1 -1
  217. data/core/services/dashmap/src/backend.rs +2 -2
  218. data/core/services/dashmap/src/reader.rs +4 -1
  219. data/core/services/dashmap/src/writer.rs +11 -8
  220. data/core/services/dbfs/Cargo.toml +1 -1
  221. data/core/services/dbfs/src/backend.rs +21 -15
  222. data/core/services/dbfs/src/core.rs +65 -60
  223. data/core/services/dbfs/src/deleter.rs +4 -1
  224. data/core/services/dbfs/src/lister.rs +10 -10
  225. data/core/services/dbfs/src/writer.rs +6 -3
  226. data/core/services/dropbox/Cargo.toml +2 -2
  227. data/core/services/dropbox/src/backend.rs +30 -23
  228. data/core/services/dropbox/src/core.rs +65 -63
  229. data/core/services/dropbox/src/deleter.rs +1 -1
  230. data/core/services/dropbox/src/lister.rs +17 -8
  231. data/core/services/dropbox/src/reader.rs +4 -1
  232. data/core/services/dropbox/src/writer.rs +13 -10
  233. data/core/services/etcd/Cargo.toml +1 -1
  234. data/core/services/etcd/src/backend.rs +8 -5
  235. data/core/services/etcd/src/core.rs +5 -11
  236. data/core/services/etcd/src/lister.rs +6 -1
  237. data/core/services/etcd/src/reader.rs +4 -1
  238. data/core/services/etcd/src/writer.rs +6 -3
  239. data/core/services/foundationdb/Cargo.toml +1 -1
  240. data/core/services/foundationdb/src/backend.rs +6 -5
  241. data/core/services/foundationdb/src/reader.rs +4 -1
  242. data/core/services/foundationdb/src/writer.rs +6 -2
  243. data/core/services/foyer/Cargo.toml +3 -3
  244. data/core/services/foyer/src/backend.rs +6 -5
  245. data/core/services/foyer/src/reader.rs +4 -1
  246. data/core/services/foyer/src/writer.rs +6 -2
  247. data/core/services/fs/Cargo.toml +1 -1
  248. data/core/services/fs/src/backend.rs +4 -3
  249. data/core/services/fs/src/core.rs +44 -26
  250. data/core/services/fs/src/lister.rs +9 -6
  251. data/core/services/fs/src/writer.rs +20 -13
  252. data/core/services/ftp/Cargo.toml +4 -6
  253. data/core/services/ftp/src/backend.rs +9 -5
  254. data/core/services/ftp/src/core.rs +9 -7
  255. data/core/services/ftp/src/lister.rs +7 -8
  256. data/core/services/ftp/src/reader.rs +3 -3
  257. data/core/services/ftp/src/writer.rs +3 -3
  258. data/core/services/gcs/Cargo.toml +4 -2
  259. data/core/services/gcs/src/backend.rs +53 -7
  260. data/core/services/gcs/src/composer.rs +705 -0
  261. data/core/services/gcs/src/copier.rs +14 -4
  262. data/core/services/gcs/src/core.rs +828 -124
  263. data/core/services/gcs/src/deleter.rs +33 -18
  264. data/core/services/gcs/src/lib.rs +1 -0
  265. data/core/services/gcs/src/lister.rs +21 -12
  266. data/core/services/gcs/src/reader.rs +12 -5
  267. data/core/services/gcs/src/writer.rs +632 -10
  268. data/core/services/gcs-grpc/Cargo.toml +58 -0
  269. data/core/services/gcs-grpc/README.md +19 -0
  270. data/core/services/gcs-grpc/proto/README.md +23 -0
  271. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
  272. data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
  273. data/core/services/gcs-grpc/src/backend.rs +449 -0
  274. data/core/services/gcs-grpc/src/config.rs +116 -0
  275. data/core/services/gcs-grpc/src/copier.rs +98 -0
  276. data/core/services/gcs-grpc/src/core.rs +298 -0
  277. data/core/services/gcs-grpc/src/deleter.rs +65 -0
  278. data/core/services/gcs-grpc/src/docs.md +19 -0
  279. data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
  280. data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
  281. data/core/services/gcs-grpc/src/lib.rs +43 -0
  282. data/core/services/gcs-grpc/src/lister.rs +125 -0
  283. data/core/services/gcs-grpc/src/reader.rs +140 -0
  284. data/core/services/gcs-grpc/src/writer.rs +780 -0
  285. data/core/services/gdrive/Cargo.toml +2 -2
  286. data/core/services/gdrive/src/backend.rs +123 -69
  287. data/core/services/gdrive/src/core.rs +87 -69
  288. data/core/services/gdrive/src/deleter.rs +8 -2
  289. data/core/services/gdrive/src/lister.rs +38 -26
  290. data/core/services/gdrive/src/path_index.rs +2 -2
  291. data/core/services/gdrive/src/reader.rs +9 -3
  292. data/core/services/gdrive/src/writer.rs +13 -7
  293. data/core/services/ghac/Cargo.toml +2 -2
  294. data/core/services/ghac/src/backend.rs +5 -2
  295. data/core/services/ghac/src/core.rs +59 -37
  296. data/core/services/ghac/src/reader.rs +5 -2
  297. data/core/services/ghac/src/writer.rs +16 -6
  298. data/core/services/github/Cargo.toml +1 -1
  299. data/core/services/github/src/backend.rs +16 -8
  300. data/core/services/github/src/core.rs +88 -76
  301. data/core/services/github/src/lister.rs +17 -15
  302. data/core/services/github/src/reader.rs +5 -2
  303. data/core/services/github/src/writer.rs +12 -6
  304. data/core/services/goosefs/Cargo.toml +2 -2
  305. data/core/services/goosefs/src/backend.rs +2 -2
  306. data/core/services/goosefs/src/core.rs +291 -208
  307. data/core/services/goosefs/src/docs.md +3 -0
  308. data/core/services/goosefs/src/lister.rs +2 -3
  309. data/core/services/goosefs/src/reader.rs +7 -3
  310. data/core/services/goosefs/src/writer.rs +21 -10
  311. data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
  312. data/core/services/gridfs/Cargo.toml +2 -2
  313. data/core/services/gridfs/src/backend.rs +12 -8
  314. data/core/services/gridfs/src/core.rs +1 -1
  315. data/core/services/gridfs/src/reader.rs +4 -1
  316. data/core/services/gridfs/src/writer.rs +6 -2
  317. data/core/services/hdfs/Cargo.toml +1 -1
  318. data/core/services/hdfs/src/backend.rs +1 -1
  319. data/core/services/hdfs/src/core.rs +7 -4
  320. data/core/services/hdfs/src/lister.rs +7 -9
  321. data/core/services/hdfs/src/writer.rs +5 -1
  322. data/core/services/hdfs-native/Cargo.toml +1 -1
  323. data/core/services/hdfs-native/src/backend.rs +1 -1
  324. data/core/services/hdfs-native/src/core.rs +44 -50
  325. data/core/services/hdfs-native/src/lister.rs +8 -10
  326. data/core/services/hdfs-native/src/writer.rs +5 -1
  327. data/core/services/hf/Cargo.toml +4 -3
  328. data/core/services/hf/src/backend.rs +4 -3
  329. data/core/services/hf/src/core.rs +907 -153
  330. data/core/services/hf/src/lister.rs +16 -6
  331. data/core/services/hf/src/reader.rs +377 -57
  332. data/core/services/hf/src/writer.rs +6 -2
  333. data/core/services/http/Cargo.toml +3 -3
  334. data/core/services/http/src/backend.rs +8 -6
  335. data/core/services/http/src/core.rs +57 -35
  336. data/core/services/http/src/reader.rs +5 -2
  337. data/core/services/ipfs/Cargo.toml +1 -1
  338. data/core/services/ipfs/src/backend.rs +20 -9
  339. data/core/services/ipfs/src/core.rs +43 -34
  340. data/core/services/ipfs/src/reader.rs +5 -2
  341. data/core/services/ipmfs/Cargo.toml +1 -1
  342. data/core/services/ipmfs/src/backend.rs +17 -8
  343. data/core/services/ipmfs/src/core.rs +66 -60
  344. data/core/services/ipmfs/src/deleter.rs +5 -2
  345. data/core/services/ipmfs/src/lister.rs +15 -12
  346. data/core/services/ipmfs/src/reader.rs +5 -2
  347. data/core/services/ipmfs/src/writer.rs +6 -3
  348. data/core/services/koofr/Cargo.toml +2 -2
  349. data/core/services/koofr/src/backend.rs +46 -17
  350. data/core/services/koofr/src/core.rs +64 -48
  351. data/core/services/koofr/src/deleter.rs +4 -1
  352. data/core/services/koofr/src/lister.rs +11 -10
  353. data/core/services/koofr/src/reader.rs +5 -2
  354. data/core/services/koofr/src/writer.rs +13 -7
  355. data/core/services/lakefs/Cargo.toml +1 -1
  356. data/core/services/lakefs/src/backend.rs +25 -8
  357. data/core/services/lakefs/src/core.rs +79 -73
  358. data/core/services/lakefs/src/deleter.rs +4 -1
  359. data/core/services/lakefs/src/lister.rs +17 -14
  360. data/core/services/lakefs/src/reader.rs +5 -2
  361. data/core/services/lakefs/src/writer.rs +11 -10
  362. data/core/services/memcached/Cargo.toml +1 -1
  363. data/core/services/memcached/src/backend.rs +57 -7
  364. data/core/services/memcached/src/config.rs +13 -2
  365. data/core/services/memcached/src/core.rs +11 -1
  366. data/core/services/memcached/src/reader.rs +4 -1
  367. data/core/services/memcached/src/writer.rs +6 -2
  368. data/core/services/mini_moka/Cargo.toml +1 -1
  369. data/core/services/mini_moka/src/backend.rs +8 -8
  370. data/core/services/mini_moka/src/reader.rs +4 -1
  371. data/core/services/mini_moka/src/writer.rs +11 -7
  372. data/core/services/moka/Cargo.toml +1 -1
  373. data/core/services/moka/src/backend.rs +7 -7
  374. data/core/services/moka/src/reader.rs +4 -1
  375. data/core/services/moka/src/writer.rs +10 -6
  376. data/core/services/mongodb/Cargo.toml +2 -2
  377. data/core/services/mongodb/src/backend.rs +7 -6
  378. data/core/services/mongodb/src/core.rs +1 -1
  379. data/core/services/mongodb/src/reader.rs +4 -1
  380. data/core/services/mongodb/src/writer.rs +6 -2
  381. data/core/services/monoiofs/Cargo.toml +2 -2
  382. data/core/services/monoiofs/src/backend.rs +16 -10
  383. data/core/services/monoiofs/src/core.rs +8 -9
  384. data/core/services/monoiofs/src/reader.rs +8 -10
  385. data/core/services/monoiofs/src/writer.rs +19 -24
  386. data/core/services/mysql/Cargo.toml +2 -2
  387. data/core/services/mysql/src/backend.rs +7 -6
  388. data/core/services/mysql/src/core.rs +1 -1
  389. data/core/services/mysql/src/lister.rs +7 -3
  390. data/core/services/mysql/src/reader.rs +4 -1
  391. data/core/services/mysql/src/writer.rs +6 -2
  392. data/core/services/obs/Cargo.toml +1 -1
  393. data/core/services/obs/src/backend.rs +29 -11
  394. data/core/services/obs/src/core.rs +81 -82
  395. data/core/services/obs/src/deleter.rs +4 -1
  396. data/core/services/obs/src/lister.rs +12 -5
  397. data/core/services/obs/src/reader.rs +5 -2
  398. data/core/services/obs/src/writer.rs +40 -19
  399. data/core/services/onedrive/Cargo.toml +2 -2
  400. data/core/services/onedrive/src/backend.rs +30 -21
  401. data/core/services/onedrive/src/config.rs +2 -2
  402. data/core/services/onedrive/src/core.rs +191 -90
  403. data/core/services/onedrive/src/deleter.rs +5 -2
  404. data/core/services/onedrive/src/graph_model.rs +10 -51
  405. data/core/services/onedrive/src/lister.rs +60 -24
  406. data/core/services/onedrive/src/reader.rs +17 -2
  407. data/core/services/onedrive/src/writer.rs +26 -15
  408. data/core/services/opfs/Cargo.toml +1 -1
  409. data/core/services/opfs/src/backend.rs +6 -13
  410. data/core/services/opfs/src/core.rs +33 -37
  411. data/core/services/opfs/src/lister.rs +4 -4
  412. data/core/services/opfs/src/writer.rs +1 -3
  413. data/core/services/oss/Cargo.toml +1 -1
  414. data/core/services/oss/src/backend.rs +40 -8
  415. data/core/services/oss/src/core.rs +192 -92
  416. data/core/services/oss/src/deleter.rs +16 -6
  417. data/core/services/oss/src/lister.rs +43 -27
  418. data/core/services/oss/src/reader.rs +6 -2
  419. data/core/services/oss/src/writer.rs +40 -22
  420. data/core/services/pcloud/Cargo.toml +1 -1
  421. data/core/services/pcloud/src/backend.rs +40 -6
  422. data/core/services/pcloud/src/core.rs +80 -66
  423. data/core/services/pcloud/src/deleter.rs +8 -1
  424. data/core/services/pcloud/src/lister.rs +4 -1
  425. data/core/services/pcloud/src/reader.rs +5 -2
  426. data/core/services/pcloud/src/writer.rs +6 -3
  427. data/core/services/persy/Cargo.toml +1 -1
  428. data/core/services/persy/src/backend.rs +6 -5
  429. data/core/services/persy/src/core.rs +60 -15
  430. data/core/services/persy/src/reader.rs +4 -1
  431. data/core/services/persy/src/writer.rs +6 -2
  432. data/core/services/postgresql/Cargo.toml +2 -2
  433. data/core/services/postgresql/src/backend.rs +7 -6
  434. data/core/services/postgresql/src/core.rs +1 -1
  435. data/core/services/postgresql/src/reader.rs +4 -1
  436. data/core/services/postgresql/src/writer.rs +6 -2
  437. data/core/services/redb/Cargo.toml +1 -1
  438. data/core/services/redb/src/backend.rs +6 -5
  439. data/core/services/redb/src/reader.rs +4 -1
  440. data/core/services/redb/src/writer.rs +6 -2
  441. data/core/services/redis/Cargo.toml +1 -1
  442. data/core/services/redis/src/backend.rs +18 -8
  443. data/core/services/redis/src/config.rs +13 -2
  444. data/core/services/redis/src/reader.rs +8 -4
  445. data/core/services/redis/src/writer.rs +6 -2
  446. data/core/services/rocksdb/Cargo.toml +1 -1
  447. data/core/services/rocksdb/src/backend.rs +6 -5
  448. data/core/services/rocksdb/src/lister.rs +6 -5
  449. data/core/services/rocksdb/src/reader.rs +4 -1
  450. data/core/services/rocksdb/src/writer.rs +6 -2
  451. data/core/services/s3/Cargo.toml +5 -4
  452. data/core/services/s3/src/backend.rs +995 -39
  453. data/core/services/s3/src/compatible_services.md +57 -0
  454. data/core/services/s3/src/config.rs +16 -0
  455. data/core/services/s3/src/copier.rs +190 -55
  456. data/core/services/s3/src/core.rs +299 -190
  457. data/core/services/s3/src/deleter.rs +26 -7
  458. data/core/services/s3/src/docs.md +57 -0
  459. data/core/services/s3/src/lister.rs +55 -36
  460. data/core/services/s3/src/reader.rs +22 -2
  461. data/core/services/s3/src/writer.rs +154 -32
  462. data/core/services/seafile/Cargo.toml +2 -2
  463. data/core/services/seafile/src/backend.rs +3 -3
  464. data/core/services/seafile/src/core.rs +105 -79
  465. data/core/services/seafile/src/lister.rs +11 -8
  466. data/core/services/seafile/src/reader.rs +5 -2
  467. data/core/services/seafile/src/writer.rs +6 -3
  468. data/core/services/sftp/Cargo.toml +1 -1
  469. data/core/services/sftp/src/backend.rs +19 -4
  470. data/core/services/sftp/src/core.rs +51 -46
  471. data/core/services/sftp/src/lister.rs +4 -4
  472. data/core/services/sftp/src/writer.rs +1 -1
  473. data/core/services/sled/Cargo.toml +1 -1
  474. data/core/services/sled/src/backend.rs +6 -5
  475. data/core/services/sled/src/core.rs +28 -6
  476. data/core/services/sled/src/lister.rs +6 -5
  477. data/core/services/sled/src/reader.rs +4 -1
  478. data/core/services/sled/src/writer.rs +6 -2
  479. data/core/services/sqlite/Cargo.toml +2 -2
  480. data/core/services/sqlite/src/backend.rs +12 -7
  481. data/core/services/sqlite/src/core.rs +3 -3
  482. data/core/services/sqlite/src/reader.rs +4 -1
  483. data/core/services/sqlite/src/writer.rs +7 -3
  484. data/core/services/surrealdb/Cargo.toml +2 -2
  485. data/core/services/surrealdb/src/backend.rs +7 -6
  486. data/core/services/surrealdb/src/core.rs +1 -1
  487. data/core/services/surrealdb/src/reader.rs +4 -1
  488. data/core/services/surrealdb/src/writer.rs +6 -2
  489. data/core/services/swift/Cargo.toml +1 -1
  490. data/core/services/swift/src/backend.rs +38 -8
  491. data/core/services/swift/src/core.rs +63 -67
  492. data/core/services/swift/src/deleter.rs +8 -2
  493. data/core/services/swift/src/lister.rs +11 -8
  494. data/core/services/swift/src/reader.rs +5 -2
  495. data/core/services/swift/src/writer.rs +17 -8
  496. data/core/services/tikv/Cargo.toml +2 -2
  497. data/core/services/tikv/src/backend.rs +7 -6
  498. data/core/services/tikv/src/core.rs +31 -14
  499. data/core/services/tikv/src/reader.rs +4 -1
  500. data/core/services/tikv/src/writer.rs +6 -2
  501. data/core/services/tos/Cargo.toml +2 -2
  502. data/core/services/tos/src/backend.rs +45 -21
  503. data/core/services/tos/src/copier.rs +44 -17
  504. data/core/services/tos/src/core.rs +147 -117
  505. data/core/services/tos/src/deleter.rs +9 -3
  506. data/core/services/tos/src/docs.md +1 -1
  507. data/core/services/tos/src/lister.rs +43 -28
  508. data/core/services/tos/src/reader.rs +5 -2
  509. data/core/services/tos/src/writer.rs +37 -16
  510. data/core/services/upyun/Cargo.toml +1 -1
  511. data/core/services/upyun/src/backend.rs +30 -7
  512. data/core/services/upyun/src/core.rs +122 -87
  513. data/core/services/upyun/src/deleter.rs +4 -1
  514. data/core/services/upyun/src/lister.rs +10 -10
  515. data/core/services/upyun/src/reader.rs +5 -2
  516. data/core/services/upyun/src/writer.rs +19 -7
  517. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  518. data/core/services/vercel-artifacts/src/backend.rs +6 -3
  519. data/core/services/vercel-artifacts/src/core.rs +32 -27
  520. data/core/services/vercel-artifacts/src/reader.rs +5 -2
  521. data/core/services/vercel-artifacts/src/writer.rs +6 -3
  522. data/core/services/vercel-blob/Cargo.toml +1 -1
  523. data/core/services/vercel-blob/src/backend.rs +23 -6
  524. data/core/services/vercel-blob/src/core.rs +92 -81
  525. data/core/services/vercel-blob/src/lister.rs +12 -3
  526. data/core/services/vercel-blob/src/reader.rs +5 -2
  527. data/core/services/vercel-blob/src/writer.rs +19 -7
  528. data/core/services/webdav/Cargo.toml +2 -2
  529. data/core/services/webdav/src/backend.rs +25 -5
  530. data/core/services/webdav/src/core.rs +116 -59
  531. data/core/services/webdav/src/deleter.rs +4 -1
  532. data/core/services/webdav/src/lister.rs +4 -1
  533. data/core/services/webdav/src/reader.rs +6 -2
  534. data/core/services/webdav/src/writer.rs +17 -7
  535. data/core/services/webhdfs/Cargo.toml +2 -2
  536. data/core/services/webhdfs/src/backend.rs +24 -11
  537. data/core/services/webhdfs/src/core.rs +99 -98
  538. data/core/services/webhdfs/src/deleter.rs +5 -2
  539. data/core/services/webhdfs/src/lister.rs +21 -9
  540. data/core/services/webhdfs/src/reader.rs +5 -2
  541. data/core/services/webhdfs/src/writer.rs +49 -14
  542. data/core/services/yandex-disk/Cargo.toml +1 -1
  543. data/core/services/yandex-disk/src/backend.rs +28 -6
  544. data/core/services/yandex-disk/src/core.rs +97 -79
  545. data/core/services/yandex-disk/src/deleter.rs +4 -1
  546. data/core/services/yandex-disk/src/lister.rs +25 -9
  547. data/core/services/yandex-disk/src/reader.rs +5 -2
  548. data/core/services/yandex-disk/src/writer.rs +6 -3
  549. data/core/src/lib.rs +5 -0
  550. data/core/testkit/Cargo.toml +4 -4
  551. data/core/testkit/src/utils.rs +53 -8
  552. data/core/tests/behavior/async_compose.rs +297 -0
  553. data/core/tests/behavior/async_copy.rs +162 -1
  554. data/core/tests/behavior/async_delete.rs +364 -3
  555. data/core/tests/behavior/async_read.rs +86 -0
  556. data/core/tests/behavior/async_restore.rs +162 -0
  557. data/core/tests/behavior/async_stat.rs +90 -0
  558. data/core/tests/behavior/async_write.rs +330 -2
  559. data/core/tests/behavior/main.rs +4 -0
  560. data/core/upgrade.md +114 -1
  561. data/src/capability.rs +0 -1
  562. data/src/io.rs +0 -1
  563. data/src/lister.rs +0 -1
  564. data/src/metadata.rs +0 -1
  565. data/src/operator_info.rs +0 -1
  566. metadata +44 -2
@@ -0,0 +1,705 @@
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ use std::collections::HashMap;
19
+ use std::sync::Arc;
20
+
21
+ use super::core::ErrorContext;
22
+ use super::core::GcsComposeSource;
23
+ use super::core::GcsCore;
24
+ use super::core::parse_error;
25
+ use super::deleter::GcsDeleter;
26
+ use opendal_core::raw::oio::Delete;
27
+ use opendal_core::raw::*;
28
+ use opendal_core::*;
29
+
30
+ const GCS_COMPOSE_MAX_SOURCES: usize = 32;
31
+ const COMPOSE_TOKEN_KEY: &str = "opendal-compose-token";
32
+
33
+ pub struct GcsComposer {
34
+ core: Arc<GcsCore>,
35
+ ctx: OperationContext,
36
+ to: String,
37
+ args: OpCompose,
38
+ buffered: Vec<GcsComposeSource>,
39
+ tasks: ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
40
+ input_count: usize,
41
+ intermediates: Vec<GcsComposeSource>,
42
+ metadata: Option<Metadata>,
43
+ errored: bool,
44
+ }
45
+
46
+ impl GcsComposer {
47
+ pub fn new(core: Arc<GcsCore>, ctx: OperationContext, to: &str, args: OpCompose) -> Self {
48
+ let tasks = new_tasks(ctx.executor().clone(), args.concurrent());
49
+ Self {
50
+ core,
51
+ ctx,
52
+ to: to.to_string(),
53
+ args,
54
+ buffered: Vec::with_capacity(GCS_COMPOSE_MAX_SOURCES + 1),
55
+ tasks,
56
+ input_count: 0,
57
+ intermediates: Vec::new(),
58
+ metadata: None,
59
+ errored: false,
60
+ }
61
+ }
62
+
63
+ fn new_intermediate_task(&self, sources: Vec<GcsComposeSource>) -> Result<GcsComposeTask> {
64
+ let token = uuid::Uuid::new_v4().to_string();
65
+ let to = format!("__opendal/compose/{token}");
66
+ let mut metadata = HashMap::new();
67
+ metadata.insert(COMPOSE_TOKEN_KEY.to_string(), token.clone());
68
+ let args = OpCompose::from_options(
69
+ &self.core.capability,
70
+ options::ComposeOptions {
71
+ if_not_exists: true,
72
+ content_type: Some("application/octet-stream".to_string()),
73
+ user_metadata: Some(metadata),
74
+ ..Default::default()
75
+ },
76
+ )?;
77
+
78
+ Ok(GcsComposeTask {
79
+ core: self.core.clone(),
80
+ ctx: self.ctx.clone(),
81
+ sources,
82
+ to,
83
+ args,
84
+ token: Some(token),
85
+ })
86
+ }
87
+
88
+ async fn collect_tasks(
89
+ tasks: &mut ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
90
+ ) -> Result<Vec<GcsComposeSource>> {
91
+ let mut outputs = Vec::new();
92
+ while let Some(output) = tasks.next().await {
93
+ match output {
94
+ Ok(output) => outputs.push(output),
95
+ Err(err) if err.is_temporary() => continue,
96
+ Err(err) => return Err(err),
97
+ }
98
+ }
99
+ Ok(outputs)
100
+ }
101
+
102
+ async fn execute_task(
103
+ tasks: &mut ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
104
+ task: GcsComposeTask,
105
+ ) -> Result<()> {
106
+ loop {
107
+ match tasks.execute(task.clone()).await {
108
+ Ok(()) => return Ok(()),
109
+ Err(err) if err.is_temporary() => continue,
110
+ Err(err) => return Err(err),
111
+ }
112
+ }
113
+ }
114
+
115
+ async fn reduce_level(
116
+ &mut self,
117
+ sources: Vec<GcsComposeSource>,
118
+ ) -> Result<Vec<GcsComposeSource>> {
119
+ let mut tasks = new_tasks(self.ctx.executor().clone(), self.args.concurrent());
120
+ let mut carried = None;
121
+
122
+ for chunk in sources.chunks(GCS_COMPOSE_MAX_SOURCES) {
123
+ if chunk.len() == 1 {
124
+ carried = Some(chunk[0].clone());
125
+ continue;
126
+ }
127
+ Self::execute_task(&mut tasks, self.new_intermediate_task(chunk.to_vec())?).await?;
128
+ }
129
+
130
+ let mut outputs = Self::collect_tasks(&mut tasks).await?;
131
+ self.intermediates.extend(outputs.iter().cloned());
132
+ if let Some(source) = carried {
133
+ outputs.push(source);
134
+ }
135
+ Ok(outputs)
136
+ }
137
+
138
+ async fn compose_final(&self, sources: &[GcsComposeSource]) -> Result<Metadata> {
139
+ let resp = self
140
+ .core
141
+ .gcs_compose_object(&self.ctx, sources, &self.to, &self.args)
142
+ .await?;
143
+ if !resp.status().is_success() {
144
+ return Err(parse_error(
145
+ ErrorContext::new(ServiceOperation("ComposeObject"))
146
+ .with_caller_condition(self.args.is_conditional()),
147
+ resp,
148
+ ));
149
+ }
150
+ GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())
151
+ }
152
+
153
+ async fn cleanup_intermediates(&self) -> Result<()> {
154
+ let mut deleter = oio::BatchDeleter::new(
155
+ GcsDeleter::new(self.core.clone(), self.ctx.clone()),
156
+ self.core.capability.delete_max_size,
157
+ );
158
+ for source in &self.intermediates {
159
+ let Some(version) = source.version.as_deref() else {
160
+ continue;
161
+ };
162
+ let args = OpDelete::from_options(
163
+ &self.core.capability,
164
+ options::DeleteOptions {
165
+ if_version_match: Some(version.to_string()),
166
+ ..Default::default()
167
+ },
168
+ )?;
169
+ deleter.delete(&source.path, args).await?;
170
+ }
171
+ deleter.close().await
172
+ }
173
+
174
+ async fn close_inner(&mut self) -> Result<Metadata> {
175
+ if self.input_count == 0 {
176
+ return Err(Error::new(
177
+ ErrorKind::ConfigInvalid,
178
+ "compose requires at least one source object",
179
+ ));
180
+ }
181
+
182
+ let mut sources = Self::collect_tasks(&mut self.tasks).await?;
183
+ self.intermediates.extend(sources.iter().cloned());
184
+ sources.append(&mut self.buffered);
185
+
186
+ while sources.len() > GCS_COMPOSE_MAX_SOURCES {
187
+ sources = self.reduce_level(sources).await?;
188
+ }
189
+
190
+ let metadata = self.compose_final(&sources).await?;
191
+ if let Err(err) = self.cleanup_intermediates().await {
192
+ log::warn!("failed to clean GCS compose intermediate objects: {err}");
193
+ }
194
+ Ok(metadata)
195
+ }
196
+ }
197
+
198
+ impl oio::Compose for GcsComposer {
199
+ async fn compose(&mut self, path: &str, args: OpRead) -> Result<()> {
200
+ if self.errored {
201
+ return Err(Error::new(
202
+ ErrorKind::Unexpected,
203
+ "GCS composer has already failed",
204
+ ));
205
+ }
206
+ if self.metadata.is_some() {
207
+ return Err(Error::new(
208
+ ErrorKind::Unexpected,
209
+ "GCS composer is already closed",
210
+ ));
211
+ }
212
+ if path == self.to {
213
+ return Err(Error::new(
214
+ ErrorKind::IsSameFile,
215
+ "source and destination paths are same",
216
+ ));
217
+ }
218
+ if args.if_match().is_some()
219
+ || args.if_none_match().is_some()
220
+ || args.if_version_not_match().is_some()
221
+ {
222
+ return Err(Error::new(
223
+ ErrorKind::Unsupported,
224
+ "GCS compose supports source generation conditions only",
225
+ ));
226
+ }
227
+
228
+ self.buffered.push(GcsComposeSource {
229
+ path: path.to_string(),
230
+ version: args.version().map(str::to_string),
231
+ if_version_match: args.if_version_match().map(str::to_string),
232
+ });
233
+ self.input_count += 1;
234
+
235
+ if self.buffered.len() > GCS_COMPOSE_MAX_SOURCES {
236
+ let remaining = self.buffered.split_off(GCS_COMPOSE_MAX_SOURCES);
237
+ let sources = std::mem::replace(&mut self.buffered, remaining);
238
+ let task = match self.new_intermediate_task(sources) {
239
+ Ok(task) => task,
240
+ Err(err) => {
241
+ self.errored = true;
242
+ return Err(err);
243
+ }
244
+ };
245
+ if let Err(err) = Self::execute_task(&mut self.tasks, task).await {
246
+ self.errored = true;
247
+ return Err(err);
248
+ }
249
+ }
250
+ Ok(())
251
+ }
252
+
253
+ async fn close(&mut self) -> Result<Metadata> {
254
+ if self.errored {
255
+ return Err(Error::new(
256
+ ErrorKind::Unexpected,
257
+ "GCS composer has already failed and can't be closed",
258
+ ));
259
+ }
260
+ if let Some(metadata) = self.metadata.clone() {
261
+ return Ok(metadata);
262
+ }
263
+
264
+ match self.close_inner().await {
265
+ Ok(metadata) => {
266
+ self.metadata = Some(metadata.clone());
267
+ Ok(metadata)
268
+ }
269
+ Err(err) => {
270
+ self.errored = true;
271
+ Err(err)
272
+ }
273
+ }
274
+ }
275
+ }
276
+
277
+ #[derive(Clone)]
278
+ struct GcsComposeTask {
279
+ core: Arc<GcsCore>,
280
+ ctx: OperationContext,
281
+ sources: Vec<GcsComposeSource>,
282
+ to: String,
283
+ args: OpCompose,
284
+ token: Option<String>,
285
+ }
286
+
287
+ impl GcsComposeTask {
288
+ async fn run(&self) -> Result<GcsComposeSource> {
289
+ let resp = self
290
+ .core
291
+ .gcs_compose_object(&self.ctx, &self.sources, &self.to, &self.args)
292
+ .await?;
293
+ if !resp.status().is_success() {
294
+ let err = parse_error(
295
+ ErrorContext::new(ServiceOperation("ComposeObject"))
296
+ .with_caller_condition(self.args.is_conditional())
297
+ .with_internal_condition(self.token.is_some()),
298
+ resp,
299
+ );
300
+ let recoverable_condition = if self.token.is_some() {
301
+ ErrorKind::Conflict
302
+ } else {
303
+ ErrorKind::ConditionNotMatch
304
+ };
305
+ if err.kind() == recoverable_condition
306
+ && let Some(source) = self.recover_completed_intermediate().await?
307
+ {
308
+ return Ok(source);
309
+ }
310
+ return Err(err);
311
+ }
312
+
313
+ let metadata = GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())?;
314
+ self.output_from_metadata(metadata)
315
+ }
316
+
317
+ async fn recover_completed_intermediate(&self) -> Result<Option<GcsComposeSource>> {
318
+ let Some(token) = self.token.as_deref() else {
319
+ return Ok(None);
320
+ };
321
+ let resp = self
322
+ .core
323
+ .gcs_get_object_metadata(&self.ctx, &self.to, &OpStat::new())
324
+ .await?;
325
+ if !resp.status().is_success() {
326
+ return Ok(None);
327
+ }
328
+ let metadata = GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())?;
329
+ if metadata
330
+ .user_metadata()
331
+ .and_then(|values| values.get(COMPOSE_TOKEN_KEY))
332
+ .is_none_or(|value| value != token)
333
+ {
334
+ return Ok(None);
335
+ }
336
+ self.output_from_metadata(metadata).map(Some)
337
+ }
338
+
339
+ fn output_from_metadata(&self, metadata: Metadata) -> Result<GcsComposeSource> {
340
+ let version = metadata.version().ok_or_else(|| {
341
+ Error::new(
342
+ ErrorKind::Unexpected,
343
+ "GCS compose response is missing object generation",
344
+ )
345
+ })?;
346
+ Ok(GcsComposeSource {
347
+ path: self.to.clone(),
348
+ version: Some(version.to_string()),
349
+ if_version_match: None,
350
+ })
351
+ }
352
+ }
353
+
354
+ fn new_tasks(
355
+ executor: Executor,
356
+ concurrent: usize,
357
+ ) -> ConcurrentTasks<GcsComposeTask, GcsComposeSource> {
358
+ ConcurrentTasks::new(executor, concurrent.max(1), concurrent.max(1), |task| {
359
+ Box::pin(async move {
360
+ let result = task.run().await;
361
+ (task, result)
362
+ })
363
+ })
364
+ }
365
+
366
+ #[cfg(test)]
367
+ mod tests {
368
+ use std::sync::Mutex;
369
+ use std::sync::atomic::AtomicUsize;
370
+ use std::sync::atomic::Ordering;
371
+ use std::time::Duration;
372
+
373
+ use futures::stream;
374
+ use http::Method;
375
+ use http::Response;
376
+ use reqsign_core::Context;
377
+ use reqsign_core::ProvideCredentialChain;
378
+ use reqsign_core::Signer;
379
+ use reqsign_google::RequestSigner;
380
+ use reqsign_google::TokenCredentialProvider;
381
+ use tokio::sync::Notify;
382
+
383
+ use super::*;
384
+ use opendal_core::raw::oio::Compose;
385
+
386
+ struct ComposeTransport {
387
+ started: AtomicUsize,
388
+ active: AtomicUsize,
389
+ max_active: AtomicUsize,
390
+ generation: AtomicUsize,
391
+ notify: Notify,
392
+ requests: Mutex<Vec<serde_json::Value>>,
393
+ }
394
+
395
+ impl ComposeTransport {
396
+ fn new() -> Self {
397
+ Self {
398
+ started: AtomicUsize::new(0),
399
+ active: AtomicUsize::new(0),
400
+ max_active: AtomicUsize::new(0),
401
+ generation: AtomicUsize::new(1),
402
+ notify: Notify::new(),
403
+ requests: Mutex::new(Vec::new()),
404
+ }
405
+ }
406
+
407
+ fn response(status: http::StatusCode, body: Buffer) -> Result<Response<HttpBody>> {
408
+ let size = body.len() as u64;
409
+ Response::builder()
410
+ .status(status)
411
+ .body(HttpBody::new(stream::iter([Ok(body)]), Some(size)))
412
+ .map_err(new_request_build_error)
413
+ }
414
+ }
415
+
416
+ #[derive(Clone)]
417
+ struct SharedComposeTransport(Arc<ComposeTransport>);
418
+
419
+ struct TokioTestExecutor;
420
+
421
+ impl Execute for TokioTestExecutor {
422
+ fn execute(&self, future: BoxedStaticFuture<()>) {
423
+ tokio::spawn(future);
424
+ }
425
+ }
426
+
427
+ impl HttpTransport for SharedComposeTransport {
428
+ async fn fetch(&self, req: http::Request<Buffer>) -> Result<Response<HttpBody>> {
429
+ if req.method() == Method::DELETE {
430
+ return ComposeTransport::response(http::StatusCode::NO_CONTENT, Buffer::new());
431
+ }
432
+
433
+ let value: serde_json::Value = serde_json::from_slice(&req.body().to_bytes())
434
+ .expect("compose request body must be valid JSON");
435
+ self.0.requests.lock().unwrap().push(value.clone());
436
+
437
+ if req.uri().path().contains("__opendal%2Fcompose%2F") {
438
+ let active = self.0.active.fetch_add(1, Ordering::SeqCst) + 1;
439
+ self.0.max_active.fetch_max(active, Ordering::SeqCst);
440
+ self.0.started.fetch_add(1, Ordering::SeqCst);
441
+ self.0.notify.notify_waiters();
442
+
443
+ loop {
444
+ let notified = self.0.notify.notified();
445
+ if self.0.started.load(Ordering::SeqCst) >= 2 {
446
+ break;
447
+ }
448
+ notified.await;
449
+ }
450
+ self.0.active.fetch_sub(1, Ordering::SeqCst);
451
+ }
452
+
453
+ let generation = self.0.generation.fetch_add(1, Ordering::SeqCst);
454
+ let metadata = value
455
+ .get("destination")
456
+ .and_then(|destination| destination.get("metadata"))
457
+ .cloned()
458
+ .unwrap_or_else(|| serde_json::json!({}));
459
+ let body = Buffer::from(
460
+ serde_json::to_vec(&serde_json::json!({
461
+ "size": "1",
462
+ "generation": generation.to_string(),
463
+ "metadata": metadata,
464
+ }))
465
+ .expect("response JSON must serialize"),
466
+ );
467
+ ComposeTransport::response(http::StatusCode::OK, body)
468
+ }
469
+ }
470
+
471
+ struct RecoveryTransport {
472
+ intermediate_posts: AtomicUsize,
473
+ metadata_reads: AtomicUsize,
474
+ token: Mutex<Option<String>>,
475
+ }
476
+
477
+ #[derive(Clone)]
478
+ struct SharedRecoveryTransport(Arc<RecoveryTransport>);
479
+
480
+ impl HttpTransport for SharedRecoveryTransport {
481
+ async fn fetch(&self, req: http::Request<Buffer>) -> Result<Response<HttpBody>> {
482
+ if req.method() == Method::DELETE {
483
+ return ComposeTransport::response(http::StatusCode::NO_CONTENT, Buffer::new());
484
+ }
485
+
486
+ if req.method() == Method::GET {
487
+ self.0.metadata_reads.fetch_add(1, Ordering::SeqCst);
488
+ let token = self.0.token.lock().unwrap().clone().unwrap();
489
+ let body = Buffer::from(
490
+ serde_json::to_vec(&serde_json::json!({
491
+ "size": "32",
492
+ "generation": "7",
493
+ "metadata": {(COMPOSE_TOKEN_KEY): token},
494
+ }))
495
+ .unwrap(),
496
+ );
497
+ return ComposeTransport::response(http::StatusCode::OK, body);
498
+ }
499
+
500
+ if req.uri().path().contains("__opendal%2Fcompose%2F") {
501
+ let value: serde_json::Value = serde_json::from_slice(&req.body().to_bytes())
502
+ .expect("compose request body must be valid JSON");
503
+ let token = value["destination"]["metadata"][COMPOSE_TOKEN_KEY]
504
+ .as_str()
505
+ .unwrap()
506
+ .to_string();
507
+ *self.0.token.lock().unwrap() = Some(token);
508
+
509
+ if self.0.intermediate_posts.fetch_add(1, Ordering::SeqCst) == 0 {
510
+ return Err(Error::new(
511
+ ErrorKind::Unexpected,
512
+ "compose response was lost after the object was created",
513
+ )
514
+ .set_temporary());
515
+ }
516
+ return ComposeTransport::response(
517
+ http::StatusCode::PRECONDITION_FAILED,
518
+ Buffer::new(),
519
+ );
520
+ }
521
+
522
+ let body = Buffer::from(
523
+ serde_json::to_vec(&serde_json::json!({
524
+ "size": "33",
525
+ "generation": "8",
526
+ }))
527
+ .unwrap(),
528
+ );
529
+ ComposeTransport::response(http::StatusCode::OK, body)
530
+ }
531
+ }
532
+
533
+ fn test_core() -> Arc<GcsCore> {
534
+ let sign_ctx = Context::new();
535
+ Arc::new(GcsCore {
536
+ info: ServiceInfo::new("gcs", "/", "test-bucket"),
537
+ capability: Capability {
538
+ delete_max_size: Some(1),
539
+ ..Default::default()
540
+ },
541
+ endpoint: "https://storage.googleapis.com".to_string(),
542
+ bucket: "test-bucket".to_string(),
543
+ root: "/".to_string(),
544
+ signer: Signer::new(
545
+ sign_ctx.clone(),
546
+ ProvideCredentialChain::new().push(TokenCredentialProvider::new("test-token")),
547
+ RequestSigner::new("storage"),
548
+ ),
549
+ sign_ctx,
550
+ predefined_acl: None,
551
+ default_storage_class: None,
552
+ skip_signature: true,
553
+ })
554
+ }
555
+
556
+ #[tokio::test]
557
+ async fn test_compose_builds_concurrent_ordered_tree() {
558
+ let transport = Arc::new(ComposeTransport::new());
559
+ let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
560
+ SharedComposeTransport(transport.clone()),
561
+ ));
562
+ let ctx = ctx.with_executor(Executor::with(TokioTestExecutor));
563
+ let mut composer = GcsComposer::new(
564
+ test_core(),
565
+ ctx,
566
+ "target",
567
+ OpCompose::from_options(
568
+ &Capability::default(),
569
+ options::ComposeOptions {
570
+ concurrent: 2,
571
+ ..Default::default()
572
+ },
573
+ )
574
+ .unwrap(),
575
+ );
576
+
577
+ for index in 0..65 {
578
+ composer
579
+ .compose(&format!("source-{index:02}"), OpRead::new())
580
+ .await
581
+ .expect("source must be accepted");
582
+ }
583
+ tokio::time::timeout(Duration::from_secs(5), composer.close())
584
+ .await
585
+ .expect("compose tasks must make concurrent progress")
586
+ .expect("composition must succeed");
587
+
588
+ assert_eq!(transport.max_active.load(Ordering::SeqCst), 2);
589
+ let requests = transport.requests.lock().unwrap();
590
+ assert_eq!(requests.len(), 3);
591
+
592
+ let source_lists: Vec<&Vec<serde_json::Value>> = requests
593
+ .iter()
594
+ .map(|request| request["sourceObjects"].as_array().unwrap())
595
+ .collect();
596
+ assert_eq!(
597
+ source_lists
598
+ .iter()
599
+ .filter(|sources| sources.len() == 32)
600
+ .count(),
601
+ 2
602
+ );
603
+ let final_sources = source_lists
604
+ .iter()
605
+ .find(|sources| sources.len() == 3)
606
+ .expect("final composition must contain the reduced source sequence");
607
+ assert!(
608
+ final_sources[0]["name"]
609
+ .as_str()
610
+ .unwrap()
611
+ .starts_with("__opendal/compose/")
612
+ );
613
+ assert!(
614
+ final_sources[1]["name"]
615
+ .as_str()
616
+ .unwrap()
617
+ .starts_with("__opendal/compose/")
618
+ );
619
+ assert_eq!(final_sources[2]["name"], "source-64");
620
+ }
621
+
622
+ #[tokio::test]
623
+ async fn test_compose_recovers_ambiguous_intermediate_completion() {
624
+ let transport = Arc::new(RecoveryTransport {
625
+ intermediate_posts: AtomicUsize::new(0),
626
+ metadata_reads: AtomicUsize::new(0),
627
+ token: Mutex::new(None),
628
+ });
629
+ let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
630
+ SharedRecoveryTransport(transport.clone()),
631
+ ));
632
+ let mut composer = GcsComposer::new(test_core(), ctx, "target", OpCompose::new());
633
+
634
+ for index in 0..33 {
635
+ composer
636
+ .compose(&format!("source-{index:02}"), OpRead::new())
637
+ .await
638
+ .expect("source must be accepted");
639
+ }
640
+ composer.close().await.expect("composition must recover");
641
+
642
+ assert_eq!(transport.intermediate_posts.load(Ordering::SeqCst), 2);
643
+ assert_eq!(transport.metadata_reads.load(Ordering::SeqCst), 1);
644
+ }
645
+
646
+ #[tokio::test]
647
+ async fn test_compose_hides_request_limit_across_tree_levels() {
648
+ let transport = Arc::new(ComposeTransport::new());
649
+ let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
650
+ SharedComposeTransport(transport.clone()),
651
+ ));
652
+ let ctx = ctx.with_executor(Executor::with(TokioTestExecutor));
653
+ let mut composer = GcsComposer::new(
654
+ test_core(),
655
+ ctx,
656
+ "target",
657
+ OpCompose::from_options(
658
+ &Capability::default(),
659
+ options::ComposeOptions {
660
+ concurrent: 4,
661
+ ..Default::default()
662
+ },
663
+ )
664
+ .unwrap(),
665
+ );
666
+
667
+ for index in 0..1025 {
668
+ composer
669
+ .compose(&format!("source-{index:04}"), OpRead::new())
670
+ .await
671
+ .expect("source must be accepted");
672
+ }
673
+ composer.close().await.expect("composition must succeed");
674
+
675
+ let requests = transport.requests.lock().unwrap();
676
+ assert_eq!(requests.len(), 34);
677
+ let source_lists: Vec<&Vec<serde_json::Value>> = requests
678
+ .iter()
679
+ .map(|request| request["sourceObjects"].as_array().unwrap())
680
+ .collect();
681
+ assert!(
682
+ source_lists
683
+ .iter()
684
+ .all(|sources| !sources.is_empty() && sources.len() <= GCS_COMPOSE_MAX_SOURCES)
685
+ );
686
+ assert_eq!(
687
+ source_lists
688
+ .iter()
689
+ .filter(|sources| sources.len() == 32)
690
+ .count(),
691
+ 33
692
+ );
693
+ let final_sources = source_lists
694
+ .iter()
695
+ .find(|sources| sources.len() == 2)
696
+ .expect("final composition must contain the reduced source sequence");
697
+ assert!(
698
+ final_sources[0]["name"]
699
+ .as_str()
700
+ .unwrap()
701
+ .starts_with("__opendal/compose/")
702
+ );
703
+ assert_eq!(final_sources[1]["name"], "source-1024");
704
+ }
705
+ }