opendal 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +31 -36
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +25 -35
  5. data/README.md +1 -1
  6. data/core/CHANGELOG.md +4 -0
  7. data/core/Cargo.lock +128 -187
  8. data/core/Cargo.toml +95 -95
  9. data/core/DEPENDENCIES.rust.tsv +259 -289
  10. data/core/LICENSE +16 -0
  11. data/core/benches/vs_fs/Cargo.toml +1 -1
  12. data/core/benches/vs_s3/Cargo.toml +1 -1
  13. data/core/core/CHANGELOG.md +4 -0
  14. data/core/core/Cargo.toml +1 -1
  15. data/core/core/LICENSE +16 -0
  16. data/core/core/src/docs/upgrade.md +8 -0
  17. data/core/core/src/lib.rs +1 -3
  18. data/core/core/src/raw/futures_util.rs +26 -21
  19. data/core/core/src/raw/http_util/error.rs +9 -0
  20. data/core/core/src/raw/ops.rs +1 -0
  21. data/core/core/src/types/http_transport/mod.rs +30 -2
  22. data/core/core/src/types/http_transport/uri.rs +166 -0
  23. data/core/core/src/types/metadata.rs +1 -0
  24. data/core/core/src/types/mod.rs +2 -0
  25. data/core/http-transports/reqwest/Cargo.toml +1 -1
  26. data/core/http-transports/reqwest/LICENSE +16 -0
  27. data/core/http-transports/reqwest/src/lib.rs +46 -4
  28. data/core/layers/async-backtrace/Cargo.toml +2 -2
  29. data/core/layers/async-backtrace/LICENSE +16 -0
  30. data/core/layers/await-tree/Cargo.toml +2 -2
  31. data/core/layers/await-tree/LICENSE +16 -0
  32. data/core/layers/capability-check/Cargo.toml +2 -2
  33. data/core/layers/capability-check/LICENSE +16 -0
  34. data/core/layers/chaos/Cargo.toml +2 -2
  35. data/core/layers/chaos/LICENSE +16 -0
  36. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  37. data/core/layers/concurrent-limit/LICENSE +16 -0
  38. data/core/layers/dtrace/Cargo.toml +2 -2
  39. data/core/layers/dtrace/LICENSE +16 -0
  40. data/core/layers/fastmetrics/Cargo.toml +3 -3
  41. data/core/layers/fastmetrics/LICENSE +16 -0
  42. data/core/layers/fastrace/Cargo.toml +2 -2
  43. data/core/layers/fastrace/LICENSE +16 -0
  44. data/core/layers/foyer/Cargo.toml +2 -2
  45. data/core/layers/foyer/LICENSE +16 -0
  46. data/core/layers/hotpath/Cargo.toml +2 -2
  47. data/core/layers/hotpath/LICENSE +16 -0
  48. data/core/layers/immutable-index/Cargo.toml +2 -2
  49. data/core/layers/immutable-index/LICENSE +16 -0
  50. data/core/layers/logging/Cargo.toml +2 -2
  51. data/core/layers/logging/LICENSE +16 -0
  52. data/core/layers/metrics/Cargo.toml +3 -3
  53. data/core/layers/metrics/LICENSE +16 -0
  54. data/core/layers/mime-guess/Cargo.toml +2 -2
  55. data/core/layers/mime-guess/LICENSE +16 -0
  56. data/core/layers/observe-metrics-common/Cargo.toml +1 -1
  57. data/core/layers/observe-metrics-common/LICENSE +16 -0
  58. data/core/layers/otelmetrics/Cargo.toml +3 -3
  59. data/core/layers/otelmetrics/LICENSE +16 -0
  60. data/core/layers/oteltrace/Cargo.toml +2 -2
  61. data/core/layers/oteltrace/LICENSE +16 -0
  62. data/core/layers/prometheus/Cargo.toml +3 -3
  63. data/core/layers/prometheus/LICENSE +16 -0
  64. data/core/layers/prometheus-client/Cargo.toml +3 -3
  65. data/core/layers/prometheus-client/LICENSE +16 -0
  66. data/core/layers/retry/Cargo.toml +4 -4
  67. data/core/layers/retry/LICENSE +16 -0
  68. data/core/layers/route/Cargo.toml +3 -3
  69. data/core/layers/route/LICENSE +16 -0
  70. data/core/layers/tail-cut/Cargo.toml +2 -2
  71. data/core/layers/tail-cut/LICENSE +16 -0
  72. data/core/layers/throttle/Cargo.toml +2 -2
  73. data/core/layers/throttle/LICENSE +16 -0
  74. data/core/layers/timeout/Cargo.toml +3 -3
  75. data/core/layers/timeout/LICENSE +16 -0
  76. data/core/layers/tracing/Cargo.toml +2 -2
  77. data/core/layers/tracing/LICENSE +16 -0
  78. data/core/services/aliyun-drive/Cargo.toml +2 -2
  79. data/core/services/aliyun-drive/LICENSE +16 -0
  80. data/core/services/alluxio/Cargo.toml +1 -1
  81. data/core/services/alluxio/LICENSE +16 -0
  82. data/core/services/azblob/Cargo.toml +2 -2
  83. data/core/services/azblob/LICENSE +16 -0
  84. data/core/services/azblob/src/backend.rs +21 -7
  85. data/core/services/azblob/src/config.rs +4 -1
  86. data/core/services/azdls/Cargo.toml +3 -3
  87. data/core/services/azdls/LICENSE +16 -0
  88. data/core/services/azfile/Cargo.toml +2 -2
  89. data/core/services/azfile/LICENSE +16 -0
  90. data/core/services/azure-common/Cargo.toml +1 -1
  91. data/core/services/azure-common/LICENSE +16 -0
  92. data/core/services/b2/Cargo.toml +2 -2
  93. data/core/services/b2/LICENSE +16 -0
  94. data/core/services/cacache/Cargo.toml +1 -1
  95. data/core/services/cacache/LICENSE +16 -0
  96. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  97. data/core/services/cloudflare-kv/LICENSE +16 -0
  98. data/core/services/compfs/Cargo.toml +1 -1
  99. data/core/services/compfs/LICENSE +16 -0
  100. data/core/services/compfs/src/core.rs +27 -11
  101. data/core/services/cos/Cargo.toml +1 -1
  102. data/core/services/cos/LICENSE +16 -0
  103. data/core/services/d1/Cargo.toml +1 -1
  104. data/core/services/d1/LICENSE +16 -0
  105. data/core/services/dashmap/Cargo.toml +1 -1
  106. data/core/services/dashmap/LICENSE +16 -0
  107. data/core/services/dbfs/Cargo.toml +1 -1
  108. data/core/services/dbfs/LICENSE +16 -0
  109. data/core/services/dropbox/Cargo.toml +2 -2
  110. data/core/services/dropbox/LICENSE +16 -0
  111. data/core/services/etcd/Cargo.toml +2 -2
  112. data/core/services/etcd/LICENSE +16 -0
  113. data/core/services/etcd/src/core.rs +3 -3
  114. data/core/services/foundationdb/Cargo.toml +1 -1
  115. data/core/services/foundationdb/LICENSE +16 -0
  116. data/core/services/foyer/Cargo.toml +3 -3
  117. data/core/services/foyer/LICENSE +16 -0
  118. data/core/services/fs/Cargo.toml +1 -1
  119. data/core/services/fs/LICENSE +16 -0
  120. data/core/services/fs/src/core.rs +69 -10
  121. data/core/services/ftp/Cargo.toml +2 -2
  122. data/core/services/ftp/LICENSE +16 -0
  123. data/core/services/ftp/src/core.rs +1 -1
  124. data/core/services/ftp/src/reader.rs +1 -1
  125. data/core/services/ftp/src/writer.rs +1 -1
  126. data/core/services/gcs/Cargo.toml +3 -2
  127. data/core/services/gcs/LICENSE +16 -0
  128. data/core/services/gcs/src/composer.rs +11 -21
  129. data/core/services/gcs-grpc/Cargo.toml +1 -1
  130. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
  131. data/core/services/gdrive/Cargo.toml +3 -2
  132. data/core/services/gdrive/LICENSE +16 -0
  133. data/core/services/gdrive/src/docs.md +2 -0
  134. data/core/services/gdrive/src/path_index.rs +109 -18
  135. data/core/services/ghac/Cargo.toml +2 -2
  136. data/core/services/ghac/LICENSE +16 -0
  137. data/core/services/github/Cargo.toml +1 -1
  138. data/core/services/github/LICENSE +16 -0
  139. data/core/services/goosefs/Cargo.toml +13 -4
  140. data/core/services/goosefs/LICENSE +16 -0
  141. data/core/services/goosefs/README.md +11 -0
  142. data/core/services/goosefs/src/backend.rs +3 -5
  143. data/core/services/goosefs/src/core.rs +1 -1
  144. data/core/services/goosefs/src/docs.md +44 -3
  145. data/core/services/goosefs/src/reader.rs +22 -89
  146. data/core/services/gridfs/Cargo.toml +2 -2
  147. data/core/services/gridfs/LICENSE +16 -0
  148. data/core/services/hdfs/Cargo.toml +1 -1
  149. data/core/services/hdfs/LICENSE +16 -0
  150. data/core/services/hdfs-native/Cargo.toml +1 -1
  151. data/core/services/hdfs-native/LICENSE +16 -0
  152. data/core/services/hf/Cargo.toml +3 -3
  153. data/core/services/hf/LICENSE +16 -0
  154. data/core/services/hf/src/backend.rs +77 -9
  155. data/core/services/hf/src/config.rs +58 -0
  156. data/core/services/hf/src/core.rs +695 -318
  157. data/core/services/hf/src/docs.md +29 -0
  158. data/core/services/hf/src/lister.rs +3 -3
  159. data/core/services/hf/src/reader.rs +128 -143
  160. data/core/services/http/Cargo.toml +3 -3
  161. data/core/services/http/LICENSE +16 -0
  162. data/core/services/ipfs/Cargo.toml +1 -1
  163. data/core/services/ipfs/LICENSE +16 -0
  164. data/core/services/ipmfs/Cargo.toml +1 -1
  165. data/core/services/ipmfs/LICENSE +16 -0
  166. data/core/services/koofr/Cargo.toml +2 -2
  167. data/core/services/koofr/LICENSE +16 -0
  168. data/core/services/lakefs/Cargo.toml +1 -1
  169. data/core/services/lakefs/LICENSE +16 -0
  170. data/core/services/memcached/Cargo.toml +2 -2
  171. data/core/services/memcached/LICENSE +16 -0
  172. data/core/services/memcached/src/core.rs +3 -3
  173. data/core/services/mini_moka/Cargo.toml +1 -1
  174. data/core/services/mini_moka/LICENSE +16 -0
  175. data/core/services/moka/Cargo.toml +1 -1
  176. data/core/services/moka/LICENSE +16 -0
  177. data/core/services/mongodb/Cargo.toml +2 -2
  178. data/core/services/mongodb/LICENSE +16 -0
  179. data/core/services/monoiofs/Cargo.toml +2 -2
  180. data/core/services/monoiofs/LICENSE +16 -0
  181. data/core/services/monoiofs/src/core.rs +27 -11
  182. data/core/services/mysql/Cargo.toml +2 -2
  183. data/core/services/mysql/LICENSE +16 -0
  184. data/core/services/obs/Cargo.toml +1 -1
  185. data/core/services/obs/LICENSE +16 -0
  186. data/core/services/onedrive/Cargo.toml +2 -2
  187. data/core/services/onedrive/LICENSE +16 -0
  188. data/core/services/opfs/Cargo.toml +1 -1
  189. data/core/services/opfs/LICENSE +16 -0
  190. data/core/services/oss/Cargo.toml +1 -1
  191. data/core/services/oss/LICENSE +16 -0
  192. data/core/services/pcloud/Cargo.toml +1 -1
  193. data/core/services/pcloud/LICENSE +16 -0
  194. data/core/services/persy/Cargo.toml +1 -1
  195. data/core/services/persy/LICENSE +16 -0
  196. data/core/services/postgresql/Cargo.toml +2 -2
  197. data/core/services/postgresql/LICENSE +16 -0
  198. data/core/services/redb/Cargo.toml +1 -1
  199. data/core/services/redb/LICENSE +16 -0
  200. data/core/services/redis/Cargo.toml +2 -2
  201. data/core/services/redis/LICENSE +16 -0
  202. data/core/services/redis/src/core.rs +1 -1
  203. data/core/services/rocksdb/Cargo.toml +1 -1
  204. data/core/services/rocksdb/LICENSE +16 -0
  205. data/core/services/s3/Cargo.toml +1 -1
  206. data/core/services/s3/LICENSE +16 -0
  207. data/core/services/seafile/Cargo.toml +4 -2
  208. data/core/services/seafile/LICENSE +16 -0
  209. data/core/services/seafile/src/backend.rs +2 -3
  210. data/core/services/seafile/src/core.rs +189 -86
  211. data/core/services/seafile/src/docs.md +2 -0
  212. data/core/services/sftp/Cargo.toml +2 -2
  213. data/core/services/sftp/LICENSE +16 -0
  214. data/core/services/sftp/src/core.rs +1 -1
  215. data/core/services/sftp/src/reader.rs +1 -1
  216. data/core/services/sled/Cargo.toml +1 -1
  217. data/core/services/sled/LICENSE +16 -0
  218. data/core/services/sqlite/Cargo.toml +2 -2
  219. data/core/services/sqlite/LICENSE +16 -0
  220. data/core/services/surrealdb/Cargo.toml +2 -2
  221. data/core/services/surrealdb/LICENSE +16 -0
  222. data/core/services/swift/Cargo.toml +1 -1
  223. data/core/services/swift/LICENSE +16 -0
  224. data/core/services/tikv/Cargo.toml +2 -2
  225. data/core/services/tikv/LICENSE +16 -0
  226. data/core/services/tos/Cargo.toml +1 -1
  227. data/core/services/tos/LICENSE +16 -0
  228. data/core/services/upyun/Cargo.toml +1 -1
  229. data/core/services/upyun/LICENSE +16 -0
  230. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  231. data/core/services/vercel-artifacts/LICENSE +16 -0
  232. data/core/services/vercel-blob/Cargo.toml +1 -1
  233. data/core/services/vercel-blob/LICENSE +16 -0
  234. data/core/services/webdav/Cargo.toml +1 -2
  235. data/core/services/webdav/LICENSE +16 -0
  236. data/core/services/webhdfs/Cargo.toml +2 -2
  237. data/core/services/webhdfs/LICENSE +16 -0
  238. data/core/services/yandex-disk/Cargo.toml +1 -1
  239. data/core/services/yandex-disk/LICENSE +16 -0
  240. data/core/src/lib.rs +1 -3
  241. data/core/testkit/Cargo.toml +4 -4
  242. data/core/testkit/LICENSE +16 -0
  243. data/core/upgrade.md +8 -0
  244. metadata +3 -2
data/core/core/LICENSE CHANGED
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -1,3 +1,11 @@
1
+ # Unreleased
2
+
3
+ ## Raw API
4
+
5
+ ### Submit tasks through `ConcurrentTasks::execute`
6
+
7
+ `raw::ConcurrentTasks::create_task` has been removed. Submit inputs with `execute()` and collect outputs in submission order with `next()`. These methods keep task ownership, completion accounting, and retry handling inside the queue. If code needs input data after successful completion, include that data in the factory's output type.
8
+
1
9
  # Upgrade to v0.59
2
10
 
3
11
  ## Public API
data/core/core/src/lib.rs CHANGED
@@ -15,9 +15,7 @@
15
15
  // specific language governing permissions and limitations
16
16
  // under the License.
17
17
 
18
- #![doc(
19
- html_logo_url = "https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"
20
- )]
18
+ #![doc(html_logo_url = "https://opendal.apache.org/img/logo.svg")]
21
19
  #![doc = include_str!("../README.md")]
22
20
  #![cfg_attr(docsrs, feature(doc_cfg))]
23
21
  #![cfg_attr(docsrs, doc(auto_cfg))]
@@ -63,7 +63,10 @@ impl<T: Send> MaybeSend for T {}
63
63
  #[cfg(target_arch = "wasm32")]
64
64
  impl<T> MaybeSend for T {}
65
65
 
66
- /// ConcurrentTasks is used to execute tasks concurrently.
66
+ /// ConcurrentTasks executes tasks concurrently and collects outputs in submission order.
67
+ ///
68
+ /// Submit inputs with [`Self::execute`] and collect outputs with [`Self::next`].
69
+ /// The queue owns the task handles and tracks their completion for concurrency control.
67
70
  ///
68
71
  /// ConcurrentTasks has two generic types:
69
72
  ///
@@ -75,20 +78,20 @@ impl<T> MaybeSend for T {}
75
78
  /// The code patterns below are intentional; please do not modify them unless you fully understand these notes.
76
79
  ///
77
80
  /// ```skip
78
- /// let (i, o) = self
81
+ /// let result = self
79
82
  /// .tasks
80
83
  /// .front_mut() // Use `front_mut` instead of `pop_front`
81
84
  /// .expect("tasks must be available")
82
85
  /// .await;
83
86
  /// ...
84
- /// match o {
87
+ /// match result {
85
88
  /// Ok(o) => {
86
89
  /// let _ = self.tasks.pop_front(); // `pop_front` after got `Ok(o)`
87
90
  /// self.results.push_back(o)
88
91
  /// }
89
- /// Err(err) => {
92
+ /// Err((i, err)) => {
90
93
  /// if err.is_temporary() {
91
- /// let task = self.create_task(i);
94
+ /// let task = self.spawn_task(i);
92
95
  /// self.tasks
93
96
  /// .front_mut()
94
97
  /// .expect("tasks must be available")
@@ -128,7 +131,7 @@ pub struct ConcurrentTasks<I, O> {
128
131
  /// to poll the tasks to see if they are ready.
129
132
  ///
130
133
  /// Dropping task without `await` it will cancel the task.
131
- tasks: VecDeque<Task<(I, Result<O>)>>,
134
+ tasks: VecDeque<Task<Result<O, (I, Error)>>>,
132
135
  /// `results` stores the successful results.
133
136
  results: VecDeque<O>,
134
137
 
@@ -202,13 +205,15 @@ impl<I: Send + 'static, O: Send + 'static> ConcurrentTasks<I, O> {
202
205
  !self.results.is_empty()
203
206
  }
204
207
 
205
- /// Create a task with given input.
206
- pub fn create_task(&self, input: I) -> Task<(I, Result<O>)> {
208
+ fn spawn_task(&self, input: I) -> Task<Result<O, (I, Error)>> {
207
209
  let completed = self.completed_but_unretrieved.clone();
208
-
209
- let fut = (self.factory)(input).inspect(move |_| {
210
- completed.fetch_add(1, Ordering::Relaxed);
211
- });
210
+ let fut = (self.factory)(input)
211
+ // Completed tasks can remain queued while the caller produces more work.
212
+ // Only failures need to retain their input for a retry.
213
+ .map(|(input, result)| result.map_err(|err| (input, err)))
214
+ .inspect(move |_| {
215
+ completed.fetch_add(1, Ordering::Relaxed);
216
+ });
212
217
 
213
218
  self.executor.execute(fut)
214
219
  }
@@ -240,22 +245,22 @@ impl<I: Send + 'static, O: Send + 'static> ConcurrentTasks<I, O> {
240
245
  }
241
246
 
242
247
  if !self.has_remaining() {
243
- let (i, o) = self
248
+ let result = self
244
249
  .tasks
245
250
  .front_mut()
246
251
  .expect("tasks must be available")
247
252
  .await;
248
253
  self.completed_but_unretrieved
249
254
  .fetch_sub(1, Ordering::Relaxed);
250
- match o {
255
+ match result {
251
256
  Ok(o) => {
252
257
  let _ = self.tasks.pop_front();
253
258
  self.results.push_back(o)
254
259
  }
255
- Err(err) => {
260
+ Err((i, err)) => {
256
261
  // Retry this task if the error is temporary
257
262
  if err.is_temporary() {
258
- let task = self.create_task(i);
263
+ let task = self.spawn_task(i);
259
264
  self.tasks
260
265
  .front_mut()
261
266
  .expect("tasks must be available")
@@ -269,7 +274,7 @@ impl<I: Send + 'static, O: Send + 'static> ConcurrentTasks<I, O> {
269
274
  }
270
275
  }
271
276
 
272
- self.tasks.push_back(self.create_task(input));
277
+ self.tasks.push_back(self.spawn_task(input));
273
278
  Ok(())
274
279
  }
275
280
 
@@ -287,18 +292,18 @@ impl<I: Send + 'static, O: Send + 'static> ConcurrentTasks<I, O> {
287
292
  }
288
293
 
289
294
  if let Some(task) = self.tasks.front_mut() {
290
- let (i, o) = task.await;
295
+ let result = task.await;
291
296
  self.completed_but_unretrieved
292
297
  .fetch_sub(1, Ordering::Relaxed);
293
- return match o {
298
+ return match result {
294
299
  Ok(o) => {
295
300
  let _ = self.tasks.pop_front();
296
301
  Some(Ok(o))
297
302
  }
298
- Err(err) => {
303
+ Err((i, err)) => {
299
304
  // Retry this task if the error is temporary
300
305
  if err.is_temporary() {
301
- let task = self.create_task(i);
306
+ let task = self.spawn_task(i);
302
307
  self.tasks
303
308
  .front_mut()
304
309
  .expect("tasks must be available")
@@ -62,6 +62,15 @@ pub fn with_error_response_context(mut err: Error, mut parts: Parts) -> Error {
62
62
  parts.headers.remove("WWW-Authenticate");
63
63
  parts.headers.remove("Proxy-Authenticate");
64
64
 
65
+ if parts.headers.contains_key(http::header::LOCATION) {
66
+ let location = crate::HttpUri::from_response_location(&mut parts)
67
+ .map(|uri| uri.redacted_uri())
68
+ .unwrap_or("<invalid URL>")
69
+ .parse()
70
+ .unwrap_or_else(|_| http::HeaderValue::from_static("<invalid URL>"));
71
+ parts.headers.insert(http::header::LOCATION, location);
72
+ }
73
+
65
74
  err = err.with_context("response", format!("{parts:?}"));
66
75
 
67
76
  err
@@ -1520,6 +1520,7 @@ mod tests {
1520
1520
  }
1521
1521
 
1522
1522
  #[test]
1523
+ #[cfg(target_pointer_width = "64")]
1523
1524
  fn compact_operation_layouts() {
1524
1525
  assert_eq!(size_of::<OpRead>(), 16);
1525
1526
  assert_eq!(size_of::<OpStat>(), 16);
@@ -36,12 +36,16 @@ use crate::types::Result;
36
36
  mod body;
37
37
  pub use body::HttpBody;
38
38
 
39
+ mod uri;
40
+ pub use uri::{HttpRedirect, HttpUri};
41
+
39
42
  static DEFAULT_HTTP_TRANSPORTER: OnceLock<HttpTransporter> = OnceLock::new();
40
43
 
41
44
  /// HTTP transport used by OpenDAL services.
42
45
  ///
43
46
  /// Implement this trait to provide a custom HTTP backend. A transport must
44
47
  /// support 3xx redirection.
48
+ /// It can support [`HttpRedirect`] to let services reuse redirect destinations.
45
49
  pub trait HttpTransport: Send + Sync + Unpin + 'static {
46
50
  /// Fetch a request and return a streamable [`HttpBody`].
47
51
  #[cfg(not(target_arch = "wasm32"))]
@@ -114,8 +118,32 @@ impl HttpTransporter {
114
118
  }
115
119
 
116
120
  /// Fetch a request and return a streamable [`HttpBody`].
117
- pub async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
118
- self.inner.fetch_dyn(req).await
121
+ ///
122
+ /// Mark credential headers as sensitive for diagnostics without changing
123
+ /// their values.
124
+ pub async fn fetch(&self, mut req: Request<Buffer>) -> Result<Response<HttpBody>> {
125
+ // Mark credentials at the shared boundary before any transport wrapper
126
+ // formats the request. Values on the wire remain unchanged.
127
+ for (name, value) in req.headers_mut() {
128
+ if matches!(
129
+ name.as_str(),
130
+ "authorization" | "proxy-authorization" | "cookie" | "cookie2"
131
+ ) {
132
+ value.set_sensitive(true);
133
+ }
134
+ }
135
+ let mut resp = self.inner.fetch_dyn(req).await?;
136
+ for (name, value) in resp.headers_mut() {
137
+ if matches!(
138
+ name.as_str(),
139
+ "set-cookie" | "www-authenticate" | "proxy-authenticate"
140
+ ) {
141
+ value.set_sensitive(true);
142
+ }
143
+ }
144
+ let (mut parts, body) = resp.into_parts();
145
+ HttpUri::from_response_location(&mut parts);
146
+ Ok(Response::from_parts(parts, body))
119
147
  }
120
148
  }
121
149
 
@@ -0,0 +1,166 @@
1
+ // Licensed to the Apache Software Foundation (ASF) under one
2
+ // or more contributor license agreements. See the NOTICE file
3
+ // distributed with this work for additional information
4
+ // regarding copyright ownership. The ASF licenses this file
5
+ // to you under the Apache License, Version 2.0 (the
6
+ // "License"); you may not use this file except in compliance
7
+ // with the License. You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing,
12
+ // software distributed under the License is distributed on an
13
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ // KIND, either express or implied. See the License for the
15
+ // specific language governing permissions and limitations
16
+ // under the License.
17
+
18
+ use std::fmt::{Debug, Formatter};
19
+ use std::sync::Arc;
20
+
21
+ use percent_encoding::percent_decode_str;
22
+
23
+ /// An HTTP URI or relative location with a precomputed diagnostic representation.
24
+ ///
25
+ /// Construction preserves the original spelling and masks user information and
26
+ /// recognized credential query values in a separate representation. It does not
27
+ /// validate, resolve, or normalize the URI. Ordinary query values remain intact.
28
+ /// Clones share both representations; formatting does not repeat redaction.
29
+ ///
30
+ /// Recognized query names are `Signature`, `Policy`, `token`, `access_token`,
31
+ /// `X-Amz-Signature`, and `X-Amz-Security-Token`, matched case-insensitively after
32
+ /// decoding the name. Arbitrary application parameters are not classified.
33
+ #[derive(Clone)]
34
+ pub struct HttpUri {
35
+ original: Arc<str>,
36
+ redacted: Arc<str>,
37
+ }
38
+
39
+ impl HttpUri {
40
+ /// Retain the original URI and compute its diagnostic representation once.
41
+ pub fn new(uri: impl Into<Arc<str>>) -> Self {
42
+ let original = uri.into();
43
+ let redacted = redact_uri(&original).into();
44
+ Self { original, redacted }
45
+ }
46
+
47
+ /// Return the original URI, including any credentials it contains.
48
+ pub fn original_uri(&self) -> &str {
49
+ &self.original
50
+ }
51
+
52
+ /// Return the URI with recognized credentials masked for diagnostics.
53
+ pub fn redacted_uri(&self) -> &str {
54
+ &self.redacted
55
+ }
56
+
57
+ /// Get or create the diagnostic value for a response's `Location` header.
58
+ ///
59
+ /// Store it in response extensions so service errors and shared response
60
+ /// diagnostics reuse the same value. If a wrapper changes the header,
61
+ /// replace the cached value. Return `None` for missing or non-text headers.
62
+ /// The original header is not changed.
63
+ pub fn from_response_location(parts: &mut http::response::Parts) -> Option<&Self> {
64
+ let location = parts.headers.get(http::header::LOCATION)?.to_str().ok()?;
65
+ if parts
66
+ .extensions
67
+ .get::<Self>()
68
+ .is_none_or(|uri| uri.original_uri() != location)
69
+ {
70
+ let uri = Self::new(location);
71
+ parts.extensions.insert(uri);
72
+ }
73
+ parts.extensions.get::<Self>()
74
+ }
75
+ }
76
+
77
+ impl Debug for HttpUri {
78
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
79
+ f.debug_tuple("HttpUri")
80
+ .field(&self.redacted_uri())
81
+ .finish()
82
+ }
83
+ }
84
+
85
+ /// A reusable HTTP redirect handled by the transport.
86
+ ///
87
+ /// A transport that supports this extension may return it with a response after
88
+ /// following redirects. Services can copy it into later request extensions to
89
+ /// reuse that destination while keeping the original request URI for diagnostics.
90
+ /// The transport must send to the destination and apply its redirect credential
91
+ /// policy, including credentials supplied by client defaults.
92
+ ///
93
+ /// The destination retains its original and redacted representations in
94
+ /// [`HttpUri`]. Only the transport should use the original as a request URI.
95
+ /// Services may inspect it to determine whether reuse is valid, but must use
96
+ /// the redacted representation for diagnostics. This extension alone does not
97
+ /// establish that the original path still selects the same object. Transports
98
+ /// that do not support reusable redirects must not return it.
99
+ #[derive(Clone)]
100
+ pub struct HttpRedirect(HttpUri);
101
+
102
+ impl HttpRedirect {
103
+ /// Record a destination that this transport can reuse.
104
+ pub fn new(uri: http::Uri) -> Self {
105
+ Self(HttpUri::new(uri.to_string()))
106
+ }
107
+
108
+ /// Return the destination's original and diagnostic representations.
109
+ pub fn uri(&self) -> &HttpUri {
110
+ &self.0
111
+ }
112
+ }
113
+
114
+ impl Debug for HttpRedirect {
115
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
116
+ f.debug_tuple("HttpRedirect")
117
+ .field(&self.0.redacted_uri())
118
+ .finish()
119
+ }
120
+ }
121
+
122
+ fn redact_uri(value: &str) -> String {
123
+ let mut value = value.to_string();
124
+ let authority_start = value
125
+ .split_once("://")
126
+ .filter(|(scheme, _)| {
127
+ scheme.eq_ignore_ascii_case("http") || scheme.eq_ignore_ascii_case("https")
128
+ })
129
+ .map(|(scheme, _)| scheme.len() + 3)
130
+ .or_else(|| value.starts_with("//").then_some(2));
131
+ if let Some(start) = authority_start {
132
+ let end = value[start..]
133
+ .find(['/', '?', '#'])
134
+ .map_or(value.len(), |offset| start + offset);
135
+ if let Some(offset) = value[start..end].rfind('@') {
136
+ value.replace_range(start..start + offset, "[REDACTED]");
137
+ }
138
+ }
139
+
140
+ let end = value.find('#').unwrap_or(value.len());
141
+ if let Some(start) = value[..end].find('?') {
142
+ let query = value[start + 1..end]
143
+ .split('&')
144
+ .map(|pair| {
145
+ let Some((key, _)) = pair.split_once('=') else {
146
+ return pair.to_string();
147
+ };
148
+ let name = percent_decode_str(key).decode_utf8_lossy();
149
+ match name.to_ascii_lowercase().as_str() {
150
+ "signature"
151
+ | "policy"
152
+ | "token"
153
+ | "access_token"
154
+ | "x-amz-signature"
155
+ | "x-amz-security-token" => {
156
+ format!("{key}=[REDACTED]")
157
+ }
158
+ _ => pair.to_string(),
159
+ }
160
+ })
161
+ .collect::<Vec<_>>()
162
+ .join("&");
163
+ value.replace_range(start + 1..end, &query);
164
+ }
165
+ value
166
+ }
@@ -743,6 +743,7 @@ mod tests {
743
743
  use super::*;
744
744
 
745
745
  #[test]
746
+ #[cfg(target_pointer_width = "64")]
746
747
  fn metadata_layout_is_40_bytes() {
747
748
  assert_eq!(size_of::<MetadataHeader>(), 24);
748
749
  assert_eq!(size_of::<CompactValues>(), 16);
@@ -25,8 +25,10 @@ pub(crate) mod compact;
25
25
 
26
26
  mod http_transport;
27
27
  pub use http_transport::HttpBody;
28
+ pub use http_transport::HttpRedirect;
28
29
  pub use http_transport::HttpTransport;
29
30
  pub use http_transport::HttpTransporter;
31
+ pub use http_transport::HttpUri;
30
32
 
31
33
  mod entry;
32
34
  pub use entry::Entry;
@@ -53,7 +53,7 @@ bytes = { workspace = true }
53
53
  futures = { workspace = true, features = ["std"] }
54
54
  http = { workspace = true }
55
55
  http-body = "1"
56
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
56
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
57
57
  reqwest = { version = "0.13.4", features = [
58
58
  "stream",
59
59
  ], default-features = false }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -32,6 +32,7 @@ use opendal_core::Buffer;
32
32
  use opendal_core::Error;
33
33
  use opendal_core::ErrorKind;
34
34
  use opendal_core::HttpBody;
35
+ use opendal_core::HttpRedirect;
35
36
  use opendal_core::HttpTransport;
36
37
  use opendal_core::HttpTransporter;
37
38
  use opendal_core::Result;
@@ -79,15 +80,43 @@ impl HttpTransport for ReqwestTransport {
79
80
  let uri = req.uri().clone();
80
81
  let is_head = req.method() == http::Method::HEAD;
81
82
 
82
- let (parts, body) = req.into_parts();
83
+ let (mut parts, body) = req.into_parts();
84
+ let original = uri.to_string();
85
+ let target = parts
86
+ .extensions
87
+ .get::<HttpRedirect>()
88
+ .map(|redirect| redirect.uri().original_uri())
89
+ .unwrap_or(&original);
83
90
 
84
- let url = reqwest::Url::parse(&uri.to_string()).map_err(|err| {
91
+ let url = reqwest::Url::parse(target).map_err(|err| {
85
92
  Error::new(ErrorKind::Unexpected, "request url is invalid")
86
93
  .with_operation("reqwest::fetch")
87
94
  .with_context("url", uri.to_string())
88
95
  .set_source(err)
89
96
  })?;
90
97
 
98
+ if parts.extensions.get::<HttpRedirect>().is_some() {
99
+ let original = reqwest::Url::parse(&uri.to_string()).map_err(|err| {
100
+ Error::new(ErrorKind::Unexpected, "original request url is invalid").set_source(err)
101
+ })?;
102
+ if original.origin() != url.origin() {
103
+ // A reused redirect must not gain credentials from client
104
+ // defaults or cookie storage that a normal redirect strips.
105
+ for name in [
106
+ "authorization",
107
+ "proxy-authorization",
108
+ "cookie",
109
+ "cookie2",
110
+ "www-authenticate",
111
+ ] {
112
+ parts.headers.insert(
113
+ http::header::HeaderName::from_static(name),
114
+ http::HeaderValue::from_static(""),
115
+ );
116
+ }
117
+ }
118
+ }
119
+
91
120
  let mut req_builder = self
92
121
  .client
93
122
  .request(parts.method, url)
@@ -116,7 +145,7 @@ impl HttpTransport for ReqwestTransport {
116
145
  .with_operation("reqwest::send")
117
146
  .with_context("url", uri.to_string())
118
147
  .with_temporary(is_temporary_error(&err))
119
- .set_source(err)
148
+ .set_source(err.without_url())
120
149
  })?;
121
150
 
122
151
  // Get content length from header so that we can check it.
@@ -135,6 +164,19 @@ impl HttpTransport for ReqwestTransport {
135
164
  // it later.
136
165
  .extension(uri.clone());
137
166
 
167
+ // Optional metadata must not turn a successful fetch into an error.
168
+ if let Ok(target) = resp.url().as_str().parse::<http::Uri>()
169
+ && target != uri
170
+ {
171
+ let redirect = parts
172
+ .extensions
173
+ .get::<HttpRedirect>()
174
+ .filter(|redirect| redirect.uri().original_uri() == resp.url().as_str())
175
+ .cloned()
176
+ .unwrap_or_else(|| HttpRedirect::new(target));
177
+ hr = hr.extension(redirect);
178
+ }
179
+
138
180
  // Response builder under wasm doesn't support set version.
139
181
  #[cfg(not(target_arch = "wasm32"))]
140
182
  {
@@ -153,7 +195,7 @@ impl HttpTransport for ReqwestTransport {
153
195
  .with_operation("reqwest::fetch")
154
196
  .with_context("url", uri.to_string())
155
197
  .with_temporary(is_temporary_error(&err))
156
- .set_source(err)
198
+ .set_source(err.without_url())
157
199
  }),
158
200
  content_length,
159
201
  );
@@ -33,7 +33,7 @@ all-features = true
33
33
 
34
34
  [dependencies]
35
35
  async-backtrace = "0.2.6"
36
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
36
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
37
37
 
38
38
  [dev-dependencies]
39
- opendal-core = { path = "../../core", version = "0.59.1" }
39
+ opendal-core = { path = "../../core", version = "0.59.2" }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -34,7 +34,7 @@ all-features = true
34
34
  [dependencies]
35
35
  await-tree = "0.3"
36
36
  futures = { workspace = true }
37
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
38
38
 
39
39
  [dev-dependencies]
40
- opendal-core = { path = "../../core", version = "0.59.1" }
40
+ opendal-core = { path = "../../core", version = "0.59.2" }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -32,8 +32,8 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
35
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
36
36
 
37
37
  [dev-dependencies]
38
- opendal-core = { path = "../../core", version = "0.59.1" }
38
+ opendal-core = { path = "../../core", version = "0.59.2" }
39
39
  tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -32,8 +32,8 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
35
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
36
36
  rand = { workspace = true }
37
37
 
38
38
  [dev-dependencies]
39
- opendal-core = { path = "../../core", version = "0.59.1" }
39
+ opendal-core = { path = "../../core", version = "0.59.2" }