opendal 0.1.12 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +31 -36
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +25 -35
  5. data/README.md +1 -1
  6. data/core/CHANGELOG.md +4 -0
  7. data/core/Cargo.lock +128 -187
  8. data/core/Cargo.toml +95 -95
  9. data/core/DEPENDENCIES.rust.tsv +259 -289
  10. data/core/LICENSE +16 -0
  11. data/core/benches/vs_fs/Cargo.toml +1 -1
  12. data/core/benches/vs_s3/Cargo.toml +1 -1
  13. data/core/core/CHANGELOG.md +4 -0
  14. data/core/core/Cargo.toml +1 -1
  15. data/core/core/LICENSE +16 -0
  16. data/core/core/src/docs/upgrade.md +8 -0
  17. data/core/core/src/lib.rs +1 -3
  18. data/core/core/src/raw/futures_util.rs +26 -21
  19. data/core/core/src/raw/http_util/error.rs +9 -0
  20. data/core/core/src/raw/ops.rs +1 -0
  21. data/core/core/src/types/http_transport/mod.rs +30 -2
  22. data/core/core/src/types/http_transport/uri.rs +166 -0
  23. data/core/core/src/types/metadata.rs +1 -0
  24. data/core/core/src/types/mod.rs +2 -0
  25. data/core/http-transports/reqwest/Cargo.toml +1 -1
  26. data/core/http-transports/reqwest/LICENSE +16 -0
  27. data/core/http-transports/reqwest/src/lib.rs +46 -4
  28. data/core/layers/async-backtrace/Cargo.toml +2 -2
  29. data/core/layers/async-backtrace/LICENSE +16 -0
  30. data/core/layers/await-tree/Cargo.toml +2 -2
  31. data/core/layers/await-tree/LICENSE +16 -0
  32. data/core/layers/capability-check/Cargo.toml +2 -2
  33. data/core/layers/capability-check/LICENSE +16 -0
  34. data/core/layers/chaos/Cargo.toml +2 -2
  35. data/core/layers/chaos/LICENSE +16 -0
  36. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  37. data/core/layers/concurrent-limit/LICENSE +16 -0
  38. data/core/layers/dtrace/Cargo.toml +2 -2
  39. data/core/layers/dtrace/LICENSE +16 -0
  40. data/core/layers/fastmetrics/Cargo.toml +3 -3
  41. data/core/layers/fastmetrics/LICENSE +16 -0
  42. data/core/layers/fastrace/Cargo.toml +2 -2
  43. data/core/layers/fastrace/LICENSE +16 -0
  44. data/core/layers/foyer/Cargo.toml +2 -2
  45. data/core/layers/foyer/LICENSE +16 -0
  46. data/core/layers/hotpath/Cargo.toml +2 -2
  47. data/core/layers/hotpath/LICENSE +16 -0
  48. data/core/layers/immutable-index/Cargo.toml +2 -2
  49. data/core/layers/immutable-index/LICENSE +16 -0
  50. data/core/layers/logging/Cargo.toml +2 -2
  51. data/core/layers/logging/LICENSE +16 -0
  52. data/core/layers/metrics/Cargo.toml +3 -3
  53. data/core/layers/metrics/LICENSE +16 -0
  54. data/core/layers/mime-guess/Cargo.toml +2 -2
  55. data/core/layers/mime-guess/LICENSE +16 -0
  56. data/core/layers/observe-metrics-common/Cargo.toml +1 -1
  57. data/core/layers/observe-metrics-common/LICENSE +16 -0
  58. data/core/layers/otelmetrics/Cargo.toml +3 -3
  59. data/core/layers/otelmetrics/LICENSE +16 -0
  60. data/core/layers/oteltrace/Cargo.toml +2 -2
  61. data/core/layers/oteltrace/LICENSE +16 -0
  62. data/core/layers/prometheus/Cargo.toml +3 -3
  63. data/core/layers/prometheus/LICENSE +16 -0
  64. data/core/layers/prometheus-client/Cargo.toml +3 -3
  65. data/core/layers/prometheus-client/LICENSE +16 -0
  66. data/core/layers/retry/Cargo.toml +4 -4
  67. data/core/layers/retry/LICENSE +16 -0
  68. data/core/layers/route/Cargo.toml +3 -3
  69. data/core/layers/route/LICENSE +16 -0
  70. data/core/layers/tail-cut/Cargo.toml +2 -2
  71. data/core/layers/tail-cut/LICENSE +16 -0
  72. data/core/layers/throttle/Cargo.toml +2 -2
  73. data/core/layers/throttle/LICENSE +16 -0
  74. data/core/layers/timeout/Cargo.toml +3 -3
  75. data/core/layers/timeout/LICENSE +16 -0
  76. data/core/layers/tracing/Cargo.toml +2 -2
  77. data/core/layers/tracing/LICENSE +16 -0
  78. data/core/services/aliyun-drive/Cargo.toml +2 -2
  79. data/core/services/aliyun-drive/LICENSE +16 -0
  80. data/core/services/alluxio/Cargo.toml +1 -1
  81. data/core/services/alluxio/LICENSE +16 -0
  82. data/core/services/azblob/Cargo.toml +2 -2
  83. data/core/services/azblob/LICENSE +16 -0
  84. data/core/services/azblob/src/backend.rs +21 -7
  85. data/core/services/azblob/src/config.rs +4 -1
  86. data/core/services/azdls/Cargo.toml +3 -3
  87. data/core/services/azdls/LICENSE +16 -0
  88. data/core/services/azfile/Cargo.toml +2 -2
  89. data/core/services/azfile/LICENSE +16 -0
  90. data/core/services/azure-common/Cargo.toml +1 -1
  91. data/core/services/azure-common/LICENSE +16 -0
  92. data/core/services/b2/Cargo.toml +2 -2
  93. data/core/services/b2/LICENSE +16 -0
  94. data/core/services/cacache/Cargo.toml +1 -1
  95. data/core/services/cacache/LICENSE +16 -0
  96. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  97. data/core/services/cloudflare-kv/LICENSE +16 -0
  98. data/core/services/compfs/Cargo.toml +1 -1
  99. data/core/services/compfs/LICENSE +16 -0
  100. data/core/services/compfs/src/core.rs +27 -11
  101. data/core/services/cos/Cargo.toml +1 -1
  102. data/core/services/cos/LICENSE +16 -0
  103. data/core/services/d1/Cargo.toml +1 -1
  104. data/core/services/d1/LICENSE +16 -0
  105. data/core/services/dashmap/Cargo.toml +1 -1
  106. data/core/services/dashmap/LICENSE +16 -0
  107. data/core/services/dbfs/Cargo.toml +1 -1
  108. data/core/services/dbfs/LICENSE +16 -0
  109. data/core/services/dropbox/Cargo.toml +2 -2
  110. data/core/services/dropbox/LICENSE +16 -0
  111. data/core/services/etcd/Cargo.toml +2 -2
  112. data/core/services/etcd/LICENSE +16 -0
  113. data/core/services/etcd/src/core.rs +3 -3
  114. data/core/services/foundationdb/Cargo.toml +1 -1
  115. data/core/services/foundationdb/LICENSE +16 -0
  116. data/core/services/foyer/Cargo.toml +3 -3
  117. data/core/services/foyer/LICENSE +16 -0
  118. data/core/services/fs/Cargo.toml +1 -1
  119. data/core/services/fs/LICENSE +16 -0
  120. data/core/services/fs/src/core.rs +69 -10
  121. data/core/services/ftp/Cargo.toml +2 -2
  122. data/core/services/ftp/LICENSE +16 -0
  123. data/core/services/ftp/src/core.rs +1 -1
  124. data/core/services/ftp/src/reader.rs +1 -1
  125. data/core/services/ftp/src/writer.rs +1 -1
  126. data/core/services/gcs/Cargo.toml +3 -2
  127. data/core/services/gcs/LICENSE +16 -0
  128. data/core/services/gcs/src/composer.rs +11 -21
  129. data/core/services/gcs-grpc/Cargo.toml +1 -1
  130. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
  131. data/core/services/gdrive/Cargo.toml +3 -2
  132. data/core/services/gdrive/LICENSE +16 -0
  133. data/core/services/gdrive/src/docs.md +2 -0
  134. data/core/services/gdrive/src/path_index.rs +109 -18
  135. data/core/services/ghac/Cargo.toml +2 -2
  136. data/core/services/ghac/LICENSE +16 -0
  137. data/core/services/github/Cargo.toml +1 -1
  138. data/core/services/github/LICENSE +16 -0
  139. data/core/services/goosefs/Cargo.toml +13 -4
  140. data/core/services/goosefs/LICENSE +16 -0
  141. data/core/services/goosefs/README.md +11 -0
  142. data/core/services/goosefs/src/backend.rs +3 -5
  143. data/core/services/goosefs/src/core.rs +1 -1
  144. data/core/services/goosefs/src/docs.md +44 -3
  145. data/core/services/goosefs/src/reader.rs +22 -89
  146. data/core/services/gridfs/Cargo.toml +2 -2
  147. data/core/services/gridfs/LICENSE +16 -0
  148. data/core/services/hdfs/Cargo.toml +1 -1
  149. data/core/services/hdfs/LICENSE +16 -0
  150. data/core/services/hdfs-native/Cargo.toml +1 -1
  151. data/core/services/hdfs-native/LICENSE +16 -0
  152. data/core/services/hf/Cargo.toml +3 -3
  153. data/core/services/hf/LICENSE +16 -0
  154. data/core/services/hf/src/backend.rs +77 -9
  155. data/core/services/hf/src/config.rs +58 -0
  156. data/core/services/hf/src/core.rs +695 -318
  157. data/core/services/hf/src/docs.md +29 -0
  158. data/core/services/hf/src/lister.rs +3 -3
  159. data/core/services/hf/src/reader.rs +128 -143
  160. data/core/services/http/Cargo.toml +3 -3
  161. data/core/services/http/LICENSE +16 -0
  162. data/core/services/ipfs/Cargo.toml +1 -1
  163. data/core/services/ipfs/LICENSE +16 -0
  164. data/core/services/ipmfs/Cargo.toml +1 -1
  165. data/core/services/ipmfs/LICENSE +16 -0
  166. data/core/services/koofr/Cargo.toml +2 -2
  167. data/core/services/koofr/LICENSE +16 -0
  168. data/core/services/lakefs/Cargo.toml +1 -1
  169. data/core/services/lakefs/LICENSE +16 -0
  170. data/core/services/memcached/Cargo.toml +2 -2
  171. data/core/services/memcached/LICENSE +16 -0
  172. data/core/services/memcached/src/core.rs +3 -3
  173. data/core/services/mini_moka/Cargo.toml +1 -1
  174. data/core/services/mini_moka/LICENSE +16 -0
  175. data/core/services/moka/Cargo.toml +1 -1
  176. data/core/services/moka/LICENSE +16 -0
  177. data/core/services/mongodb/Cargo.toml +2 -2
  178. data/core/services/mongodb/LICENSE +16 -0
  179. data/core/services/monoiofs/Cargo.toml +2 -2
  180. data/core/services/monoiofs/LICENSE +16 -0
  181. data/core/services/monoiofs/src/core.rs +27 -11
  182. data/core/services/mysql/Cargo.toml +2 -2
  183. data/core/services/mysql/LICENSE +16 -0
  184. data/core/services/obs/Cargo.toml +1 -1
  185. data/core/services/obs/LICENSE +16 -0
  186. data/core/services/onedrive/Cargo.toml +2 -2
  187. data/core/services/onedrive/LICENSE +16 -0
  188. data/core/services/opfs/Cargo.toml +1 -1
  189. data/core/services/opfs/LICENSE +16 -0
  190. data/core/services/oss/Cargo.toml +1 -1
  191. data/core/services/oss/LICENSE +16 -0
  192. data/core/services/pcloud/Cargo.toml +1 -1
  193. data/core/services/pcloud/LICENSE +16 -0
  194. data/core/services/persy/Cargo.toml +1 -1
  195. data/core/services/persy/LICENSE +16 -0
  196. data/core/services/postgresql/Cargo.toml +2 -2
  197. data/core/services/postgresql/LICENSE +16 -0
  198. data/core/services/redb/Cargo.toml +1 -1
  199. data/core/services/redb/LICENSE +16 -0
  200. data/core/services/redis/Cargo.toml +2 -2
  201. data/core/services/redis/LICENSE +16 -0
  202. data/core/services/redis/src/core.rs +1 -1
  203. data/core/services/rocksdb/Cargo.toml +1 -1
  204. data/core/services/rocksdb/LICENSE +16 -0
  205. data/core/services/s3/Cargo.toml +1 -1
  206. data/core/services/s3/LICENSE +16 -0
  207. data/core/services/seafile/Cargo.toml +4 -2
  208. data/core/services/seafile/LICENSE +16 -0
  209. data/core/services/seafile/src/backend.rs +2 -3
  210. data/core/services/seafile/src/core.rs +189 -86
  211. data/core/services/seafile/src/docs.md +2 -0
  212. data/core/services/sftp/Cargo.toml +2 -2
  213. data/core/services/sftp/LICENSE +16 -0
  214. data/core/services/sftp/src/core.rs +1 -1
  215. data/core/services/sftp/src/reader.rs +1 -1
  216. data/core/services/sled/Cargo.toml +1 -1
  217. data/core/services/sled/LICENSE +16 -0
  218. data/core/services/sqlite/Cargo.toml +2 -2
  219. data/core/services/sqlite/LICENSE +16 -0
  220. data/core/services/surrealdb/Cargo.toml +2 -2
  221. data/core/services/surrealdb/LICENSE +16 -0
  222. data/core/services/swift/Cargo.toml +1 -1
  223. data/core/services/swift/LICENSE +16 -0
  224. data/core/services/tikv/Cargo.toml +2 -2
  225. data/core/services/tikv/LICENSE +16 -0
  226. data/core/services/tos/Cargo.toml +1 -1
  227. data/core/services/tos/LICENSE +16 -0
  228. data/core/services/upyun/Cargo.toml +1 -1
  229. data/core/services/upyun/LICENSE +16 -0
  230. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  231. data/core/services/vercel-artifacts/LICENSE +16 -0
  232. data/core/services/vercel-blob/Cargo.toml +1 -1
  233. data/core/services/vercel-blob/LICENSE +16 -0
  234. data/core/services/webdav/Cargo.toml +1 -2
  235. data/core/services/webdav/LICENSE +16 -0
  236. data/core/services/webhdfs/Cargo.toml +2 -2
  237. data/core/services/webhdfs/LICENSE +16 -0
  238. data/core/services/yandex-disk/Cargo.toml +1 -1
  239. data/core/services/yandex-disk/LICENSE +16 -0
  240. data/core/src/lib.rs +1 -3
  241. data/core/testkit/Cargo.toml +4 -4
  242. data/core/testkit/LICENSE +16 -0
  243. data/core/upgrade.md +8 -0
  244. metadata +3 -2
@@ -32,6 +32,35 @@ Use [`crate::HfConfig`] for serializable configuration and this builder's
32
32
  methods for direct construction. The field and method documentation defines
33
33
  accepted values, defaults, and environment interaction.
34
34
 
35
+ ### Caching resolve results
36
+
37
+ By default, every range resolves through the Hugging Face Hub in both HTTP and
38
+ XET download modes. Enable
39
+ [`enable_resolve_cache`](crate::Hf::enable_resolve_cache) to share resolved HTTP
40
+ download addresses and XET file metadata across readers and batches on the same
41
+ backend:
42
+
43
+ ```rust,no_run
44
+ let builder = opendal_service_hf::Hf::default()
45
+ .repo_type("dataset")
46
+ .repo_id("username/my-dataset")
47
+ .enable_resolve_cache(true);
48
+ ```
49
+
50
+ Enable the cache only when previously written files are not modified. Changed
51
+ files can remain invisible while a resolved address or XET file metadata is
52
+ reused, including changes from other clients or a floating repository revision.
53
+ Issued signed URLs can also remain usable after Hub permissions change.
54
+
55
+ The shared cache retains bounded in-memory metadata. HTTP addresses refresh
56
+ 30 seconds before their signed expiry, on the next read, or once after a cached
57
+ request returns 401 or 403. In HTTP mode, the first resolve returns its body
58
+ directly. Responses without supported redirect metadata or expiry continue to
59
+ resolve normally.
60
+ The reqwest transport supports the optional
61
+ [`HttpRedirect`](opendal_core::HttpRedirect) extension; custom transports may omit it.
62
+ Separate authorization identities require separately constructed backends.
63
+
35
64
  ## Examples
36
65
 
37
66
  ### Via Builder (Git-based dataset)
@@ -86,7 +86,7 @@ impl HfLister {
86
86
  recursive: bool,
87
87
  cursor: Option<&str>,
88
88
  ) -> Result<FileTree> {
89
- let uri = self.core.canonical_uri(&self.ctx, path).await?;
89
+ let uri = self.core.repo.uri(&self.core.root, path);
90
90
  let url = uri.file_tree_url(&self.core.endpoint, recursive, cursor);
91
91
 
92
92
  let req = self
@@ -94,7 +94,7 @@ impl HfLister {
94
94
  .request(http::Method::GET, &url, Operation::List, "FileTree")?
95
95
  .body(Buffer::new())
96
96
  .map_err(new_request_build_error)?;
97
- let resp = self.ctx.http_transport().fetch(req).await?;
97
+ let resp = self.core.send(&self.ctx, req).await?;
98
98
  if !resp.status().is_success() {
99
99
  let (parts, _) = resp.into_parts();
100
100
  return Err(parse_error(
@@ -210,7 +210,7 @@ mod tests {
210
210
 
211
211
  assert_eq!(
212
212
  mock_client.get_captured_url(),
213
- "https://huggingface.co/api/models/test-user/test-repo/tree/main/?expand=True"
213
+ "https://huggingface.co/api/models/test-user/test-repo/tree/main?expand=True"
214
214
  );
215
215
  assert!(page_ctx.done);
216
216
  let entry = page_ctx.entries.pop_front().expect("entry must exist");
@@ -16,9 +16,10 @@
16
16
  // under the License.
17
17
 
18
18
  use super::backend::*;
19
- use super::core::{HfCore, HfDownloadMode, XetFileResponse};
19
+ #[cfg(test)]
20
+ use super::core::HfDownloadMode;
21
+ use super::core::{HfCore, HfReadResponse};
20
22
  use asyncband::once::OnceCell;
21
- use bytes::Buf;
22
23
  use http::Response;
23
24
  use opendal_core::raw::*;
24
25
  use opendal_core::*;
@@ -45,8 +46,7 @@ fn xet_range(range: BytesRange) -> Option<Range<u64>> {
45
46
 
46
47
  impl HfReadStream {
47
48
  /// Build the stream directly from an already-known XET hash + size,
48
- /// whether that came from a resolve response just parsed in [`HfReader::open`]
49
- /// or was reused from its cache.
49
+ /// resolved or reused by [`HfCore`].
50
50
  async fn new_xet(
51
51
  group: &XetDownloadStreamGroup,
52
52
  hash: &str,
@@ -79,22 +79,6 @@ impl HfReadStream {
79
79
  let metadata = parse_into_metadata(path, resp.headers())?;
80
80
  Ok((RpRead::new(metadata), Self::Http(resp.into_body())))
81
81
  }
82
-
83
- /// Retrieve a resolve response's XET metadata, or hand the response
84
- /// back untouched if it doesn't carry any -- its body is then the
85
- /// actual range content, still there for [`Self::new_http`].
86
- async fn maybe_xet(
87
- resp: Response<HttpBody>,
88
- ) -> Result<std::result::Result<XetFileResponse, Response<HttpBody>>> {
89
- if !resp.headers().contains_key("x-xet-hash") {
90
- return Ok(Err(resp));
91
- }
92
- let (_, mut body) = resp.into_parts();
93
- let buf = body.to_buffer().await?;
94
- let info: XetFileResponse =
95
- serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
96
- Ok(Ok(info))
97
- }
98
82
  }
99
83
 
100
84
  fn map_session_error(e: SessionError) -> Error {
@@ -119,17 +103,6 @@ pub struct HfReader {
119
103
  backend: HfBackend,
120
104
  ctx: OperationContext,
121
105
  path: String,
122
- // `Some((hash, size))`/`None` once this path is known to be (or not be)
123
- // XET-backed; only consulted in `HfDownloadMode::Xet`. Single-flighted via
124
- // `get_or_try_init`, so concurrent cold opens share one classifying
125
- // resolve and every later range reuses it. If not XET-backed, that first
126
- // resolve's body goes unused and every open falls back to its own fresh
127
- // resolve -- one wasted request per reader, not per range.
128
- //
129
- // Never re-validated for this reader's life, so a file changed mid-read
130
- // (e.g. a new commit on a floating `main`) can serve stale content --
131
- // scoped to one reader, not the `Operator`, to bound that window.
132
- resolved: OnceCell<Option<(String, u64)>>,
133
106
  // Built once per reader and reused for every later XET range, seeded
134
107
  // from `HfCore`'s cached CAS token. Reuse across concurrent ranges
135
108
  // relies on the `xet` crate's documented (not type-enforced) support
@@ -143,7 +116,6 @@ impl HfReader {
143
116
  backend,
144
117
  ctx,
145
118
  path: path.to_string(),
146
- resolved: OnceCell::new(),
147
119
  xet_group: OnceCell::new(),
148
120
  }
149
121
  }
@@ -163,26 +135,6 @@ impl HfReader {
163
135
  .await?;
164
136
  HfReadStream::new_xet(group, hash, size, range).await
165
137
  }
166
-
167
- /// Classify an already-fetched resolve response and dispatch to the
168
- /// matching stream. Used by [`Self::open`]'s Http-mode branch, which
169
- /// classifies the same response it fetches bytes from. The Xet-mode
170
- /// fallback (re-resolving in `HfDownloadMode::Http` after classification
171
- /// came back non-XET) does not call this -- it never re-checks for
172
- /// XET metadata on that second resolve, so it goes straight to
173
- /// `new_http` instead.
174
- async fn dispatch(
175
- &self,
176
- core: &HfCore,
177
- path: &str,
178
- range: BytesRange,
179
- resp: Response<HttpBody>,
180
- ) -> Result<(RpRead, HfReadStream)> {
181
- match HfReadStream::maybe_xet(resp).await? {
182
- Ok(info) => self.xet_stream(core, &info.hash, info.size, range).await,
183
- Err(resp) => HfReadStream::new_http(path, resp),
184
- }
185
- }
186
138
  }
187
139
 
188
140
  impl oio::StreamRead for HfReader {
@@ -190,40 +142,11 @@ impl oio::StreamRead for HfReader {
190
142
  let core = &self.backend.core;
191
143
  let path = self.path.as_str();
192
144
 
193
- if core.download_mode != HfDownloadMode::Xet {
194
- // Http mode: resolve() is itself the byte-fetching request, done
195
- // fresh per range -- there's no separate metadata step to cache.
196
- let resp = core
197
- .resolve(&self.ctx, path, range, core.download_mode)
198
- .await?;
199
- let (rp, stream) = self.dispatch(core, path, range, resp).await?;
200
- return Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>));
201
- }
202
-
203
- // Xet mode: classification is single-flighted, see `resolved`'s
204
- // doc comment for why that's free for the XET outcome and cheap
205
- // (one extra request, once per reader) for the NotXet one.
206
- let classification = self
207
- .resolved
208
- .get_or_try_init(|| async {
209
- let resp = core
210
- .resolve(&self.ctx, path, range, HfDownloadMode::Xet)
211
- .await?;
212
- match HfReadStream::maybe_xet(resp).await? {
213
- Ok(info) => Ok(Some((info.hash, info.size))),
214
- Err(_) => Ok(None),
215
- }
216
- })
217
- .await?;
218
-
219
- let (rp, stream) = match classification {
220
- Some((hash, size)) => self.xet_stream(core, hash, *size, range).await?,
221
- None => {
222
- let resp = core
223
- .resolve(&self.ctx, path, range, HfDownloadMode::Http)
224
- .await?;
225
- HfReadStream::new_http(path, resp)?
145
+ let (rp, stream) = match core.read(&self.ctx, path, range).await? {
146
+ HfReadResponse::Xet(info) => {
147
+ self.xet_stream(core, &info.hash, info.size, range).await?
226
148
  }
149
+ HfReadResponse::Http(resp) => HfReadStream::new_http(path, resp)?,
227
150
  };
228
151
 
229
152
  Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
@@ -232,7 +155,9 @@ impl oio::StreamRead for HfReader {
232
155
 
233
156
  #[cfg(test)]
234
157
  mod tests {
235
- use super::super::backend::test_utils::{mbpp_operator, testing_dataset_core};
158
+ use super::super::backend::test_utils::{
159
+ mbpp_operator, miscased_mbpp_operator, testing_dataset_core,
160
+ };
236
161
  use super::super::core::HfRepoType;
237
162
  use super::super::core::test_utils::create_test_core;
238
163
  use super::super::core::{CommitFile, DeletedFile, HfCore};
@@ -260,6 +185,29 @@ mod tests {
260
185
  assert_eq!(xet_range(BytesRange::new(4, None)), Some(4..u64::MAX));
261
186
  }
262
187
 
188
+ #[tokio::test]
189
+ async fn test_default_reads_resolve_each_range_across_readers() -> Result<()> {
190
+ for mode in [HfDownloadMode::Http, HfDownloadMode::Xet] {
191
+ let (mut core, ctx, mock_client) = create_test_core(
192
+ HfRepoType::Model,
193
+ "test-user/test-repo",
194
+ "main",
195
+ "https://huggingface.co",
196
+ );
197
+ core.download_mode = mode;
198
+ let core = Arc::new(core);
199
+ let first = hf_reader_from_arc(core.clone(), ctx.clone(), "plain.txt");
200
+ let second = hf_reader_from_arc(core, ctx, "plain.txt");
201
+
202
+ for (index, reader) in [&first, &first, &second].into_iter().enumerate() {
203
+ let (_, mut stream) = reader.open(BytesRange::new(index as u64, Some(1))).await?;
204
+ assert!(!stream.read().await?.is_empty());
205
+ assert_eq!(mock_client.request_count(), index + 1);
206
+ }
207
+ }
208
+ Ok(())
209
+ }
210
+
263
211
  #[tokio::test]
264
212
  async fn test_http_read_uses_resolve_url() -> Result<()> {
265
213
  let (core, ctx, mock_client) = create_test_core(
@@ -308,26 +256,26 @@ mod tests {
308
256
  /// file has no separate metadata step to skip for its actual bytes: the
309
257
  /// classifying resolve's body is discarded and every open, including the
310
258
  /// first, fetches its own bytes with its own resolve. So the first open
311
- /// costs 3 (canonical repo id + classify + fetch) and every later one
312
- /// costs 1 (fetch only, classification and the canonical repo id already
313
- /// cached).
259
+ /// costs 2 (classify + fetch) and every later one costs 1 (fetch only,
260
+ /// classification already cached).
314
261
  #[tokio::test]
315
262
  async fn test_non_xet_reads_cache_classification_but_still_fetch_each_range() -> Result<()> {
316
- let (core, ctx, mock_client) = create_test_core(
263
+ let (mut core, ctx, mock_client) = create_test_core(
317
264
  HfRepoType::Model,
318
265
  "test-user/test-repo",
319
266
  "main",
320
267
  "https://huggingface.co",
321
268
  );
269
+ core.enable_resolve_cache = true;
322
270
  let reader = hf_reader(core, ctx, "plain.txt");
323
271
 
324
272
  let (_, mut s1) = reader.open(BytesRange::new(0, Some(1))).await?;
325
273
  s1.read().await?;
326
- assert_eq!(mock_client.request_count(), 3);
274
+ assert_eq!(mock_client.request_count(), 2);
327
275
 
328
276
  let (_, mut s2) = reader.open(BytesRange::new(1, Some(1))).await?;
329
277
  s2.read().await?;
330
- assert_eq!(mock_client.request_count(), 4);
278
+ assert_eq!(mock_client.request_count(), 3);
331
279
 
332
280
  Ok(())
333
281
  }
@@ -337,17 +285,16 @@ mod tests {
337
285
  /// stuck erroring for the reader's whole lifetime.
338
286
  #[tokio::test]
339
287
  async fn test_classification_retries_after_resolve_failure() -> Result<()> {
340
- let (core, ctx, mock_client) = create_test_core(
288
+ let (mut core, ctx, mock_client) = create_test_core(
341
289
  HfRepoType::Model,
342
290
  "test-user/test-repo",
343
291
  "main",
344
292
  "https://huggingface.co",
345
293
  );
294
+ core.enable_resolve_cache = true;
346
295
  let reader = hf_reader(core, ctx, "plain.txt");
347
296
  mock_client.fail_next_requests(1);
348
297
 
349
- // The injected failure hits the canonical repo id lookup, the first
350
- // request a classifying resolve makes.
351
298
  let result = reader.open(BytesRange::new(0, Some(1))).await;
352
299
  assert!(
353
300
  result.is_err(),
@@ -359,8 +306,8 @@ mod tests {
359
306
  stream.read().await?;
360
307
  assert_eq!(
361
308
  mock_client.request_count(),
362
- 4,
363
- "classification must retry (canonical repo id + resolve) then fetch the range (1 more)"
309
+ 3,
310
+ "classification must retry (1 resolve) then fetch the range (1 more)"
364
311
  );
365
312
 
366
313
  Ok(())
@@ -373,12 +320,13 @@ mod tests {
373
320
  /// racing to build its own group.
374
321
  #[tokio::test]
375
322
  async fn test_concurrent_cold_opens_on_xet_file_share_one_group() -> Result<()> {
376
- let (core, ctx, mock_client) = create_test_core(
323
+ let (mut core, ctx, mock_client) = create_test_core(
377
324
  HfRepoType::Model,
378
325
  "test-user/test-repo",
379
326
  "main",
380
327
  "https://huggingface.co",
381
328
  );
329
+ core.enable_resolve_cache = true;
382
330
  mock_client.set_xet_backed(&"00".repeat(32), 4);
383
331
  mock_client.set_xet_token_expires_at(u64::MAX);
384
332
  let reader = hf_reader(core, ctx, "xet-file.bin");
@@ -392,63 +340,68 @@ mod tests {
392
340
  r2?;
393
341
  r3?;
394
342
 
395
- // 1 shared canonical repo id lookup + 1 shared classifying resolve +
396
- // 1 shared xet-read-token fetch for the shared group build. Fetching
397
- // actual bytes from the group would need a real CAS server, so this
398
- // test only covers open().
399
- assert_eq!(mock_client.request_count(), 3);
343
+ // 1 shared classifying resolve + 1 shared xet-read-token fetch for
344
+ // the shared group build. Fetching actual bytes from the group would
345
+ // need a real CAS server, so this test only covers open().
346
+ assert_eq!(mock_client.request_count(), 2);
400
347
 
401
348
  Ok(())
402
349
  }
403
350
 
404
- /// Documents a known sharp edge in the single-flighted classification:
405
- /// the `Range` header on the one shared classifying resolve comes from
406
- /// whichever concurrent `open()` call happens to win `get_or_try_init`'s
407
- /// race, not from the caller that ends up consuming the result. All
408
- /// racing callers get that winner's classification regardless of their
409
- /// own requested range. See `resolved`'s field doc and the review notes
410
- /// on this PR -- not yet confirmed against the real HF service, so no
411
- /// fix has been applied; this test only pins today's behavior so a
412
- /// future change here is deliberate.
351
+ /// The probe asks for a fixed single byte, never the caller's range.
352
+ /// Verified against the live API: XET metadata is byte-identical for no
353
+ /// range, a head range, and a range far into the file, so one response
354
+ /// serves every caller -- and for a path that turns out not to be
355
+ /// XET-backed the discarded body is one byte instead of the whole file
356
+ /// (248,987 B measured before this).
413
357
  #[tokio::test]
414
- async fn test_concurrent_cold_opens_classify_with_one_racing_callers_range() -> Result<()> {
415
- let (core, ctx, mock_client) = create_test_core(
358
+ async fn test_xet_probe_uses_fixed_single_byte_range() -> Result<()> {
359
+ let (mut core, ctx, mock_client) = create_test_core(
416
360
  HfRepoType::Model,
417
361
  "test-user/test-repo",
418
362
  "main",
419
363
  "https://huggingface.co",
420
364
  );
421
- mock_client.set_xet_backed(&"00".repeat(32), 4);
365
+ core.enable_resolve_cache = true;
366
+ mock_client.set_xet_backed(&"00".repeat(32), 64);
422
367
  mock_client.set_xet_token_expires_at(u64::MAX);
423
368
  let reader = hf_reader(core, ctx, "xet-file.bin");
424
369
 
425
- let ranges = [
426
- BytesRange::new(0, Some(1)),
427
- BytesRange::new(10, Some(1)),
428
- BytesRange::new(20, Some(1)),
429
- ];
430
- let (r1, r2, r3) = futures::join!(
431
- reader.open(ranges[0]),
432
- reader.open(ranges[1]),
433
- reader.open(ranges[2]),
370
+ reader.open(BytesRange::new(8, Some(4))).await?;
371
+
372
+ assert_eq!(
373
+ mock_client.get_captured_classify_range_header().as_deref(),
374
+ Some("bytes=0-0"),
375
+ "the probe must not inherit the caller's range"
434
376
  );
435
- r1?;
436
- r2?;
437
- r3?;
438
377
 
439
- // Exactly one classifying resolve fires (see
440
- // `test_concurrent_cold_opens_on_xet_file_share_one_group`), and its
441
- // `Range` header must be exactly one of the three racing callers'
442
- // own ranges -- not absent, and not some range not asked for by any
443
- // of them.
444
- let captured = mock_client
445
- .get_captured_classify_range_header()
446
- .expect("the shared classifying resolve must carry a Range header");
378
+ Ok(())
379
+ }
380
+
381
+ /// Http mode never classifies. HF puts `x-xet-hash` on the 302 it
382
+ /// answers with, not on the CDN response the transport follows it to, so
383
+ /// a response that does carry the header is still streamed as bytes
384
+ /// rather than parsed as metadata -- and no separate probe is issued.
385
+ #[tokio::test]
386
+ async fn test_http_mode_streams_bytes_without_probing() -> Result<()> {
387
+ let (mut core, ctx, mock_client) = create_test_core(
388
+ HfRepoType::Model,
389
+ "test-user/test-repo",
390
+ "main",
391
+ "https://huggingface.co",
392
+ );
393
+ core.download_mode = HfDownloadMode::Http;
394
+ mock_client.set_xet_backed(&"11".repeat(32), 64);
395
+ let reader = hf_reader(core, ctx, "file.bin");
396
+
397
+ let (_, mut stream) = reader.open(BytesRange::new(0, Some(4))).await?;
398
+ let chunk = stream.read().await?;
399
+
447
400
  assert!(
448
- ranges.iter().any(|r| r.to_header() == captured),
449
- "the shared classifying resolve's Range header ({captured:?}) must match \
450
- one of the racing callers' own ranges"
401
+ chunk.to_bytes().starts_with(br#"{"hash""#),
402
+ "the body must be handed back verbatim, not parsed"
451
403
  );
404
+ assert_eq!(mock_client.request_count(), 1, "one resolve, no probe");
452
405
 
453
406
  Ok(())
454
407
  }
@@ -459,12 +412,13 @@ mod tests {
459
412
  /// it isn't XET-backed. Each still fetches its own range afterward.
460
413
  #[tokio::test]
461
414
  async fn test_concurrent_cold_opens_share_one_classifying_resolve() -> Result<()> {
462
- let (core, ctx, mock_client) = create_test_core(
415
+ let (mut core, ctx, mock_client) = create_test_core(
463
416
  HfRepoType::Model,
464
417
  "test-user/test-repo",
465
418
  "main",
466
419
  "https://huggingface.co",
467
420
  );
421
+ core.enable_resolve_cache = true;
468
422
  let reader = hf_reader(core, ctx, "plain.txt");
469
423
 
470
424
  let (r1, r2, r3) = futures::join!(
@@ -476,12 +430,43 @@ mod tests {
476
430
  r2?.1.read().await?;
477
431
  r3?.1.read().await?;
478
432
 
479
- // 1 canonical repo id lookup + 1 shared classifying resolve + 3 individual fetches.
480
- assert_eq!(mock_client.request_count(), 5);
433
+ // 1 shared classifying resolve + 3 individual fetches.
434
+ assert_eq!(mock_client.request_count(), 4);
481
435
 
482
436
  Ok(())
483
437
  }
484
438
 
439
+ /// Regression test for #8107 against the live API: a repo id whose case
440
+ /// differs from the canonical one.
441
+ ///
442
+ /// `stat` is the operation that matters here. It posts to `paths-info`,
443
+ /// and HF answers a miscased id with a `307` carrying a path-only
444
+ /// `Location`. A transport will follow that for a `GET` but hands a
445
+ /// bodied `POST` back unfollowed, which is exactly the failure #8107
446
+ /// reported, so this only passes if `HfCore::send` re-issues the request.
447
+ /// The read then covers the `resolve` path through the same repo. Both
448
+ /// use a public dataset, so no token is needed.
449
+ #[tokio::test]
450
+ #[ignore = "requires network access"]
451
+ async fn test_miscased_repo_id_follows_redirects() {
452
+ let path = "full/train-00000-of-00001.parquet";
453
+ let op = miscased_mbpp_operator();
454
+
455
+ let meta = op
456
+ .stat(path)
457
+ .await
458
+ .expect("stat must follow the case redirect on its paths-info POST");
459
+ assert!(meta.content_length() > 0);
460
+
461
+ let bytes = op
462
+ .read_with(path)
463
+ .range(0..4)
464
+ .await
465
+ .expect("read must succeed against a miscased repo id")
466
+ .to_vec();
467
+ assert_eq!(bytes, PARQUET_MAGIC);
468
+ }
469
+
485
470
  /// Exercises the XET download code path against a public dataset known to
486
471
  /// have XET-stored files. Behavior tests cannot reliably cover this path
487
472
  /// because the test dataset may not contain any XET files.
@@ -576,8 +561,8 @@ mod tests {
576
561
 
577
562
  /// Exercises the group-reuse path for real: two ranges fetched through
578
563
  /// the same `Reader` (mirroring `object_store::get_ranges`) must both
579
- /// return correct bytes even though only the first call resolves the
580
- /// XET hash and builds the CAS download group.
564
+ /// return correct bytes while each call resolves the XET hash and the
565
+ /// reader reuses its CAS download group.
581
566
  #[tokio::test]
582
567
  #[ignore = "requires network access"]
583
568
  async fn test_read_xet_multiple_ranges_on_one_reader() {
@@ -22,7 +22,7 @@ license = "Apache-2.0"
22
22
  name = "opendal-service-http"
23
23
  readme = "README.md"
24
24
  repository = "https://github.com/apache/opendal"
25
- version = "0.59.1"
25
+ version = "0.59.2"
26
26
 
27
27
  [package.metadata.docs.rs]
28
28
  all-features = true
@@ -30,9 +30,9 @@ all-features = true
30
30
  [dependencies]
31
31
  http = "1.4"
32
32
  log = "0.4"
33
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
33
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
34
34
  serde = { version = "1", features = ["derive"] }
35
35
 
36
36
  [dev-dependencies]
37
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
37
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
38
38
  tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -35,7 +35,7 @@ all-features = true
35
35
  bytes = { workspace = true }
36
36
  http = { workspace = true }
37
37
  log = { workspace = true }
38
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
38
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
39
39
  prost = "0.14.3"
40
40
  serde = { workspace = true, features = ["derive"] }
41
41
 
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -32,7 +32,7 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
35
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
36
36
 
37
37
  bytes = { workspace = true }
38
38
  http = { workspace = true }
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -32,11 +32,11 @@ version = { workspace = true }
32
32
  all-features = true
33
33
 
34
34
  [dependencies]
35
- asyncband = { workspace = true }
35
+ asyncband = { workspace = true, features = ["mutex", "once-cell"] }
36
36
  bytes = { workspace = true }
37
37
  http = { workspace = true }
38
38
  log = { workspace = true }
39
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
39
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
40
40
  serde = { workspace = true, features = ["derive"] }
41
41
  serde_json = { workspace = true }
42
42
 
@@ -199,3 +199,19 @@
199
199
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
200
  See the License for the specific language governing permissions and
201
201
  limitations under the License.
202
+
203
+
204
+ ------------------------------------------------------------------------------------
205
+ This product bundles third-party components under other open source licenses.
206
+ This section summarizes those components and their licenses. See the referenced
207
+ license files for text of these licenses.
208
+
209
+
210
+ Creative Commons Zero v1.0 Universal
211
+ ------------------------------------
212
+
213
+ bindings/dotnet/.gitignore
214
+
215
+ The above file is derived from VisualStudio.gitignore in
216
+ https://github.com/github/gitignore. Its license text is in
217
+ bindings/dotnet/LICENSE-CC0-1.0.txt.
@@ -35,7 +35,7 @@ all-features = true
35
35
  bytes = { workspace = true }
36
36
  http = { workspace = true }
37
37
  log = { workspace = true }
38
- opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
38
+ opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
39
39
  serde = { workspace = true, features = ["derive"] }
40
40
  serde_json = { workspace = true }
41
41