opendal 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +31 -36
  3. data/Cargo.toml +1 -1
  4. data/DEPENDENCIES.rust.tsv +25 -35
  5. data/README.md +1 -1
  6. data/core/CHANGELOG.md +22 -0
  7. data/core/Cargo.lock +145 -189
  8. data/core/Cargo.toml +95 -95
  9. data/core/DEPENDENCIES.rust.tsv +259 -290
  10. data/core/LICENSE +16 -0
  11. data/core/benches/vs_fs/Cargo.toml +1 -1
  12. data/core/benches/vs_s3/Cargo.toml +1 -1
  13. data/core/core/CHANGELOG.md +6031 -0
  14. data/core/core/Cargo.toml +1 -1
  15. data/core/core/LICENSE +16 -0
  16. data/core/core/src/docs/mod.rs +1 -1
  17. data/core/core/src/docs/upgrade.md +8 -0
  18. data/core/core/src/lib.rs +1 -3
  19. data/core/core/src/raw/futures_util.rs +26 -21
  20. data/core/core/src/raw/http_util/error.rs +9 -0
  21. data/core/core/src/raw/ops.rs +1 -0
  22. data/core/core/src/types/http_transport/mod.rs +30 -2
  23. data/core/core/src/types/http_transport/uri.rs +166 -0
  24. data/core/core/src/types/metadata.rs +1 -0
  25. data/core/core/src/types/mod.rs +2 -0
  26. data/core/http-transports/reqwest/Cargo.toml +1 -1
  27. data/core/http-transports/reqwest/LICENSE +16 -0
  28. data/core/http-transports/reqwest/src/lib.rs +46 -4
  29. data/core/layers/async-backtrace/Cargo.toml +2 -2
  30. data/core/layers/async-backtrace/LICENSE +16 -0
  31. data/core/layers/await-tree/Cargo.toml +2 -2
  32. data/core/layers/await-tree/LICENSE +16 -0
  33. data/core/layers/capability-check/Cargo.toml +2 -2
  34. data/core/layers/capability-check/LICENSE +16 -0
  35. data/core/layers/chaos/Cargo.toml +2 -2
  36. data/core/layers/chaos/LICENSE +16 -0
  37. data/core/layers/concurrent-limit/Cargo.toml +3 -3
  38. data/core/layers/concurrent-limit/LICENSE +16 -0
  39. data/core/layers/dtrace/Cargo.toml +2 -2
  40. data/core/layers/dtrace/LICENSE +16 -0
  41. data/core/layers/fastmetrics/Cargo.toml +3 -3
  42. data/core/layers/fastmetrics/LICENSE +16 -0
  43. data/core/layers/fastrace/Cargo.toml +2 -2
  44. data/core/layers/fastrace/LICENSE +16 -0
  45. data/core/layers/foyer/Cargo.toml +2 -2
  46. data/core/layers/foyer/LICENSE +16 -0
  47. data/core/layers/hotpath/Cargo.toml +2 -2
  48. data/core/layers/hotpath/LICENSE +16 -0
  49. data/core/layers/immutable-index/Cargo.toml +2 -2
  50. data/core/layers/immutable-index/LICENSE +16 -0
  51. data/core/layers/logging/Cargo.toml +2 -2
  52. data/core/layers/logging/LICENSE +16 -0
  53. data/core/layers/metrics/Cargo.toml +3 -3
  54. data/core/layers/metrics/LICENSE +16 -0
  55. data/core/layers/mime-guess/Cargo.toml +2 -2
  56. data/core/layers/mime-guess/LICENSE +16 -0
  57. data/core/layers/observe-metrics-common/Cargo.toml +1 -1
  58. data/core/layers/observe-metrics-common/LICENSE +16 -0
  59. data/core/layers/otelmetrics/Cargo.toml +3 -3
  60. data/core/layers/otelmetrics/LICENSE +16 -0
  61. data/core/layers/oteltrace/Cargo.toml +2 -2
  62. data/core/layers/oteltrace/LICENSE +16 -0
  63. data/core/layers/prometheus/Cargo.toml +3 -3
  64. data/core/layers/prometheus/LICENSE +16 -0
  65. data/core/layers/prometheus-client/Cargo.toml +3 -3
  66. data/core/layers/prometheus-client/LICENSE +16 -0
  67. data/core/layers/retry/Cargo.toml +4 -4
  68. data/core/layers/retry/LICENSE +16 -0
  69. data/core/layers/route/Cargo.toml +3 -3
  70. data/core/layers/route/LICENSE +16 -0
  71. data/core/layers/tail-cut/Cargo.toml +2 -2
  72. data/core/layers/tail-cut/LICENSE +16 -0
  73. data/core/layers/throttle/Cargo.toml +2 -2
  74. data/core/layers/throttle/LICENSE +16 -0
  75. data/core/layers/timeout/Cargo.toml +3 -3
  76. data/core/layers/timeout/LICENSE +16 -0
  77. data/core/layers/tracing/Cargo.toml +2 -2
  78. data/core/layers/tracing/LICENSE +16 -0
  79. data/core/services/aliyun-drive/Cargo.toml +2 -2
  80. data/core/services/aliyun-drive/LICENSE +16 -0
  81. data/core/services/alluxio/Cargo.toml +1 -1
  82. data/core/services/alluxio/LICENSE +16 -0
  83. data/core/services/azblob/Cargo.toml +2 -2
  84. data/core/services/azblob/LICENSE +16 -0
  85. data/core/services/azblob/src/backend.rs +21 -7
  86. data/core/services/azblob/src/config.rs +4 -1
  87. data/core/services/azdls/Cargo.toml +3 -3
  88. data/core/services/azdls/LICENSE +16 -0
  89. data/core/services/azfile/Cargo.toml +2 -2
  90. data/core/services/azfile/LICENSE +16 -0
  91. data/core/services/azure-common/Cargo.toml +1 -1
  92. data/core/services/azure-common/LICENSE +16 -0
  93. data/core/services/b2/Cargo.toml +2 -2
  94. data/core/services/b2/LICENSE +16 -0
  95. data/core/services/cacache/Cargo.toml +1 -1
  96. data/core/services/cacache/LICENSE +16 -0
  97. data/core/services/cloudflare-kv/Cargo.toml +1 -1
  98. data/core/services/cloudflare-kv/LICENSE +16 -0
  99. data/core/services/compfs/Cargo.toml +1 -1
  100. data/core/services/compfs/LICENSE +16 -0
  101. data/core/services/compfs/src/core.rs +27 -11
  102. data/core/services/cos/Cargo.toml +1 -1
  103. data/core/services/cos/LICENSE +16 -0
  104. data/core/services/d1/Cargo.toml +1 -1
  105. data/core/services/d1/LICENSE +16 -0
  106. data/core/services/dashmap/Cargo.toml +1 -1
  107. data/core/services/dashmap/LICENSE +16 -0
  108. data/core/services/dbfs/Cargo.toml +1 -1
  109. data/core/services/dbfs/LICENSE +16 -0
  110. data/core/services/dropbox/Cargo.toml +2 -2
  111. data/core/services/dropbox/LICENSE +16 -0
  112. data/core/services/etcd/Cargo.toml +2 -2
  113. data/core/services/etcd/LICENSE +16 -0
  114. data/core/services/etcd/src/core.rs +3 -3
  115. data/core/services/foundationdb/Cargo.toml +1 -1
  116. data/core/services/foundationdb/LICENSE +16 -0
  117. data/core/services/foyer/Cargo.toml +3 -3
  118. data/core/services/foyer/LICENSE +16 -0
  119. data/core/services/fs/Cargo.toml +1 -1
  120. data/core/services/fs/LICENSE +16 -0
  121. data/core/services/fs/src/core.rs +69 -10
  122. data/core/services/ftp/Cargo.toml +2 -2
  123. data/core/services/ftp/LICENSE +16 -0
  124. data/core/services/ftp/src/core.rs +1 -1
  125. data/core/services/ftp/src/reader.rs +1 -1
  126. data/core/services/ftp/src/writer.rs +1 -1
  127. data/core/services/gcs/Cargo.toml +3 -2
  128. data/core/services/gcs/LICENSE +16 -0
  129. data/core/services/gcs/src/composer.rs +11 -21
  130. data/core/services/gcs/src/core.rs +116 -2
  131. data/core/services/gcs-grpc/Cargo.toml +1 -1
  132. data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
  133. data/core/services/gdrive/Cargo.toml +3 -2
  134. data/core/services/gdrive/LICENSE +16 -0
  135. data/core/services/gdrive/src/docs.md +2 -0
  136. data/core/services/gdrive/src/path_index.rs +109 -18
  137. data/core/services/ghac/Cargo.toml +2 -2
  138. data/core/services/ghac/LICENSE +16 -0
  139. data/core/services/github/Cargo.toml +1 -1
  140. data/core/services/github/LICENSE +16 -0
  141. data/core/services/goosefs/Cargo.toml +13 -4
  142. data/core/services/goosefs/LICENSE +16 -0
  143. data/core/services/goosefs/README.md +11 -0
  144. data/core/services/goosefs/src/backend.rs +376 -49
  145. data/core/services/goosefs/src/config.rs +18 -10
  146. data/core/services/goosefs/src/core.rs +1 -1
  147. data/core/services/goosefs/src/docs.md +72 -3
  148. data/core/services/goosefs/src/reader.rs +22 -89
  149. data/core/services/goosefs/tests/master_addr_resolution.rs +103 -0
  150. data/core/services/gridfs/Cargo.toml +2 -2
  151. data/core/services/gridfs/LICENSE +16 -0
  152. data/core/services/hdfs/Cargo.toml +1 -1
  153. data/core/services/hdfs/LICENSE +16 -0
  154. data/core/services/hdfs-native/Cargo.toml +1 -1
  155. data/core/services/hdfs-native/LICENSE +16 -0
  156. data/core/services/hf/Cargo.toml +3 -3
  157. data/core/services/hf/LICENSE +16 -0
  158. data/core/services/hf/src/backend.rs +77 -9
  159. data/core/services/hf/src/config.rs +58 -0
  160. data/core/services/hf/src/core.rs +695 -318
  161. data/core/services/hf/src/docs.md +29 -0
  162. data/core/services/hf/src/lister.rs +3 -3
  163. data/core/services/hf/src/reader.rs +128 -143
  164. data/core/services/http/Cargo.toml +3 -3
  165. data/core/services/http/LICENSE +16 -0
  166. data/core/services/ipfs/Cargo.toml +1 -1
  167. data/core/services/ipfs/LICENSE +16 -0
  168. data/core/services/ipmfs/Cargo.toml +1 -1
  169. data/core/services/ipmfs/LICENSE +16 -0
  170. data/core/services/koofr/Cargo.toml +2 -2
  171. data/core/services/koofr/LICENSE +16 -0
  172. data/core/services/lakefs/Cargo.toml +1 -1
  173. data/core/services/lakefs/LICENSE +16 -0
  174. data/core/services/memcached/Cargo.toml +2 -2
  175. data/core/services/memcached/LICENSE +16 -0
  176. data/core/services/memcached/src/core.rs +3 -3
  177. data/core/services/mini_moka/Cargo.toml +1 -1
  178. data/core/services/mini_moka/LICENSE +16 -0
  179. data/core/services/moka/Cargo.toml +1 -1
  180. data/core/services/moka/LICENSE +16 -0
  181. data/core/services/mongodb/Cargo.toml +2 -2
  182. data/core/services/mongodb/LICENSE +16 -0
  183. data/core/services/monoiofs/Cargo.toml +2 -2
  184. data/core/services/monoiofs/LICENSE +16 -0
  185. data/core/services/monoiofs/src/core.rs +27 -11
  186. data/core/services/mysql/Cargo.toml +2 -2
  187. data/core/services/mysql/LICENSE +16 -0
  188. data/core/services/obs/Cargo.toml +1 -1
  189. data/core/services/obs/LICENSE +16 -0
  190. data/core/services/onedrive/Cargo.toml +2 -2
  191. data/core/services/onedrive/LICENSE +16 -0
  192. data/core/services/opfs/Cargo.toml +1 -1
  193. data/core/services/opfs/LICENSE +16 -0
  194. data/core/services/oss/Cargo.toml +1 -1
  195. data/core/services/oss/LICENSE +16 -0
  196. data/core/services/pcloud/Cargo.toml +1 -1
  197. data/core/services/pcloud/LICENSE +16 -0
  198. data/core/services/persy/Cargo.toml +1 -1
  199. data/core/services/persy/LICENSE +16 -0
  200. data/core/services/postgresql/Cargo.toml +2 -2
  201. data/core/services/postgresql/LICENSE +16 -0
  202. data/core/services/redb/Cargo.toml +1 -1
  203. data/core/services/redb/LICENSE +16 -0
  204. data/core/services/redis/Cargo.toml +2 -2
  205. data/core/services/redis/LICENSE +16 -0
  206. data/core/services/redis/src/core.rs +1 -1
  207. data/core/services/rocksdb/Cargo.toml +1 -1
  208. data/core/services/rocksdb/LICENSE +16 -0
  209. data/core/services/s3/Cargo.toml +1 -1
  210. data/core/services/s3/LICENSE +16 -0
  211. data/core/services/seafile/Cargo.toml +4 -2
  212. data/core/services/seafile/LICENSE +16 -0
  213. data/core/services/seafile/src/backend.rs +2 -3
  214. data/core/services/seafile/src/core.rs +189 -86
  215. data/core/services/seafile/src/docs.md +2 -0
  216. data/core/services/sftp/Cargo.toml +2 -2
  217. data/core/services/sftp/LICENSE +16 -0
  218. data/core/services/sftp/src/core.rs +1 -1
  219. data/core/services/sftp/src/reader.rs +1 -1
  220. data/core/services/sled/Cargo.toml +1 -1
  221. data/core/services/sled/LICENSE +16 -0
  222. data/core/services/sqlite/Cargo.toml +2 -2
  223. data/core/services/sqlite/LICENSE +16 -0
  224. data/core/services/surrealdb/Cargo.toml +2 -2
  225. data/core/services/surrealdb/LICENSE +16 -0
  226. data/core/services/swift/Cargo.toml +1 -1
  227. data/core/services/swift/LICENSE +16 -0
  228. data/core/services/tikv/Cargo.toml +2 -2
  229. data/core/services/tikv/LICENSE +16 -0
  230. data/core/services/tos/Cargo.toml +1 -1
  231. data/core/services/tos/LICENSE +16 -0
  232. data/core/services/upyun/Cargo.toml +1 -1
  233. data/core/services/upyun/LICENSE +16 -0
  234. data/core/services/vercel-artifacts/Cargo.toml +1 -1
  235. data/core/services/vercel-artifacts/LICENSE +16 -0
  236. data/core/services/vercel-blob/Cargo.toml +1 -1
  237. data/core/services/vercel-blob/LICENSE +16 -0
  238. data/core/services/webdav/Cargo.toml +1 -2
  239. data/core/services/webdav/LICENSE +16 -0
  240. data/core/services/webhdfs/Cargo.toml +2 -2
  241. data/core/services/webhdfs/LICENSE +16 -0
  242. data/core/services/yandex-disk/Cargo.toml +1 -1
  243. data/core/services/yandex-disk/LICENSE +16 -0
  244. data/core/src/lib.rs +1 -3
  245. data/core/testkit/Cargo.toml +4 -4
  246. data/core/testkit/LICENSE +16 -0
  247. data/core/upgrade.md +8 -0
  248. metadata +5 -2
@@ -16,17 +16,19 @@
16
16
  // under the License.
17
17
 
18
18
  use asyncband::mutex::Mutex;
19
- use asyncband::once::OnceCell;
20
19
  use bytes::Buf;
21
20
  use bytes::Bytes;
22
21
  use http::Request;
23
22
  use http::Response;
23
+ use http::StatusCode;
24
24
  use http::header;
25
25
  use log::debug;
26
26
  use serde::{Deserialize, Serialize};
27
+ use std::collections::HashMap;
27
28
  use std::fmt::Debug;
28
29
  use std::sync::Arc;
29
- use std::time::{SystemTime, UNIX_EPOCH};
30
+ use std::sync::Mutex as StdMutex;
31
+ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
30
32
 
31
33
  use xet::xet_session::{XetDownloadStreamGroup, XetSession, XetSessionBuilder, XetUploadCommit};
32
34
 
@@ -165,11 +167,6 @@ pub(super) struct XetFileResponse {
165
167
  pub size: u64,
166
168
  }
167
169
 
168
- #[derive(Deserialize)]
169
- pub(super) struct RepoInfoResponse {
170
- pub id: String,
171
- }
172
-
173
170
  /// Response shape of HF's `xet-{read,write}-token` endpoint. Matches
174
171
  /// `CasJWTInfo` in the vendored `xet` crate (`xet_client::hub_client::types`)
175
172
  /// — the same three fields regardless of read or write token.
@@ -215,10 +212,9 @@ impl XetTokenScope {
215
212
  }
216
213
  }
217
214
 
218
- /// The `Operation` a fetch for this scope's token should be tagged
219
- /// with, so a token-less writer hits `request()`'s local
220
- /// `PermissionDenied` fast-path instead of an unauthenticated network
221
- /// call.
215
+ /// `Operation` to tag this scope's token fetch with, so a token-less
216
+ /// writer is rejected locally by `request()` instead of hitting the
217
+ /// network.
222
218
  fn operation(self) -> Operation {
223
219
  match self {
224
220
  Self::Read => Operation::Read,
@@ -301,6 +297,123 @@ pub(super) struct LastCommit {
301
297
  pub date: String,
302
298
  }
303
299
 
300
+ pub(super) enum HfReadResponse {
301
+ Http(Response<HttpBody>),
302
+ Xet(XetFileResponse),
303
+ }
304
+
305
+ impl HfReadResponse {
306
+ async fn from_response(resp: Response<HttpBody>, mode: HfDownloadMode) -> Result<Self> {
307
+ if mode != HfDownloadMode::Xet || !resp.headers().contains_key("x-xet-hash") {
308
+ return Ok(Self::Http(resp));
309
+ }
310
+ let (_, mut body) = resp.into_parts();
311
+ let buf = body.to_buffer().await?;
312
+ let info = serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
313
+ Ok(Self::Xet(info))
314
+ }
315
+ }
316
+
317
+ // Bound entries, paths, and HTTP URIs. Busy entries are never evicted:
318
+ // recreating their slots would allow two concurrent resolutions for one path.
319
+ const RESOLVED_FILE_MAX_ENTRIES: usize = 512;
320
+ const RESOLVED_FILE_MAX_PATH_BYTES: usize = 4096;
321
+ const HTTP_DOWNLOAD_MAX_URI_BYTES: usize = 8192;
322
+ const HTTP_DOWNLOAD_EXPIRY_MARGIN: Duration = Duration::from_secs(30);
323
+
324
+ #[derive(Default)]
325
+ struct ResolvedFiles {
326
+ entries: StdMutex<HashMap<String, ResolvedFileEntry>>,
327
+ }
328
+
329
+ struct ResolvedFileEntry {
330
+ accessed: Instant,
331
+ state: Arc<Mutex<ResolvedFile>>,
332
+ }
333
+
334
+ #[derive(Default)]
335
+ enum ResolvedFile {
336
+ #[default]
337
+ Empty,
338
+ Http(Arc<HttpDownload>),
339
+ Xet(XetFileResponse),
340
+ // The path is not XET-backed and has no reusable HTTP destination.
341
+ NotXet,
342
+ // Wake existing waiters onto the uncached path when a response cannot be
343
+ // admitted. Its slot leaves the table so later reads can try again.
344
+ Bypass,
345
+ }
346
+
347
+ struct HttpDownload {
348
+ redirect: HttpRedirect,
349
+ valid_until: Instant,
350
+ }
351
+
352
+ impl ResolvedFiles {
353
+ fn entry(&self, path: &str) -> Option<Arc<Mutex<ResolvedFile>>> {
354
+ if path.len() > RESOLVED_FILE_MAX_PATH_BYTES {
355
+ return None;
356
+ }
357
+
358
+ let mut entries = self
359
+ .entries
360
+ .lock()
361
+ .expect("resolved file cache lock poisoned");
362
+ if entries.len() == RESOLVED_FILE_MAX_ENTRIES && !entries.contains_key(path) {
363
+ let oldest = entries
364
+ .iter()
365
+ .filter(|(_, entry)| Arc::strong_count(&entry.state) == 1)
366
+ .min_by_key(|(_, entry)| entry.accessed)
367
+ .map(|(path, _)| path.clone())?;
368
+ entries.remove(&oldest);
369
+ }
370
+ let entry = entries
371
+ .entry(path.to_string())
372
+ .or_insert_with(|| ResolvedFileEntry {
373
+ accessed: Instant::now(),
374
+ state: Arc::default(),
375
+ });
376
+ entry.accessed = Instant::now();
377
+ Some(entry.state.clone())
378
+ }
379
+ }
380
+
381
+ impl HttpDownload {
382
+ fn from_response(resp: &Response<HttpBody>) -> Option<Self> {
383
+ let redirect = resp.extensions().get::<HttpRedirect>()?;
384
+ let uri = redirect.uri().original_uri().parse::<http::Uri>().ok()?;
385
+ if uri.scheme_str() != Some("https")
386
+ || uri.authority()?.as_str().contains('@')
387
+ || redirect.uri().original_uri().len() > HTTP_DOWNLOAD_MAX_URI_BYTES
388
+ {
389
+ return None;
390
+ }
391
+
392
+ // HF's signed CDN destinations advertise their authorization deadline
393
+ // as Expires. Unknown signature formats fall back to fresh resolution.
394
+ let mut expirations = uri.query()?.split('&').filter_map(|pair| {
395
+ let (key, value) = pair.split_once('=')?;
396
+ (key == "Expires").then_some(value)
397
+ });
398
+ let expires = expirations.next()?.parse::<u64>().ok()?;
399
+ if expirations.next().is_some() {
400
+ return None;
401
+ }
402
+ let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?;
403
+ let lifetime = Duration::from_secs(expires)
404
+ .checked_sub(now)?
405
+ .checked_sub(HTTP_DOWNLOAD_EXPIRY_MARGIN)?;
406
+ if lifetime.is_zero() {
407
+ return None;
408
+ }
409
+
410
+ Some(Self {
411
+ redirect: redirect.clone(),
412
+ valid_until: Instant::now().checked_add(lifetime)?,
413
+ })
414
+ }
415
+ }
416
+
304
417
  // Core HuggingFace client that manages API interactions, authentication
305
418
  // and shared logic for reader/writer/lister.
306
419
 
@@ -314,7 +427,9 @@ pub struct HfCore {
314
427
  pub endpoint: String,
315
428
  pub xet_session: XetSession,
316
429
  pub download_mode: HfDownloadMode,
317
- canonical_repo: OnceCell<HfRepo>,
430
+ pub enable_resolve_cache: bool,
431
+ /// Shared file resolutions, keyed by paths relative to this core's root.
432
+ resolved_files: Arc<ResolvedFiles>,
318
433
  /// Cached CAS read token, shared by every `XetDownloadStreamGroup` this
319
434
  /// core creates, so at most one `xet-read-token` request happens per
320
435
  /// token lifetime instead of one per group (one per file read).
@@ -355,7 +470,8 @@ impl HfCore {
355
470
  endpoint,
356
471
  xet_session,
357
472
  download_mode,
358
- canonical_repo: OnceCell::new(),
473
+ enable_resolve_cache: false,
474
+ resolved_files: Arc::default(),
359
475
  xet_read_token: Arc::new(Mutex::new(None)),
360
476
  xet_write_token: Arc::new(Mutex::new(None)),
361
477
  }
@@ -404,8 +520,7 @@ impl HfCore {
404
520
  &self,
405
521
  ctx: &OperationContext,
406
522
  ) -> Result<XetUploadCommit> {
407
- let (token, refresh_url, refresh_headers) =
408
- self.xet_auth(ctx, XetTokenScope::Write).await?;
523
+ let token = self.cached_xet_token(ctx, XetTokenScope::Write).await?;
409
524
  self.xet_session
410
525
  .new_upload_commit()
411
526
  .map_err(|err| {
@@ -414,7 +529,11 @@ impl HfCore {
414
529
  })?
415
530
  .with_endpoint(token.cas_url)
416
531
  .with_token_info(token.access_token, token.expires_at)
417
- .with_token_refresh_url(refresh_url, refresh_headers)
532
+ .with_token_refresh_url(
533
+ self.repo
534
+ .xet_token_url(&self.endpoint, XetTokenScope::Write),
535
+ self.xet_token_refresh_headers(),
536
+ )
418
537
  .build()
419
538
  .await
420
539
  .map_err(|err| {
@@ -423,32 +542,9 @@ impl HfCore {
423
542
  })
424
543
  }
425
544
 
426
- /// A cached token for `scope`, plus the URL/headers to configure
427
- /// automatic refresh on a builder that outlives this call. Shared by
428
- /// [`Self::xet_upload_commit`] and [`Self::xet_download_group`], which
429
- /// otherwise differ only in which builder they seed with it. Uses the
430
- /// canonical repo id so the refresh URL doesn't depend on redirect
431
- /// behavior (see [`Self::canonical_repo`]). Resolves the repo once and
432
- /// hands it to [`Self::cached_xet_token`] so the token lookup doesn't
433
- /// resolve it again.
434
- async fn xet_auth(
435
- &self,
436
- ctx: &OperationContext,
437
- scope: XetTokenScope,
438
- ) -> Result<(XetToken, String, http::HeaderMap)> {
439
- let repo = self.canonical_repo(ctx).await?;
440
- let token = self.cached_xet_token(ctx, &repo, scope).await?;
441
- let refresh_url = repo.xet_token_url(&self.endpoint, scope);
442
- let refresh_headers = self.xet_token_refresh_headers();
443
- Ok((token, refresh_url, refresh_headers))
444
- }
445
-
446
- /// Get a still-valid cached token for `scope` against the already
447
- /// resolved `repo`, fetching and caching a fresh one if missing or
448
- /// close to expiry. The lock is held across the refresh request so
449
- /// concurrent callers single-flight onto one fetch; `repo` is resolved
450
- /// by the caller beforehand so that lookup doesn't extend this
451
- /// critical section.
545
+ /// Get a still-valid cached token for `scope`, fetching and caching a
546
+ /// fresh one if missing or close to expiry. The lock is held across the
547
+ /// refresh request so concurrent callers single-flight onto one fetch.
452
548
  ///
453
549
  /// On a clock failure, `now` falls back to `u64::MAX` so the token is
454
550
  /// treated as stale (extra `/api` traffic) rather than `0`, which would
@@ -456,7 +552,6 @@ impl HfCore {
456
552
  async fn cached_xet_token(
457
553
  &self,
458
554
  ctx: &OperationContext,
459
- repo: &HfRepo,
460
555
  scope: XetTokenScope,
461
556
  ) -> Result<XetToken> {
462
557
  let cache = match scope {
@@ -475,12 +570,12 @@ impl HfCore {
475
570
  return Ok(token.clone());
476
571
  }
477
572
 
478
- let url = repo.xet_token_url(&self.endpoint, scope);
573
+ let url = self.repo.xet_token_url(&self.endpoint, scope);
479
574
  let req = self
480
575
  .request(http::Method::GET, &url, scope.operation(), "XetToken")?
481
576
  .body(Buffer::new())
482
577
  .map_err(new_request_build_error)?;
483
- let resp = ctx.http_transport().fetch(req).await?;
578
+ let resp = self.send(ctx, req).await?;
484
579
  if !resp.status().is_success() {
485
580
  let (parts, _) = resp.into_parts();
486
581
  return Err(parse_error(
@@ -513,7 +608,7 @@ impl HfCore {
513
608
  &self,
514
609
  ctx: &OperationContext,
515
610
  ) -> Result<XetDownloadStreamGroup> {
516
- let (token, refresh_url, refresh_headers) = self.xet_auth(ctx, XetTokenScope::Read).await?;
611
+ let token = self.cached_xet_token(ctx, XetTokenScope::Read).await?;
517
612
  self.xet_session
518
613
  .new_download_stream_group()
519
614
  .map_err(|err| {
@@ -525,7 +620,10 @@ impl HfCore {
525
620
  })?
526
621
  .with_endpoint(token.cas_url)
527
622
  .with_token_info(token.access_token, token.expires_at)
528
- .with_token_refresh_url(refresh_url, refresh_headers)
623
+ .with_token_refresh_url(
624
+ self.repo.xet_token_url(&self.endpoint, XetTokenScope::Read),
625
+ self.xet_token_refresh_headers(),
626
+ )
529
627
  .build()
530
628
  .await
531
629
  .map_err(|err| {
@@ -548,15 +646,14 @@ impl HfCore {
548
646
  op: Operation,
549
647
  service_operation: &'static str,
550
648
  ) -> Result<http::request::Builder> {
551
- // Every outbound HF/CAS request is built here, so this is the one
552
- // place that can show what's actually being queried -- enable with
553
- // `RUST_LOG=opendal_service_hf::core=debug`.
649
+ let url = HttpUri::new(url);
554
650
  debug!(
555
- "hf request: service_operation={service_operation} operation={op} method={method} url={url}"
651
+ "hf request: service_operation={service_operation} operation={op} method={method} url={}",
652
+ url.redacted_uri()
556
653
  );
557
654
  let mut req = Request::builder()
558
655
  .method(method)
559
- .uri(url)
656
+ .uri(url.original_uri())
560
657
  .extension(op)
561
658
  .extension(ServiceOperation(service_operation));
562
659
  match &self.token {
@@ -576,58 +673,63 @@ impl HfCore {
576
673
  Ok(req)
577
674
  }
578
675
 
579
- /// Return the repo handle with the server's canonical repo id, resolving
580
- /// and caching it on first use.
676
+ /// Send `req`, following one same-endpoint `307`/`308` redirect.
581
677
  ///
582
- /// HF resolves repo ids case-insensitively but replies 307 to any request
583
- /// for a non-canonically-cased id (e.g. `user/repo` for `user/Repo`).
584
- /// The transport cannot replay bodied requests (commit, paths-info)
585
- /// through a redirect. Building every URL from the canonical id avoids
586
- /// depending on redirect behavior entirely, removing the inconsistency across
587
- /// operations.
588
- pub(super) async fn canonical_repo(&self, ctx: &OperationContext) -> Result<HfRepo> {
589
- // Buckets are addressed by opaque ids
590
- if self.repo.is_bucket() {
591
- return Ok(self.repo.clone());
592
- }
593
-
594
- self.canonical_repo
595
- .get_or_try_init(|| async {
596
- let url = format!(
597
- "{}/api/{}/{}",
598
- self.endpoint,
599
- self.repo.repo_type.as_plural_str(),
600
- self.repo.repo_id,
601
- );
602
- let req = self
603
- .request(http::Method::GET, &url, Operation::Stat, "RepoInfo")?
604
- .body(Buffer::new())
605
- .map_err(new_request_build_error)?;
606
- let resp = ctx.http_transport().fetch(req).await?;
607
- if !resp.status().is_success() {
608
- let (parts, _) = resp.into_parts();
609
- return Err(parse_error(
610
- ErrorContext::new(ServiceOperation("RepoInfo")),
611
- parts,
612
- ));
613
- }
614
- let (_, mut body) = resp.into_parts();
615
- let buffer = body.to_buffer().await?;
616
- let info: RepoInfoResponse =
617
- serde_json::from_reader(buffer.reader()).map_err(new_json_deserialize_error)?;
618
-
619
- let mut repo = self.repo.clone();
620
- repo.repo_id = info.id;
621
- Ok(repo)
622
- })
623
- .await
624
- .cloned()
625
- }
678
+ /// HF resolves repo ids case-insensitively but answers any request for
679
+ /// a non-canonically cased id (`user/repo` for `user/Repo`) with a 307
680
+ /// to the canonical URL. Transports follow that for `GET`s but hand a
681
+ /// bodied `POST` (commit, paths-info) back as the bare 307, which would
682
+ /// otherwise surface as an error while reads silently succeed.
683
+ /// Re-issuing the request here makes the outcome independent of the
684
+ /// transport's redirect policy. `send` returns a redirect to any other
685
+ /// host as-is, so it never sends the bearer token off the configured
686
+ /// endpoint itself; the caller then reports it via [`parse_error`].
687
+ pub(super) async fn send(
688
+ &self,
689
+ ctx: &OperationContext,
690
+ req: Request<Buffer>,
691
+ ) -> Result<Response<HttpBody>> {
692
+ let retry = req.clone();
693
+
694
+ // The response is confined to this block so it is gone before the
695
+ // retry runs: its body is never read, so the connection cannot be
696
+ // pooled and is better closed than held across another round trip.
697
+ let target = {
698
+ let resp = ctx.http_transport().fetch(req).await?;
699
+ if !matches!(
700
+ resp.status(),
701
+ StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT
702
+ ) {
703
+ return Ok(resp);
704
+ }
705
+
706
+ // Accept a path-only `Location`, which is what HF sends, or one
707
+ // already on the endpoint (a bare origin, normalized by
708
+ // `HfBuilder`). What remains after dropping the endpoint must be
709
+ // a single-slash path, which also refuses a protocol-relative
710
+ // `//host/...` naming another host.
711
+ let Some(location) = parse_location(resp.headers())? else {
712
+ return Ok(resp);
713
+ };
714
+ let path = location
715
+ .strip_prefix(self.endpoint.as_str())
716
+ .unwrap_or(location);
717
+ if !path.starts_with('/') || path.starts_with("//") {
718
+ return Ok(resp);
719
+ }
720
+ format!("{}{path}", self.endpoint)
721
+ };
626
722
 
627
- /// Build an [`HfUri`] for the given operator-relative path, using the
628
- /// canonical repo id.
629
- pub(super) async fn canonical_uri(&self, ctx: &OperationContext, path: &str) -> Result<HfUri> {
630
- Ok(self.canonical_repo(ctx).await?.uri(&self.root, path))
723
+ let target = HttpUri::new(target);
724
+ debug!("hf request redirected: url={}", target.redacted_uri());
725
+ let (mut parts, body) = retry.into_parts();
726
+ parts.uri = target
727
+ .original_uri()
728
+ .parse()
729
+ .map_err(new_http_uri_invalid_error)?;
730
+ ctx.http_transport()
731
+ .fetch(Request::from_parts(parts, body))
732
+ .await
631
733
  }
632
734
 
633
735
  /// Convert an operator-relative path to a repo-absolute path
@@ -639,7 +741,7 @@ impl HfCore {
639
741
  }
640
742
 
641
743
  pub(super) async fn path_info(&self, ctx: &OperationContext, path: &str) -> Result<PathInfo> {
642
- let uri = self.canonical_uri(ctx, path).await?;
744
+ let uri = self.repo.uri(&self.root, path);
643
745
  let url = uri.paths_info_url(&self.endpoint);
644
746
  let form_body = format!("paths={}&expand=True", percent_encode_path(&uri.path));
645
747
 
@@ -648,7 +750,7 @@ impl HfCore {
648
750
  .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
649
751
  .body(Buffer::from(Bytes::from(form_body)))
650
752
  .map_err(new_request_build_error)?;
651
- let resp = ctx.http_transport().fetch(req).await?;
753
+ let resp = self.send(ctx, req).await?;
652
754
  if !resp.status().is_success() {
653
755
  let (parts, _) = resp.into_parts();
654
756
  return Err(parse_error(
@@ -669,6 +771,125 @@ impl HfCore {
669
771
  Ok(files.remove(0))
670
772
  }
671
773
 
774
+ pub(super) async fn read(
775
+ &self,
776
+ ctx: &OperationContext,
777
+ path: &str,
778
+ range: BytesRange,
779
+ ) -> Result<HfReadResponse> {
780
+ let entry = if self.enable_resolve_cache {
781
+ self.resolved_files.entry(path)
782
+ } else {
783
+ None
784
+ };
785
+ let Some(entry) = entry else {
786
+ let resp = self.resolve(ctx, path, range, self.download_mode).await?;
787
+ return HfReadResponse::from_response(resp, self.download_mode).await;
788
+ };
789
+
790
+ let mut rejected: Option<Arc<HttpDownload>> = None;
791
+ loop {
792
+ let destination = {
793
+ let mut state = entry.lock().await;
794
+ match &*state {
795
+ ResolvedFile::Xet(info) => return Ok(HfReadResponse::Xet(info.clone())),
796
+ ResolvedFile::NotXet | ResolvedFile::Bypass => {
797
+ drop(state);
798
+ let resp = self.resolve(ctx, path, range, HfDownloadMode::Http).await?;
799
+ return Ok(HfReadResponse::Http(resp));
800
+ }
801
+ ResolvedFile::Http(current)
802
+ if current.valid_until > Instant::now()
803
+ && !rejected
804
+ .as_ref()
805
+ .is_some_and(|old| Arc::ptr_eq(old, current)) =>
806
+ {
807
+ current.clone()
808
+ }
809
+ _ => {
810
+ let mode = if matches!(&*state, ResolvedFile::Http(_)) {
811
+ HfDownloadMode::Http
812
+ } else {
813
+ self.download_mode
814
+ };
815
+ // Serialize resolution and refresh. XET metadata is
816
+ // independent of the range; a one-byte probe bounds the
817
+ // body discarded when the file turns out to use HTTP.
818
+ *state = ResolvedFile::Empty;
819
+ let resolve_range = if mode == HfDownloadMode::Xet {
820
+ BytesRange::new(0, Some(1))
821
+ } else {
822
+ range
823
+ };
824
+ let resp = self.resolve(ctx, path, resolve_range, mode).await?;
825
+ let resp = match HfReadResponse::from_response(resp, mode).await? {
826
+ HfReadResponse::Xet(info) => {
827
+ *state = ResolvedFile::Xet(info.clone());
828
+ return Ok(HfReadResponse::Xet(info));
829
+ }
830
+ HfReadResponse::Http(resp) => resp,
831
+ };
832
+ *state = match HttpDownload::from_response(&resp) {
833
+ Some(download) => ResolvedFile::Http(Arc::new(download)),
834
+ None if self.download_mode == HfDownloadMode::Xet => {
835
+ ResolvedFile::NotXet
836
+ }
837
+ None => {
838
+ // This live slot cannot have been evicted or
839
+ // replaced while we hold an Arc to it.
840
+ self.resolved_files
841
+ .entries
842
+ .lock()
843
+ .expect("resolved file cache lock poisoned")
844
+ .remove(path);
845
+ ResolvedFile::Bypass
846
+ }
847
+ };
848
+ if mode == HfDownloadMode::Http {
849
+ // The response already contains the requested range.
850
+ return Ok(HfReadResponse::Http(resp));
851
+ }
852
+ // The XET probe returned one byte of HTTP content.
853
+ // Reuse its destination to fetch the caller's range.
854
+ continue;
855
+ }
856
+ }
857
+ };
858
+
859
+ let url = self
860
+ .repo
861
+ .uri(&self.root, path)
862
+ .resolve_url(&self.endpoint, self.repo.revision());
863
+ let mut req = self
864
+ .request(http::Method::GET, &url, Operation::Read, "Download")?
865
+ .extension(destination.redirect.clone());
866
+ if !range.is_full() {
867
+ req = req.header(header::RANGE, range.to_header());
868
+ }
869
+ let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
870
+ let resp = ctx.http_transport().fetch(req).await?;
871
+ if resp.status().is_success() {
872
+ return Ok(HfReadResponse::Http(resp));
873
+ }
874
+ if rejected.is_none()
875
+ && matches!(
876
+ resp.status(),
877
+ StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
878
+ )
879
+ {
880
+ // A late rejection must not discard another caller's refreshed
881
+ // destination. The next iteration compares the Arc generation.
882
+ rejected = Some(destination);
883
+ continue;
884
+ }
885
+ let (parts, _) = resp.into_parts();
886
+ return Err(parse_error(
887
+ ErrorContext::new(ServiceOperation("Download")),
888
+ parts,
889
+ ));
890
+ }
891
+ }
892
+
672
893
  /// Send `GET /resolve` and return the raw streaming response.
673
894
  ///
674
895
  /// In `Xet` mode adds `Accept: application/vnd.xet-fileinfo+json` so the
@@ -683,7 +904,7 @@ impl HfCore {
683
904
  range: BytesRange,
684
905
  mode: HfDownloadMode,
685
906
  ) -> Result<Response<HttpBody>> {
686
- let uri = self.canonical_uri(ctx, path).await?;
907
+ let uri = self.repo.uri(&self.root, path);
687
908
  let url = uri.resolve_url(&self.endpoint, self.repo.revision());
688
909
 
689
910
  let mut req = self.request(http::Method::GET, &url, Operation::Read, "Resolve")?;
@@ -697,7 +918,7 @@ impl HfCore {
697
918
  }
698
919
 
699
920
  let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
700
- let resp = ctx.http_transport().fetch(req).await?;
921
+ let resp = self.send(ctx, req).await?;
701
922
 
702
923
  if !resp.status().is_success() {
703
924
  let status = resp.status();
@@ -726,10 +947,7 @@ impl HfCore {
726
947
  deleted_files: Vec<DeletedFile>,
727
948
  deleted_folders: Vec<DeletedFolder>,
728
949
  ) -> Result<CommitResponse> {
729
- let url = self
730
- .canonical_repo(ctx)
731
- .await?
732
- .git_commit_url(&self.endpoint);
950
+ let url = self.repo.git_commit_url(&self.endpoint);
733
951
 
734
952
  let payload = MixedCommitPayload {
735
953
  summary: "Commit via OpenDAL".to_string(),
@@ -748,7 +966,7 @@ impl HfCore {
748
966
  .body(Buffer::from(json_body))
749
967
  .map_err(new_request_build_error)?;
750
968
 
751
- let resp = ctx.http_transport().fetch(req).await?;
969
+ let resp = self.send(ctx, req).await?;
752
970
  if !resp.status().is_success() {
753
971
  let (parts, _) = resp.into_parts();
754
972
  return Err(parse_error(
@@ -792,7 +1010,7 @@ impl HfCore {
792
1010
  .body(Buffer::from(Bytes::from(body)))
793
1011
  .map_err(new_request_build_error)?;
794
1012
 
795
- let resp = ctx.http_transport().fetch(req).await?;
1013
+ let resp = self.send(ctx, req).await?;
796
1014
  if !resp.status().is_success() {
797
1015
  let (parts, _) = resp.into_parts();
798
1016
  return Err(parse_error(
@@ -806,7 +1024,7 @@ impl HfCore {
806
1024
 
807
1025
  #[cfg(test)]
808
1026
  pub(crate) mod test_utils {
809
- use http::{Request, Response, StatusCode};
1027
+ use http::{Request, Response};
810
1028
  use std::sync::{Arc, Mutex};
811
1029
 
812
1030
  use super::super::core::HfRepoType;
@@ -826,10 +1044,8 @@ pub(crate) mod test_utils {
826
1044
  /// mocked [`XetFileResponse`] body instead of plain bytes, so tests
827
1045
  /// can exercise the XET classification path without real network.
828
1046
  xet_file: Arc<Mutex<Option<XetFileResponse>>>,
829
- /// `Range` header of the most recent XET-classifying `/resolve/`
830
- /// request (one that hit the `xet_file` branch above), so tests can
831
- /// tell which racing caller's range a single-flighted classification
832
- /// actually sent.
1047
+ /// `Range` header of the most recent XET metadata probe, so tests can
1048
+ /// check it sends a fixed single byte rather than the caller's range.
833
1049
  classify_range_header: Arc<Mutex<Option<String>>>,
834
1050
  }
835
1051
 
@@ -880,9 +1096,6 @@ pub(crate) mod test_utils {
880
1096
  *self.request_count.lock().unwrap()
881
1097
  }
882
1098
 
883
- /// `Range` header of the most recent XET-classifying `/resolve/`
884
- /// request, or `None` if that range was full (no `Range` header) or
885
- /// no such request has happened yet.
886
1099
  pub(crate) fn get_captured_classify_range_header(&self) -> Option<String> {
887
1100
  self.classify_range_header.lock().unwrap().clone()
888
1101
  }
@@ -973,16 +1186,6 @@ pub(crate) mod test_utils {
973
1186
  HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
974
1187
  size,
975
1188
  )
976
- } else if let Some(rest) = req.uri().path().strip_prefix("/api/") {
977
- // Repo-info: echo the requested id back as the canonical id.
978
- let id = rest.split_once('/').map(|(_, id)| id).unwrap_or(rest);
979
- let data = Bytes::from(format!(r#"{{"id":"{id}"}}"#));
980
- let size = data.len() as u64;
981
- let buffer = Buffer::from(data);
982
- (
983
- HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
984
- size,
985
- )
986
1189
  } else {
987
1190
  let data = Bytes::from_static(b"hello");
988
1191
  let size = data.len() as u64;
@@ -1007,8 +1210,25 @@ pub(crate) mod test_utils {
1007
1210
  revision: &str,
1008
1211
  endpoint: &str,
1009
1212
  ) -> (HfCore, OperationContext, MockHttpTransport) {
1010
- let mock_client = MockHttpTransport::new();
1011
- let http_transport = HttpTransporter::new(mock_client.clone());
1213
+ create_test_core_with(
1214
+ MockHttpTransport::new(),
1215
+ repo_type,
1216
+ repo_id,
1217
+ revision,
1218
+ endpoint,
1219
+ )
1220
+ }
1221
+
1222
+ /// Like [`create_test_core`] but with a caller-supplied transport, for
1223
+ /// tests that need to script responses the shared mock doesn't.
1224
+ pub(crate) fn create_test_core_with<T: HttpTransport + Clone>(
1225
+ transport: T,
1226
+ repo_type: HfRepoType,
1227
+ repo_id: &str,
1228
+ revision: &str,
1229
+ endpoint: &str,
1230
+ ) -> (HfCore, OperationContext, T) {
1231
+ let http_transport = HttpTransporter::new(transport.clone());
1012
1232
  let ctx = OperationContext::from_parts(http_transport, Executor::default());
1013
1233
 
1014
1234
  let info = ServiceInfo::new("hf", "", "");
@@ -1028,7 +1248,7 @@ pub(crate) mod test_utils {
1028
1248
  HfDownloadMode::Xet,
1029
1249
  );
1030
1250
 
1031
- (core, ctx, mock_client)
1251
+ (core, ctx, transport)
1032
1252
  }
1033
1253
  }
1034
1254
 
@@ -1037,10 +1257,9 @@ mod tests {
1037
1257
  use std::sync::{Arc, Mutex};
1038
1258
 
1039
1259
  use http::Response;
1040
- use http::StatusCode;
1041
1260
 
1042
1261
  use super::super::core::HfRepoType;
1043
- use super::test_utils::create_test_core;
1262
+ use super::test_utils::{create_test_core, create_test_core_with};
1044
1263
  use super::*;
1045
1264
 
1046
1265
  #[tokio::test]
@@ -1123,15 +1342,26 @@ mod tests {
1123
1342
  Ok(())
1124
1343
  }
1125
1344
 
1126
- /// A scripted transport mirroring how HF serves a repo whose configured id
1127
- /// is not canonically cased: repo-info returns the canonical id, and only
1128
- /// the canonical commit URL accepts the commit.
1129
- #[derive(Clone, Default)]
1130
- struct CanonicalCaseTransport {
1131
- requests: Arc<Mutex<Vec<(String, String, String)>>>,
1345
+ struct SeenRequest {
1346
+ method: String,
1347
+ uri: String,
1348
+ auth: String,
1349
+ body: String,
1132
1350
  }
1133
1351
 
1134
- impl HttpTransport for CanonicalCaseTransport {
1352
+ /// A scripted transport mirroring how HF serves a repo whose configured
1353
+ /// id is not canonically cased: it answers that URL with `status` and a
1354
+ /// `Location` of `location`, and only the canonically cased URL accepts
1355
+ /// the commit. Tests set both, so one transport covers
1356
+ /// a followed redirect, a refused one, and a non-307 status.
1357
+ #[derive(Clone)]
1358
+ struct RedirectingTransport {
1359
+ status: StatusCode,
1360
+ location: &'static str,
1361
+ requests: Arc<Mutex<Vec<SeenRequest>>>,
1362
+ }
1363
+
1364
+ impl HttpTransport for RedirectingTransport {
1135
1365
  async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
1136
1366
  let uri = req.uri().to_string();
1137
1367
  let auth = req
@@ -1140,91 +1370,200 @@ mod tests {
1140
1370
  .and_then(|v| v.to_str().ok())
1141
1371
  .unwrap_or_default()
1142
1372
  .to_string();
1143
- self.requests
1144
- .lock()
1145
- .unwrap()
1146
- .push((req.method().to_string(), uri.clone(), auth));
1147
-
1148
- let body = match uri.as_str() {
1149
- "https://huggingface.co/api/models/test-user/uppercase-repo" => {
1150
- Bytes::from_static(br#"{"id":"test-user/Uppercase-Repo"}"#)
1373
+ let body = String::from_utf8(req.body().to_bytes().to_vec()).unwrap();
1374
+ self.requests.lock().unwrap().push(SeenRequest {
1375
+ method: req.method().to_string(),
1376
+ uri: uri.clone(),
1377
+ auth,
1378
+ body,
1379
+ });
1380
+
1381
+ let resp = match uri.as_str() {
1382
+ "https://huggingface.co/api/models/test-user/uppercase-repo/commit/main" => {
1383
+ Response::builder()
1384
+ .status(self.status)
1385
+ .header(header::LOCATION, self.location)
1386
+ .header(header::CONTENT_LENGTH, 0)
1387
+ .body(HttpBody::new(futures::stream::empty(), Some(0)))
1151
1388
  }
1152
1389
  "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main" => {
1153
- Bytes::from_static(b"{}")
1390
+ Response::builder()
1391
+ .status(StatusCode::OK)
1392
+ .header(header::CONTENT_LENGTH, 2)
1393
+ .body(HttpBody::new(
1394
+ futures::stream::iter(vec![Ok(Buffer::from(Bytes::from_static(
1395
+ b"{}",
1396
+ )))]),
1397
+ Some(2),
1398
+ ))
1154
1399
  }
1155
1400
  other => panic!("unexpected request to {other}"),
1156
1401
  };
1157
- let len = body.len() as u64;
1158
- Ok(Response::builder()
1159
- .status(http::StatusCode::OK)
1160
- .header(header::CONTENT_LENGTH, len)
1161
- .body(HttpBody::new(
1162
- futures::stream::iter(vec![Ok(Buffer::from(body))]),
1163
- Some(len),
1164
- ))
1165
- .unwrap())
1402
+ Ok(resp.unwrap())
1166
1403
  }
1167
1404
  }
1168
1405
 
1169
- #[tokio::test]
1170
- async fn test_commit_uses_canonical_repo_id() -> Result<()> {
1171
- let transport = CanonicalCaseTransport::default();
1172
- let ctx = OperationContext::from_parts(
1173
- HttpTransporter::new(transport.clone()),
1174
- Executor::default(),
1175
- );
1176
-
1177
- let xet_session = XetSessionBuilder::new()
1178
- .build()
1179
- .expect("failed to create xet session");
1180
- let core = HfCore::new(
1181
- ServiceInfo::new("hf", "", ""),
1182
- Capability::default(),
1183
- HfRepo::new(
1184
- HfRepoType::Model,
1185
- "test-user/uppercase-repo".to_string(),
1186
- Some("main".to_string()),
1187
- ),
1188
- "/".to_string(),
1189
- Some("hf_dummy".to_string()),
1190
- "https://huggingface.co".to_string(),
1191
- xet_session,
1192
- HfDownloadMode::Xet,
1406
+ fn redirecting_core(
1407
+ status: StatusCode,
1408
+ location: &'static str,
1409
+ ) -> (HfCore, OperationContext, RedirectingTransport) {
1410
+ let (mut core, ctx, transport) = create_test_core_with(
1411
+ RedirectingTransport {
1412
+ status,
1413
+ location,
1414
+ requests: Arc::default(),
1415
+ },
1416
+ HfRepoType::Model,
1417
+ "test-user/uppercase-repo",
1418
+ "main",
1419
+ "https://huggingface.co",
1193
1420
  );
1421
+ core.token = Some("hf_dummy".to_string());
1422
+ (core, ctx, transport)
1423
+ }
1194
1424
 
1195
- let lfs_file = |path: &str| LfsFile {
1425
+ fn lfs_file(path: &str) -> LfsFile {
1426
+ LfsFile {
1196
1427
  path: path.to_string(),
1197
1428
  oid: "deadbeef".to_string(),
1198
1429
  algo: "sha256".to_string(),
1199
1430
  size: 2812,
1200
- };
1431
+ }
1432
+ }
1433
+
1434
+ /// `send` re-issues a bodied `POST` answered with a 307 to the
1435
+ /// canonically cased URL with the same method, headers and body,
1436
+ /// regardless of whether the transport replays redirects itself. HF
1437
+ /// sends the `Location` as an absolute path, as in the report for #8107.
1438
+ #[tokio::test]
1439
+ async fn test_commit_follows_case_redirect() -> Result<()> {
1440
+ let (core, ctx, transport) = redirecting_core(
1441
+ StatusCode::TEMPORARY_REDIRECT,
1442
+ "/api/models/test-user/Uppercase-Repo/commit/main",
1443
+ );
1444
+
1201
1445
  core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1202
1446
  .await?;
1203
- core.commit_git(&ctx, vec![], vec![lfs_file("b.md")], vec![], vec![])
1204
- .await?;
1205
1447
 
1206
1448
  let requests = transport.requests.lock().unwrap();
1207
- let expected = [
1208
- (
1209
- "GET",
1210
- "https://huggingface.co/api/models/test-user/uppercase-repo",
1211
- ),
1212
- (
1213
- "POST",
1214
- "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
1215
- ),
1216
- // The canonical id is cached: no second repo-info request.
1217
- (
1218
- "POST",
1219
- "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
1449
+ assert_eq!(requests.len(), 2);
1450
+ let (first, redirected) = (&requests[0], &requests[1]);
1451
+ assert_eq!(first.method, "POST");
1452
+ assert_eq!(
1453
+ first.uri,
1454
+ "https://huggingface.co/api/models/test-user/uppercase-repo/commit/main"
1455
+ );
1456
+ assert_eq!(first.auth, "Bearer hf_dummy");
1457
+ assert_eq!(redirected.method, "POST");
1458
+ assert_eq!(
1459
+ redirected.uri,
1460
+ "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main"
1461
+ );
1462
+ assert_eq!(redirected.auth, "Bearer hf_dummy");
1463
+ assert_eq!(redirected.body, first.body);
1464
+ assert!(first.body.contains("a.md"));
1465
+
1466
+ Ok(())
1467
+ }
1468
+
1469
+ /// `send` also follows an absolute `Location` on the configured endpoint,
1470
+ /// and a 308 like a 307.
1471
+ #[tokio::test]
1472
+ async fn test_commit_follows_absolute_same_endpoint_redirect() -> Result<()> {
1473
+ let (core, ctx, transport) = redirecting_core(
1474
+ StatusCode::PERMANENT_REDIRECT,
1475
+ "https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
1476
+ );
1477
+
1478
+ core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1479
+ .await?;
1480
+
1481
+ assert_eq!(transport.requests.lock().unwrap().len(), 2);
1482
+ Ok(())
1483
+ }
1484
+
1485
+ /// `send` must not follow a redirect off the configured endpoint: the
1486
+ /// re-issued request would carry the bearer token to a foreign host.
1487
+ /// The 307 surfaces as an error instead.
1488
+ #[tokio::test]
1489
+ async fn test_commit_does_not_follow_foreign_redirect() -> Result<()> {
1490
+ let (core, ctx, transport) = redirecting_core(
1491
+ StatusCode::TEMPORARY_REDIRECT,
1492
+ "https://evil.example.com/api/models/test-user/Uppercase-Repo/commit/main",
1493
+ );
1494
+
1495
+ let err = core
1496
+ .commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1497
+ .await
1498
+ .expect_err("a foreign redirect must not be followed");
1499
+ assert_eq!(err.kind(), ErrorKind::Unexpected);
1500
+ assert!(
1501
+ err.to_string()
1502
+ .contains("redirect to https://evil.example.com/api/models/test-user/Uppercase-Repo/commit/main not followed"),
1503
+ "the refused redirect must be named in the error: {err}"
1504
+ );
1505
+ assert_eq!(transport.requests.lock().unwrap().len(), 1);
1506
+
1507
+ Ok(())
1508
+ }
1509
+
1510
+ /// A protocol-relative `Location` names another host, so it is refused
1511
+ /// rather than treated as a path on ours.
1512
+ #[tokio::test]
1513
+ async fn test_commit_does_not_follow_protocol_relative_redirect() -> Result<()> {
1514
+ let (core, ctx, transport) =
1515
+ redirecting_core(StatusCode::TEMPORARY_REDIRECT, "//evil.example.com/api/x");
1516
+
1517
+ let err = core
1518
+ .commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1519
+ .await
1520
+ .expect_err("a protocol-relative redirect must not be followed");
1521
+ assert!(err.to_string().contains("not followed"), "{err}");
1522
+ assert_eq!(transport.requests.lock().unwrap().len(), 1);
1523
+
1524
+ Ok(())
1525
+ }
1526
+
1527
+ /// A `Location` without a leading slash is a relative reference this
1528
+ /// service does not resolve; it is refused rather than guessed at.
1529
+ #[tokio::test]
1530
+ async fn test_commit_does_not_follow_relative_redirect() -> Result<()> {
1531
+ let (core, ctx, transport) = redirecting_core(
1532
+ StatusCode::TEMPORARY_REDIRECT,
1533
+ "api/models/test-user/Uppercase-Repo/commit/main",
1534
+ );
1535
+
1536
+ let err = core
1537
+ .commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1538
+ .await
1539
+ .expect_err("a relative redirect must not be followed");
1540
+ assert!(err.to_string().contains("not followed"), "{err}");
1541
+ assert_eq!(transport.requests.lock().unwrap().len(), 1);
1542
+
1543
+ Ok(())
1544
+ }
1545
+
1546
+ /// `send` only replays 307/308. A 302 is passed through untouched: in
1547
+ /// http download mode the transport follows the resolve 302 to the CDN
1548
+ /// itself, and `send` must not re-issue it on the API host.
1549
+ #[tokio::test]
1550
+ async fn test_send_leaves_302_to_the_transport() -> Result<()> {
1551
+ let (core, ctx, transport) = redirecting_core(
1552
+ StatusCode::FOUND,
1553
+ "/api/models/test-user/Uppercase-Repo/commit/main",
1554
+ );
1555
+
1556
+ let err = core
1557
+ .commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
1558
+ .await
1559
+ .expect_err("an unfollowed 302 must surface as an error");
1560
+ assert!(
1561
+ err.to_string().contains(
1562
+ "redirect to /api/models/test-user/Uppercase-Repo/commit/main not followed"
1220
1563
  ),
1221
- ];
1222
- assert_eq!(requests.len(), expected.len());
1223
- for ((method, uri, auth), (exp_method, exp_uri)) in requests.iter().zip(expected) {
1224
- assert_eq!(method, exp_method);
1225
- assert_eq!(uri, exp_uri);
1226
- assert_eq!(auth, "Bearer hf_dummy");
1227
- }
1564
+ "{err}"
1565
+ );
1566
+ assert_eq!(transport.requests.lock().unwrap().len(), 1);
1228
1567
 
1229
1568
  Ok(())
1230
1569
  }
@@ -1239,22 +1578,15 @@ mod tests {
1239
1578
  );
1240
1579
  mock_client.set_xet_token_expires_at(u64::MAX);
1241
1580
 
1242
- // 2 requests: one to resolve the canonical repo id, one for the
1243
- // token itself.
1244
- let repo = core.canonical_repo(&ctx).await?;
1245
- let first = core
1246
- .cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1247
- .await?;
1248
- assert_eq!(mock_client.request_count(), 2);
1581
+ let first = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1582
+ assert_eq!(mock_client.request_count(), 1);
1249
1583
  assert_eq!(first.access_token, "mock-token");
1250
1584
  assert_eq!(first.cas_url, "https://cas.example.com");
1251
1585
 
1252
1586
  // A second call with the cached token nowhere near expiry must not
1253
- // hit the network again -- the canonical repo id is cached too.
1254
- let second = core
1255
- .cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1256
- .await?;
1257
- assert_eq!(mock_client.request_count(), 2);
1587
+ // hit the network again.
1588
+ let second = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1589
+ assert_eq!(mock_client.request_count(), 1);
1258
1590
  assert_eq!(second.access_token, first.access_token);
1259
1591
 
1260
1592
  Ok(())
@@ -1271,19 +1603,13 @@ mod tests {
1271
1603
  // Already within the refresh buffer of "now" -- immediately stale.
1272
1604
  mock_client.set_xet_token_expires_at(0);
1273
1605
 
1274
- // 2 requests: one to resolve the canonical repo id, one for the
1275
- // token itself.
1276
- let repo = core.canonical_repo(&ctx).await?;
1277
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1278
- .await?;
1279
- assert_eq!(mock_client.request_count(), 2);
1606
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1607
+ assert_eq!(mock_client.request_count(), 1);
1280
1608
 
1281
- // The canonical repo id is cached, so only the token is re-fetched.
1282
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1283
- .await?;
1609
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1284
1610
  assert_eq!(
1285
1611
  mock_client.request_count(),
1286
- 3,
1612
+ 2,
1287
1613
  "a token that's always stale must be refreshed on every call, not cached"
1288
1614
  );
1289
1615
 
@@ -1304,21 +1630,15 @@ mod tests {
1304
1630
  mock_client.set_xet_token_expires_at(u64::MAX);
1305
1631
  mock_client.fail_next_requests(1);
1306
1632
 
1307
- // The injected failure hits the canonical repo id lookup, the first
1308
- // request resolving it makes.
1309
- match core.canonical_repo(&ctx).await {
1633
+ match core.cached_xet_token(&ctx, XetTokenScope::Read).await {
1310
1634
  Err(err) => assert!(err.to_string().contains("mock injected failure")),
1311
- Ok(_) => panic!("a failed canonical repo lookup must surface as an error"),
1635
+ Ok(_) => panic!("a failed token fetch must surface as an error"),
1312
1636
  }
1313
1637
  assert_eq!(mock_client.request_count(), 1);
1314
1638
 
1315
- // Retry: canonical repo id lookup succeeds, then the token itself.
1316
- let repo = core.canonical_repo(&ctx).await?;
1317
- let token = core
1318
- .cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1319
- .await?;
1639
+ let token = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1320
1640
  assert_eq!(token.access_token, "mock-token");
1321
- assert_eq!(mock_client.request_count(), 3);
1641
+ assert_eq!(mock_client.request_count(), 2);
1322
1642
 
1323
1643
  Ok(())
1324
1644
  }
@@ -1331,32 +1651,24 @@ mod tests {
1331
1651
  "main",
1332
1652
  "https://huggingface.co",
1333
1653
  );
1334
- // A write-scope token fetch now goes through `request()` tagged as
1335
- // `Operation::Write`, which requires a token to be configured.
1654
+ // Write-scope token fetches are tagged `Operation::Write`, which
1655
+ // requires a token.
1336
1656
  core.token = Some("hf_dummy".to_string());
1337
1657
  mock_client.set_xet_token_expires_at(u64::MAX);
1338
1658
 
1339
- // 2 requests: one to resolve the canonical repo id, one for the
1340
- // read token itself.
1341
- let repo = core.canonical_repo(&ctx).await?;
1342
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1343
- .await?;
1344
- assert_eq!(mock_client.request_count(), 2);
1659
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1660
+ assert_eq!(mock_client.request_count(), 1);
1345
1661
 
1346
1662
  // A write token request must not be satisfied by the read token's
1347
- // cache slot -- read and write are distinct HF API scopes. The
1348
- // canonical repo id is already cached, so only the token is fetched.
1349
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Write)
1350
- .await?;
1351
- assert_eq!(mock_client.request_count(), 3);
1663
+ // cache slot -- read and write are distinct HF API scopes.
1664
+ core.cached_xet_token(&ctx, XetTokenScope::Write).await?;
1665
+ assert_eq!(mock_client.request_count(), 2);
1352
1666
  assert!(mock_client.get_captured_url().contains("write"));
1353
1667
 
1354
1668
  // Both are now warm; neither call should hit the network again.
1355
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1356
- .await?;
1357
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Write)
1358
- .await?;
1359
- assert_eq!(mock_client.request_count(), 3);
1669
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1670
+ core.cached_xet_token(&ctx, XetTokenScope::Write).await?;
1671
+ assert_eq!(mock_client.request_count(), 2);
1360
1672
 
1361
1673
  Ok(())
1362
1674
  }
@@ -1375,15 +1687,13 @@ mod tests {
1375
1687
  );
1376
1688
  mock_client.set_xet_token_expires_at(u64::MAX);
1377
1689
 
1378
- // 2 requests: one to resolve the canonical repo id, one for the
1379
- // read token itself.
1380
1690
  core.xet_download_group(&ctx).await?;
1381
- assert_eq!(mock_client.request_count(), 2);
1691
+ assert_eq!(mock_client.request_count(), 1);
1382
1692
 
1383
- // A second group build must reuse the cached read token (and
1384
- // canonical repo id) rather than fetching its own.
1693
+ // A second group build must reuse the cached read token rather than
1694
+ // fetching its own.
1385
1695
  core.xet_download_group(&ctx).await?;
1386
- assert_eq!(mock_client.request_count(), 2);
1696
+ assert_eq!(mock_client.request_count(), 1);
1387
1697
 
1388
1698
  Ok(())
1389
1699
  }
@@ -1399,23 +1709,20 @@ mod tests {
1399
1709
  core.token = Some("hf_dummy".to_string());
1400
1710
  mock_client.set_xet_token_expires_at(u64::MAX);
1401
1711
 
1402
- // 2 requests: one to resolve the canonical repo id, one for the
1403
- // write token itself.
1404
1712
  core.xet_upload_commit(&ctx).await?;
1405
- assert_eq!(mock_client.request_count(), 2);
1713
+ assert_eq!(mock_client.request_count(), 1);
1406
1714
  assert!(mock_client.get_captured_url().contains("write"));
1407
1715
 
1408
- // A second commit build must reuse the cached write token (and
1409
- // canonical repo id) rather than fetching its own.
1716
+ // A second commit build must reuse the cached write token rather
1717
+ // than fetching its own.
1410
1718
  core.xet_upload_commit(&ctx).await?;
1411
- assert_eq!(mock_client.request_count(), 2);
1719
+ assert_eq!(mock_client.request_count(), 1);
1412
1720
 
1413
1721
  Ok(())
1414
1722
  }
1415
1723
 
1416
- /// A write-token fetch must be tagged `Operation::Write`, not `Read`, so
1417
- /// a token-less writer hits `request()`'s local `PermissionDenied`
1418
- /// fast-path instead of firing a real unauthenticated network call.
1724
+ /// A write-token fetch must be tagged `Operation::Write` so a
1725
+ /// token-less writer is rejected locally instead of hitting the network.
1419
1726
  #[tokio::test]
1420
1727
  async fn test_xet_upload_commit_without_token_fails_locally() -> Result<()> {
1421
1728
  let (core, ctx, mock_client) = create_test_core(
@@ -1430,10 +1737,7 @@ mod tests {
1430
1737
  Err(err) => assert_eq!(err.kind(), ErrorKind::PermissionDenied),
1431
1738
  Ok(_) => panic!("an upload commit without a token must fail locally"),
1432
1739
  }
1433
- // The canonical repo id lookup doesn't require a token, but the
1434
- // write-token fetch itself must be rejected before it reaches the
1435
- // network.
1436
- assert_eq!(mock_client.request_count(), 1);
1740
+ assert_eq!(mock_client.request_count(), 0);
1437
1741
 
1438
1742
  Ok(())
1439
1743
  }
@@ -1456,31 +1760,25 @@ mod tests {
1456
1760
  .as_secs();
1457
1761
 
1458
1762
  // Exactly at the buffer boundary: `expires_at > now + BUFFER` is
1459
- // false, so this must be treated as stale. The first call also pays
1460
- // for the one-time canonical repo id lookup.
1763
+ // false, so this must be treated as stale.
1461
1764
  mock_client.set_xet_token_expires_at(now + XET_TOKEN_REFRESH_BUFFER_SECS);
1462
- let repo = core.canonical_repo(&ctx).await?;
1463
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1464
- .await?;
1465
- assert_eq!(mock_client.request_count(), 2);
1466
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1467
- .await?;
1765
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1766
+ assert_eq!(mock_client.request_count(), 1);
1767
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1468
1768
  assert_eq!(
1469
1769
  mock_client.request_count(),
1470
- 3,
1770
+ 2,
1471
1771
  "a token expiring exactly at the refresh buffer must be refetched, not reused"
1472
1772
  );
1473
1773
 
1474
1774
  // One second past the boundary: now fresh, so cached and reused.
1475
1775
  mock_client.set_xet_token_expires_at(now + XET_TOKEN_REFRESH_BUFFER_SECS + 1);
1476
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1477
- .await?;
1478
- assert_eq!(mock_client.request_count(), 4);
1479
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
1480
- .await?;
1776
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1777
+ assert_eq!(mock_client.request_count(), 3);
1778
+ core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
1481
1779
  assert_eq!(
1482
1780
  mock_client.request_count(),
1483
- 4,
1781
+ 3,
1484
1782
  "a token expiring past the refresh buffer must be cached and reused"
1485
1783
  );
1486
1784
 
@@ -1500,11 +1798,10 @@ mod tests {
1500
1798
  );
1501
1799
  mock_client.set_xet_token_expires_at(u64::MAX);
1502
1800
 
1503
- let repo = core.canonical_repo(&ctx).await?;
1504
1801
  let (r1, r2, r3) = futures::join!(
1505
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read),
1506
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read),
1507
- core.cached_xet_token(&ctx, &repo, XetTokenScope::Read),
1802
+ core.cached_xet_token(&ctx, XetTokenScope::Read),
1803
+ core.cached_xet_token(&ctx, XetTokenScope::Read),
1804
+ core.cached_xet_token(&ctx, XetTokenScope::Read),
1508
1805
  );
1509
1806
  r1?;
1510
1807
  r2?;
@@ -1512,9 +1809,8 @@ mod tests {
1512
1809
 
1513
1810
  assert_eq!(
1514
1811
  mock_client.request_count(),
1515
- 2,
1516
- "concurrent callers racing on a cold/stale token must share one refresh \
1517
- (plus the one-time canonical repo id lookup)"
1812
+ 1,
1813
+ "concurrent callers racing on a cold/stale token must share one refresh"
1518
1814
  );
1519
1815
 
1520
1816
  Ok(())
@@ -1554,8 +1850,6 @@ mod tests {
1554
1850
  }
1555
1851
  }
1556
1852
 
1557
- use http::StatusCode;
1558
-
1559
1853
  /// Context needed to classify an error from this service.
1560
1854
  #[derive(Clone, Copy, Debug)]
1561
1855
  pub(crate) struct ErrorContext {
@@ -1569,16 +1863,23 @@ impl ErrorContext {
1569
1863
  }
1570
1864
 
1571
1865
  /// Parse an error response using its service request context.
1572
- pub(crate) fn parse_error(ctx: ErrorContext, parts: http::response::Parts) -> Error {
1866
+ pub(crate) fn parse_error(ctx: ErrorContext, mut parts: http::response::Parts) -> Error {
1867
+ let location = HttpUri::from_response_location(&mut parts).cloned();
1573
1868
  // HF sets x-error-message on every error response with a short human-readable
1574
1869
  // description. Using the header avoids reading the response body, which can be
1575
1870
  // a large HTML error page (e.g. 52 KB on 404s from the /resolve/ endpoint).
1576
- let message = parts
1871
+ let error_message = parts
1577
1872
  .headers
1578
1873
  .get("x-error-message")
1579
- .and_then(|v| v.to_str().ok())
1580
- .unwrap_or("unknown error")
1581
- .to_string();
1874
+ .and_then(|v| v.to_str().ok());
1875
+ let message = match (error_message, location) {
1876
+ (Some(message), _) => message.to_string(),
1877
+ (None, Some(location)) if parts.status.is_redirection() => {
1878
+ let location = location.redacted_uri();
1879
+ format!("redirect to {location} not followed")
1880
+ }
1881
+ _ => "unknown error".to_string(),
1882
+ };
1582
1883
 
1583
1884
  // HF git-style commit APIs reject stale branch snapshots with 412.
1584
1885
  // Treat this specific conflict as temporary so RetryLayer can replay
@@ -1633,7 +1934,11 @@ mod uri {
1633
1934
  Self {
1634
1935
  repo_type,
1635
1936
  repo_id,
1636
- revision,
1937
+ // An empty revision is no revision: `hf://datasets/user/repo@`
1938
+ // and `revision=""` via options both reach here, and leaving
1939
+ // `Some("")` in place would build URLs with an empty revision
1940
+ // segment instead of falling back to `main`.
1941
+ revision: revision.filter(|revision| !revision.is_empty()),
1637
1942
  }
1638
1943
  }
1639
1944
 
@@ -1779,11 +2084,7 @@ mod uri {
1779
2084
  }
1780
2085
  } else if let Some((repo_id, rev)) = path.split_once('@') {
1781
2086
  let rev = rev.replace("%2F", "/");
1782
- (
1783
- repo_id.to_string(),
1784
- if rev.is_empty() { None } else { Some(rev) },
1785
- String::new(),
1786
- )
2087
+ (repo_id.to_string(), Some(rev), String::new())
1787
2088
  } else {
1788
2089
  (path, None, String::new())
1789
2090
  };
@@ -1872,21 +2173,27 @@ mod uri {
1872
2173
  recursive: bool,
1873
2174
  cursor: Option<&str>,
1874
2175
  ) -> String {
2176
+ // HF answers a trailing slash with a 302 to the slash-less URL, so
2177
+ // the separator lives in the segment rather than the template.
2178
+ let path_segment = if self.path.is_empty() {
2179
+ String::new()
2180
+ } else {
2181
+ format!("/{}", percent_encode_path(&self.path))
2182
+ };
2183
+
1875
2184
  let mut url = if self.repo.is_bucket() {
1876
2185
  format!(
1877
- "{}/api/buckets/{}/tree/{}?expand=True",
1878
- endpoint,
1879
- self.repo.repo_id,
1880
- percent_encode_path(&self.path),
2186
+ "{}/api/buckets/{}/tree{}?expand=True",
2187
+ endpoint, self.repo.repo_id, path_segment,
1881
2188
  )
1882
2189
  } else {
1883
2190
  format!(
1884
- "{}/api/{}/{}/tree/{}/{}?expand=True",
2191
+ "{}/api/{}/{}/tree/{}{}?expand=True",
1885
2192
  endpoint,
1886
2193
  self.repo.repo_type.as_plural_str(),
1887
2194
  self.repo.repo_id,
1888
2195
  percent_encode_revision(self.revision()),
1889
- percent_encode_path(&self.path),
2196
+ path_segment,
1890
2197
  )
1891
2198
  };
1892
2199
 
@@ -2156,6 +2463,76 @@ mod uri {
2156
2463
  let url = p.repo.bucket_batch_url("https://huggingface.co");
2157
2464
  assert_eq!(url, "https://huggingface.co/api/buckets/user/bucket/batch");
2158
2465
  }
2466
+
2467
+ #[test]
2468
+ fn test_file_tree_url_root_has_no_trailing_slash() {
2469
+ let p = resolve("datasets/user/repo");
2470
+ assert_eq!(
2471
+ p.file_tree_url("https://huggingface.co", false, None),
2472
+ "https://huggingface.co/api/datasets/user/repo/tree/main?expand=True"
2473
+ );
2474
+ }
2475
+
2476
+ #[test]
2477
+ fn test_file_tree_url_subdir_keeps_path_separator() {
2478
+ let p = resolve("datasets/user/repo/data");
2479
+ assert_eq!(
2480
+ p.file_tree_url("https://huggingface.co", false, None),
2481
+ "https://huggingface.co/api/datasets/user/repo/tree/main/data?expand=True"
2482
+ );
2483
+ }
2484
+
2485
+ #[test]
2486
+ fn test_file_tree_url_encoded_revision_root() {
2487
+ let p = resolve("datasets/user/repo@refs/convert/parquet");
2488
+ assert_eq!(
2489
+ p.file_tree_url("https://huggingface.co", false, None),
2490
+ "https://huggingface.co/api/datasets/user/repo/tree/refs%2Fconvert%2Fparquet?expand=True"
2491
+ );
2492
+ }
2493
+
2494
+ /// Every construction site funnels through `HfRepo::new`, so an empty
2495
+ /// revision from a URI or from `revision=""` in options is unset.
2496
+ #[test]
2497
+ fn test_empty_revision_is_unset() {
2498
+ assert!(resolve("datasets/user/repo@").repo.revision.is_none());
2499
+ assert!(
2500
+ HfRepo::new(
2501
+ HfRepoType::Dataset,
2502
+ "user/repo".to_string(),
2503
+ Some(String::new()),
2504
+ )
2505
+ .revision
2506
+ .is_none()
2507
+ );
2508
+ }
2509
+
2510
+ #[test]
2511
+ fn test_file_tree_url_recursive_and_cursor() {
2512
+ let p = resolve("datasets/user/repo");
2513
+ assert_eq!(
2514
+ p.file_tree_url("https://huggingface.co", true, Some("abc123")),
2515
+ "https://huggingface.co/api/datasets/user/repo/tree/main?expand=True&recursive=True&cursor=abc123"
2516
+ );
2517
+ }
2518
+
2519
+ #[test]
2520
+ fn test_bucket_file_tree_url_root_ends_at_tree() {
2521
+ let p = resolve("buckets/user/bucket");
2522
+ assert_eq!(
2523
+ p.file_tree_url("https://huggingface.co", false, None),
2524
+ "https://huggingface.co/api/buckets/user/bucket/tree?expand=True&recursive=false"
2525
+ );
2526
+ }
2527
+
2528
+ #[test]
2529
+ fn test_bucket_file_tree_url_subdir_keeps_path_separator() {
2530
+ let p = resolve("buckets/user/bucket/data");
2531
+ assert_eq!(
2532
+ p.file_tree_url("https://huggingface.co", false, None),
2533
+ "https://huggingface.co/api/buckets/user/bucket/tree/data?expand=True&recursive=false"
2534
+ );
2535
+ }
2159
2536
  }
2160
2537
  }
2161
2538