opendal 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +31 -36
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +25 -35
- data/README.md +1 -1
- data/core/CHANGELOG.md +22 -0
- data/core/Cargo.lock +145 -189
- data/core/Cargo.toml +95 -95
- data/core/DEPENDENCIES.rust.tsv +259 -290
- data/core/LICENSE +16 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/CHANGELOG.md +6031 -0
- data/core/core/Cargo.toml +1 -1
- data/core/core/LICENSE +16 -0
- data/core/core/src/docs/mod.rs +1 -1
- data/core/core/src/docs/upgrade.md +8 -0
- data/core/core/src/lib.rs +1 -3
- data/core/core/src/raw/futures_util.rs +26 -21
- data/core/core/src/raw/http_util/error.rs +9 -0
- data/core/core/src/raw/ops.rs +1 -0
- data/core/core/src/types/http_transport/mod.rs +30 -2
- data/core/core/src/types/http_transport/uri.rs +166 -0
- data/core/core/src/types/metadata.rs +1 -0
- data/core/core/src/types/mod.rs +2 -0
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/http-transports/reqwest/LICENSE +16 -0
- data/core/http-transports/reqwest/src/lib.rs +46 -4
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/LICENSE +16 -0
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/LICENSE +16 -0
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/LICENSE +16 -0
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/LICENSE +16 -0
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/LICENSE +16 -0
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/LICENSE +16 -0
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastmetrics/LICENSE +16 -0
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/LICENSE +16 -0
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/LICENSE +16 -0
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/LICENSE +16 -0
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/LICENSE +16 -0
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/LICENSE +16 -0
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/metrics/LICENSE +16 -0
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/LICENSE +16 -0
- data/core/layers/observe-metrics-common/Cargo.toml +1 -1
- data/core/layers/observe-metrics-common/LICENSE +16 -0
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/otelmetrics/LICENSE +16 -0
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/LICENSE +16 -0
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus/LICENSE +16 -0
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/prometheus-client/LICENSE +16 -0
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/LICENSE +16 -0
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/LICENSE +16 -0
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/LICENSE +16 -0
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/LICENSE +16 -0
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/LICENSE +16 -0
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/LICENSE +16 -0
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/LICENSE +16 -0
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/LICENSE +16 -0
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/LICENSE +16 -0
- data/core/services/azblob/src/backend.rs +21 -7
- data/core/services/azblob/src/config.rs +4 -1
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/LICENSE +16 -0
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/LICENSE +16 -0
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/azure-common/LICENSE +16 -0
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/LICENSE +16 -0
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/LICENSE +16 -0
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/LICENSE +16 -0
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/LICENSE +16 -0
- data/core/services/compfs/src/core.rs +27 -11
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/LICENSE +16 -0
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/LICENSE +16 -0
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/LICENSE +16 -0
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/LICENSE +16 -0
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/LICENSE +16 -0
- data/core/services/etcd/Cargo.toml +2 -2
- data/core/services/etcd/LICENSE +16 -0
- data/core/services/etcd/src/core.rs +3 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/LICENSE +16 -0
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/LICENSE +16 -0
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/LICENSE +16 -0
- data/core/services/fs/src/core.rs +69 -10
- data/core/services/ftp/Cargo.toml +2 -2
- data/core/services/ftp/LICENSE +16 -0
- data/core/services/ftp/src/core.rs +1 -1
- data/core/services/ftp/src/reader.rs +1 -1
- data/core/services/ftp/src/writer.rs +1 -1
- data/core/services/gcs/Cargo.toml +3 -2
- data/core/services/gcs/LICENSE +16 -0
- data/core/services/gcs/src/composer.rs +11 -21
- data/core/services/gcs/src/core.rs +116 -2
- data/core/services/gcs-grpc/Cargo.toml +1 -1
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +17 -18
- data/core/services/gdrive/Cargo.toml +3 -2
- data/core/services/gdrive/LICENSE +16 -0
- data/core/services/gdrive/src/docs.md +2 -0
- data/core/services/gdrive/src/path_index.rs +109 -18
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/LICENSE +16 -0
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/LICENSE +16 -0
- data/core/services/goosefs/Cargo.toml +13 -4
- data/core/services/goosefs/LICENSE +16 -0
- data/core/services/goosefs/README.md +11 -0
- data/core/services/goosefs/src/backend.rs +376 -49
- data/core/services/goosefs/src/config.rs +18 -10
- data/core/services/goosefs/src/core.rs +1 -1
- data/core/services/goosefs/src/docs.md +72 -3
- data/core/services/goosefs/src/reader.rs +22 -89
- data/core/services/goosefs/tests/master_addr_resolution.rs +103 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/LICENSE +16 -0
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/LICENSE +16 -0
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/LICENSE +16 -0
- data/core/services/hf/Cargo.toml +3 -3
- data/core/services/hf/LICENSE +16 -0
- data/core/services/hf/src/backend.rs +77 -9
- data/core/services/hf/src/config.rs +58 -0
- data/core/services/hf/src/core.rs +695 -318
- data/core/services/hf/src/docs.md +29 -0
- data/core/services/hf/src/lister.rs +3 -3
- data/core/services/hf/src/reader.rs +128 -143
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/LICENSE +16 -0
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/LICENSE +16 -0
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/LICENSE +16 -0
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/LICENSE +16 -0
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/LICENSE +16 -0
- data/core/services/memcached/Cargo.toml +2 -2
- data/core/services/memcached/LICENSE +16 -0
- data/core/services/memcached/src/core.rs +3 -3
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/LICENSE +16 -0
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/LICENSE +16 -0
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/LICENSE +16 -0
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/LICENSE +16 -0
- data/core/services/monoiofs/src/core.rs +27 -11
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/LICENSE +16 -0
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/LICENSE +16 -0
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/LICENSE +16 -0
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/LICENSE +16 -0
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/LICENSE +16 -0
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/LICENSE +16 -0
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/LICENSE +16 -0
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/LICENSE +16 -0
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/LICENSE +16 -0
- data/core/services/redis/Cargo.toml +2 -2
- data/core/services/redis/LICENSE +16 -0
- data/core/services/redis/src/core.rs +1 -1
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/LICENSE +16 -0
- data/core/services/s3/Cargo.toml +1 -1
- data/core/services/s3/LICENSE +16 -0
- data/core/services/seafile/Cargo.toml +4 -2
- data/core/services/seafile/LICENSE +16 -0
- data/core/services/seafile/src/backend.rs +2 -3
- data/core/services/seafile/src/core.rs +189 -86
- data/core/services/seafile/src/docs.md +2 -0
- data/core/services/sftp/Cargo.toml +2 -2
- data/core/services/sftp/LICENSE +16 -0
- data/core/services/sftp/src/core.rs +1 -1
- data/core/services/sftp/src/reader.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/LICENSE +16 -0
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/LICENSE +16 -0
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/LICENSE +16 -0
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/LICENSE +16 -0
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/LICENSE +16 -0
- data/core/services/tos/Cargo.toml +1 -1
- data/core/services/tos/LICENSE +16 -0
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/LICENSE +16 -0
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/LICENSE +16 -0
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/LICENSE +16 -0
- data/core/services/webdav/Cargo.toml +1 -2
- data/core/services/webdav/LICENSE +16 -0
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/LICENSE +16 -0
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/LICENSE +16 -0
- data/core/src/lib.rs +1 -3
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/LICENSE +16 -0
- data/core/upgrade.md +8 -0
- metadata +5 -2
|
@@ -16,17 +16,19 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use asyncband::mutex::Mutex;
|
|
19
|
-
use asyncband::once::OnceCell;
|
|
20
19
|
use bytes::Buf;
|
|
21
20
|
use bytes::Bytes;
|
|
22
21
|
use http::Request;
|
|
23
22
|
use http::Response;
|
|
23
|
+
use http::StatusCode;
|
|
24
24
|
use http::header;
|
|
25
25
|
use log::debug;
|
|
26
26
|
use serde::{Deserialize, Serialize};
|
|
27
|
+
use std::collections::HashMap;
|
|
27
28
|
use std::fmt::Debug;
|
|
28
29
|
use std::sync::Arc;
|
|
29
|
-
use std::
|
|
30
|
+
use std::sync::Mutex as StdMutex;
|
|
31
|
+
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|
30
32
|
|
|
31
33
|
use xet::xet_session::{XetDownloadStreamGroup, XetSession, XetSessionBuilder, XetUploadCommit};
|
|
32
34
|
|
|
@@ -165,11 +167,6 @@ pub(super) struct XetFileResponse {
|
|
|
165
167
|
pub size: u64,
|
|
166
168
|
}
|
|
167
169
|
|
|
168
|
-
#[derive(Deserialize)]
|
|
169
|
-
pub(super) struct RepoInfoResponse {
|
|
170
|
-
pub id: String,
|
|
171
|
-
}
|
|
172
|
-
|
|
173
170
|
/// Response shape of HF's `xet-{read,write}-token` endpoint. Matches
|
|
174
171
|
/// `CasJWTInfo` in the vendored `xet` crate (`xet_client::hub_client::types`)
|
|
175
172
|
/// — the same three fields regardless of read or write token.
|
|
@@ -215,10 +212,9 @@ impl XetTokenScope {
|
|
|
215
212
|
}
|
|
216
213
|
}
|
|
217
214
|
|
|
218
|
-
///
|
|
219
|
-
///
|
|
220
|
-
///
|
|
221
|
-
/// call.
|
|
215
|
+
/// `Operation` to tag this scope's token fetch with, so a token-less
|
|
216
|
+
/// writer is rejected locally by `request()` instead of hitting the
|
|
217
|
+
/// network.
|
|
222
218
|
fn operation(self) -> Operation {
|
|
223
219
|
match self {
|
|
224
220
|
Self::Read => Operation::Read,
|
|
@@ -301,6 +297,123 @@ pub(super) struct LastCommit {
|
|
|
301
297
|
pub date: String,
|
|
302
298
|
}
|
|
303
299
|
|
|
300
|
+
pub(super) enum HfReadResponse {
|
|
301
|
+
Http(Response<HttpBody>),
|
|
302
|
+
Xet(XetFileResponse),
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
impl HfReadResponse {
|
|
306
|
+
async fn from_response(resp: Response<HttpBody>, mode: HfDownloadMode) -> Result<Self> {
|
|
307
|
+
if mode != HfDownloadMode::Xet || !resp.headers().contains_key("x-xet-hash") {
|
|
308
|
+
return Ok(Self::Http(resp));
|
|
309
|
+
}
|
|
310
|
+
let (_, mut body) = resp.into_parts();
|
|
311
|
+
let buf = body.to_buffer().await?;
|
|
312
|
+
let info = serde_json::from_reader(buf.reader()).map_err(new_json_deserialize_error)?;
|
|
313
|
+
Ok(Self::Xet(info))
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Bound entries, paths, and HTTP URIs. Busy entries are never evicted:
|
|
318
|
+
// recreating their slots would allow two concurrent resolutions for one path.
|
|
319
|
+
const RESOLVED_FILE_MAX_ENTRIES: usize = 512;
|
|
320
|
+
const RESOLVED_FILE_MAX_PATH_BYTES: usize = 4096;
|
|
321
|
+
const HTTP_DOWNLOAD_MAX_URI_BYTES: usize = 8192;
|
|
322
|
+
const HTTP_DOWNLOAD_EXPIRY_MARGIN: Duration = Duration::from_secs(30);
|
|
323
|
+
|
|
324
|
+
#[derive(Default)]
|
|
325
|
+
struct ResolvedFiles {
|
|
326
|
+
entries: StdMutex<HashMap<String, ResolvedFileEntry>>,
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
struct ResolvedFileEntry {
|
|
330
|
+
accessed: Instant,
|
|
331
|
+
state: Arc<Mutex<ResolvedFile>>,
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
#[derive(Default)]
|
|
335
|
+
enum ResolvedFile {
|
|
336
|
+
#[default]
|
|
337
|
+
Empty,
|
|
338
|
+
Http(Arc<HttpDownload>),
|
|
339
|
+
Xet(XetFileResponse),
|
|
340
|
+
// The path is not XET-backed and has no reusable HTTP destination.
|
|
341
|
+
NotXet,
|
|
342
|
+
// Wake existing waiters onto the uncached path when a response cannot be
|
|
343
|
+
// admitted. Its slot leaves the table so later reads can try again.
|
|
344
|
+
Bypass,
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
struct HttpDownload {
|
|
348
|
+
redirect: HttpRedirect,
|
|
349
|
+
valid_until: Instant,
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
impl ResolvedFiles {
|
|
353
|
+
fn entry(&self, path: &str) -> Option<Arc<Mutex<ResolvedFile>>> {
|
|
354
|
+
if path.len() > RESOLVED_FILE_MAX_PATH_BYTES {
|
|
355
|
+
return None;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
let mut entries = self
|
|
359
|
+
.entries
|
|
360
|
+
.lock()
|
|
361
|
+
.expect("resolved file cache lock poisoned");
|
|
362
|
+
if entries.len() == RESOLVED_FILE_MAX_ENTRIES && !entries.contains_key(path) {
|
|
363
|
+
let oldest = entries
|
|
364
|
+
.iter()
|
|
365
|
+
.filter(|(_, entry)| Arc::strong_count(&entry.state) == 1)
|
|
366
|
+
.min_by_key(|(_, entry)| entry.accessed)
|
|
367
|
+
.map(|(path, _)| path.clone())?;
|
|
368
|
+
entries.remove(&oldest);
|
|
369
|
+
}
|
|
370
|
+
let entry = entries
|
|
371
|
+
.entry(path.to_string())
|
|
372
|
+
.or_insert_with(|| ResolvedFileEntry {
|
|
373
|
+
accessed: Instant::now(),
|
|
374
|
+
state: Arc::default(),
|
|
375
|
+
});
|
|
376
|
+
entry.accessed = Instant::now();
|
|
377
|
+
Some(entry.state.clone())
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
impl HttpDownload {
|
|
382
|
+
fn from_response(resp: &Response<HttpBody>) -> Option<Self> {
|
|
383
|
+
let redirect = resp.extensions().get::<HttpRedirect>()?;
|
|
384
|
+
let uri = redirect.uri().original_uri().parse::<http::Uri>().ok()?;
|
|
385
|
+
if uri.scheme_str() != Some("https")
|
|
386
|
+
|| uri.authority()?.as_str().contains('@')
|
|
387
|
+
|| redirect.uri().original_uri().len() > HTTP_DOWNLOAD_MAX_URI_BYTES
|
|
388
|
+
{
|
|
389
|
+
return None;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// HF's signed CDN destinations advertise their authorization deadline
|
|
393
|
+
// as Expires. Unknown signature formats fall back to fresh resolution.
|
|
394
|
+
let mut expirations = uri.query()?.split('&').filter_map(|pair| {
|
|
395
|
+
let (key, value) = pair.split_once('=')?;
|
|
396
|
+
(key == "Expires").then_some(value)
|
|
397
|
+
});
|
|
398
|
+
let expires = expirations.next()?.parse::<u64>().ok()?;
|
|
399
|
+
if expirations.next().is_some() {
|
|
400
|
+
return None;
|
|
401
|
+
}
|
|
402
|
+
let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?;
|
|
403
|
+
let lifetime = Duration::from_secs(expires)
|
|
404
|
+
.checked_sub(now)?
|
|
405
|
+
.checked_sub(HTTP_DOWNLOAD_EXPIRY_MARGIN)?;
|
|
406
|
+
if lifetime.is_zero() {
|
|
407
|
+
return None;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
Some(Self {
|
|
411
|
+
redirect: redirect.clone(),
|
|
412
|
+
valid_until: Instant::now().checked_add(lifetime)?,
|
|
413
|
+
})
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
304
417
|
// Core HuggingFace client that manages API interactions, authentication
|
|
305
418
|
// and shared logic for reader/writer/lister.
|
|
306
419
|
|
|
@@ -314,7 +427,9 @@ pub struct HfCore {
|
|
|
314
427
|
pub endpoint: String,
|
|
315
428
|
pub xet_session: XetSession,
|
|
316
429
|
pub download_mode: HfDownloadMode,
|
|
317
|
-
|
|
430
|
+
pub enable_resolve_cache: bool,
|
|
431
|
+
/// Shared file resolutions, keyed by paths relative to this core's root.
|
|
432
|
+
resolved_files: Arc<ResolvedFiles>,
|
|
318
433
|
/// Cached CAS read token, shared by every `XetDownloadStreamGroup` this
|
|
319
434
|
/// core creates, so at most one `xet-read-token` request happens per
|
|
320
435
|
/// token lifetime instead of one per group (one per file read).
|
|
@@ -355,7 +470,8 @@ impl HfCore {
|
|
|
355
470
|
endpoint,
|
|
356
471
|
xet_session,
|
|
357
472
|
download_mode,
|
|
358
|
-
|
|
473
|
+
enable_resolve_cache: false,
|
|
474
|
+
resolved_files: Arc::default(),
|
|
359
475
|
xet_read_token: Arc::new(Mutex::new(None)),
|
|
360
476
|
xet_write_token: Arc::new(Mutex::new(None)),
|
|
361
477
|
}
|
|
@@ -404,8 +520,7 @@ impl HfCore {
|
|
|
404
520
|
&self,
|
|
405
521
|
ctx: &OperationContext,
|
|
406
522
|
) -> Result<XetUploadCommit> {
|
|
407
|
-
let
|
|
408
|
-
self.xet_auth(ctx, XetTokenScope::Write).await?;
|
|
523
|
+
let token = self.cached_xet_token(ctx, XetTokenScope::Write).await?;
|
|
409
524
|
self.xet_session
|
|
410
525
|
.new_upload_commit()
|
|
411
526
|
.map_err(|err| {
|
|
@@ -414,7 +529,11 @@ impl HfCore {
|
|
|
414
529
|
})?
|
|
415
530
|
.with_endpoint(token.cas_url)
|
|
416
531
|
.with_token_info(token.access_token, token.expires_at)
|
|
417
|
-
.with_token_refresh_url(
|
|
532
|
+
.with_token_refresh_url(
|
|
533
|
+
self.repo
|
|
534
|
+
.xet_token_url(&self.endpoint, XetTokenScope::Write),
|
|
535
|
+
self.xet_token_refresh_headers(),
|
|
536
|
+
)
|
|
418
537
|
.build()
|
|
419
538
|
.await
|
|
420
539
|
.map_err(|err| {
|
|
@@ -423,32 +542,9 @@ impl HfCore {
|
|
|
423
542
|
})
|
|
424
543
|
}
|
|
425
544
|
|
|
426
|
-
///
|
|
427
|
-
///
|
|
428
|
-
///
|
|
429
|
-
/// otherwise differ only in which builder they seed with it. Uses the
|
|
430
|
-
/// canonical repo id so the refresh URL doesn't depend on redirect
|
|
431
|
-
/// behavior (see [`Self::canonical_repo`]). Resolves the repo once and
|
|
432
|
-
/// hands it to [`Self::cached_xet_token`] so the token lookup doesn't
|
|
433
|
-
/// resolve it again.
|
|
434
|
-
async fn xet_auth(
|
|
435
|
-
&self,
|
|
436
|
-
ctx: &OperationContext,
|
|
437
|
-
scope: XetTokenScope,
|
|
438
|
-
) -> Result<(XetToken, String, http::HeaderMap)> {
|
|
439
|
-
let repo = self.canonical_repo(ctx).await?;
|
|
440
|
-
let token = self.cached_xet_token(ctx, &repo, scope).await?;
|
|
441
|
-
let refresh_url = repo.xet_token_url(&self.endpoint, scope);
|
|
442
|
-
let refresh_headers = self.xet_token_refresh_headers();
|
|
443
|
-
Ok((token, refresh_url, refresh_headers))
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
/// Get a still-valid cached token for `scope` against the already
|
|
447
|
-
/// resolved `repo`, fetching and caching a fresh one if missing or
|
|
448
|
-
/// close to expiry. The lock is held across the refresh request so
|
|
449
|
-
/// concurrent callers single-flight onto one fetch; `repo` is resolved
|
|
450
|
-
/// by the caller beforehand so that lookup doesn't extend this
|
|
451
|
-
/// critical section.
|
|
545
|
+
/// Get a still-valid cached token for `scope`, fetching and caching a
|
|
546
|
+
/// fresh one if missing or close to expiry. The lock is held across the
|
|
547
|
+
/// refresh request so concurrent callers single-flight onto one fetch.
|
|
452
548
|
///
|
|
453
549
|
/// On a clock failure, `now` falls back to `u64::MAX` so the token is
|
|
454
550
|
/// treated as stale (extra `/api` traffic) rather than `0`, which would
|
|
@@ -456,7 +552,6 @@ impl HfCore {
|
|
|
456
552
|
async fn cached_xet_token(
|
|
457
553
|
&self,
|
|
458
554
|
ctx: &OperationContext,
|
|
459
|
-
repo: &HfRepo,
|
|
460
555
|
scope: XetTokenScope,
|
|
461
556
|
) -> Result<XetToken> {
|
|
462
557
|
let cache = match scope {
|
|
@@ -475,12 +570,12 @@ impl HfCore {
|
|
|
475
570
|
return Ok(token.clone());
|
|
476
571
|
}
|
|
477
572
|
|
|
478
|
-
let url = repo.xet_token_url(&self.endpoint, scope);
|
|
573
|
+
let url = self.repo.xet_token_url(&self.endpoint, scope);
|
|
479
574
|
let req = self
|
|
480
575
|
.request(http::Method::GET, &url, scope.operation(), "XetToken")?
|
|
481
576
|
.body(Buffer::new())
|
|
482
577
|
.map_err(new_request_build_error)?;
|
|
483
|
-
let resp =
|
|
578
|
+
let resp = self.send(ctx, req).await?;
|
|
484
579
|
if !resp.status().is_success() {
|
|
485
580
|
let (parts, _) = resp.into_parts();
|
|
486
581
|
return Err(parse_error(
|
|
@@ -513,7 +608,7 @@ impl HfCore {
|
|
|
513
608
|
&self,
|
|
514
609
|
ctx: &OperationContext,
|
|
515
610
|
) -> Result<XetDownloadStreamGroup> {
|
|
516
|
-
let
|
|
611
|
+
let token = self.cached_xet_token(ctx, XetTokenScope::Read).await?;
|
|
517
612
|
self.xet_session
|
|
518
613
|
.new_download_stream_group()
|
|
519
614
|
.map_err(|err| {
|
|
@@ -525,7 +620,10 @@ impl HfCore {
|
|
|
525
620
|
})?
|
|
526
621
|
.with_endpoint(token.cas_url)
|
|
527
622
|
.with_token_info(token.access_token, token.expires_at)
|
|
528
|
-
.with_token_refresh_url(
|
|
623
|
+
.with_token_refresh_url(
|
|
624
|
+
self.repo.xet_token_url(&self.endpoint, XetTokenScope::Read),
|
|
625
|
+
self.xet_token_refresh_headers(),
|
|
626
|
+
)
|
|
529
627
|
.build()
|
|
530
628
|
.await
|
|
531
629
|
.map_err(|err| {
|
|
@@ -548,15 +646,14 @@ impl HfCore {
|
|
|
548
646
|
op: Operation,
|
|
549
647
|
service_operation: &'static str,
|
|
550
648
|
) -> Result<http::request::Builder> {
|
|
551
|
-
|
|
552
|
-
// place that can show what's actually being queried -- enable with
|
|
553
|
-
// `RUST_LOG=opendal_service_hf::core=debug`.
|
|
649
|
+
let url = HttpUri::new(url);
|
|
554
650
|
debug!(
|
|
555
|
-
"hf request: service_operation={service_operation} operation={op} method={method} url={
|
|
651
|
+
"hf request: service_operation={service_operation} operation={op} method={method} url={}",
|
|
652
|
+
url.redacted_uri()
|
|
556
653
|
);
|
|
557
654
|
let mut req = Request::builder()
|
|
558
655
|
.method(method)
|
|
559
|
-
.uri(url)
|
|
656
|
+
.uri(url.original_uri())
|
|
560
657
|
.extension(op)
|
|
561
658
|
.extension(ServiceOperation(service_operation));
|
|
562
659
|
match &self.token {
|
|
@@ -576,58 +673,63 @@ impl HfCore {
|
|
|
576
673
|
Ok(req)
|
|
577
674
|
}
|
|
578
675
|
|
|
579
|
-
///
|
|
580
|
-
/// and caching it on first use.
|
|
676
|
+
/// Send `req`, following one same-endpoint `307`/`308` redirect.
|
|
581
677
|
///
|
|
582
|
-
/// HF resolves repo ids case-insensitively but
|
|
583
|
-
///
|
|
584
|
-
///
|
|
585
|
-
///
|
|
586
|
-
///
|
|
587
|
-
///
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
.
|
|
625
|
-
|
|
678
|
+
/// HF resolves repo ids case-insensitively but answers any request for
|
|
679
|
+
/// a non-canonically cased id (`user/repo` for `user/Repo`) with a 307
|
|
680
|
+
/// to the canonical URL. Transports follow that for `GET`s but hand a
|
|
681
|
+
/// bodied `POST` (commit, paths-info) back as the bare 307, which would
|
|
682
|
+
/// otherwise surface as an error while reads silently succeed.
|
|
683
|
+
/// Re-issuing the request here makes the outcome independent of the
|
|
684
|
+
/// transport's redirect policy. `send` returns a redirect to any other
|
|
685
|
+
/// host as-is, so it never sends the bearer token off the configured
|
|
686
|
+
/// endpoint itself; the caller then reports it via [`parse_error`].
|
|
687
|
+
pub(super) async fn send(
|
|
688
|
+
&self,
|
|
689
|
+
ctx: &OperationContext,
|
|
690
|
+
req: Request<Buffer>,
|
|
691
|
+
) -> Result<Response<HttpBody>> {
|
|
692
|
+
let retry = req.clone();
|
|
693
|
+
|
|
694
|
+
// The response is confined to this block so it is gone before the
|
|
695
|
+
// retry runs: its body is never read, so the connection cannot be
|
|
696
|
+
// pooled and is better closed than held across another round trip.
|
|
697
|
+
let target = {
|
|
698
|
+
let resp = ctx.http_transport().fetch(req).await?;
|
|
699
|
+
if !matches!(
|
|
700
|
+
resp.status(),
|
|
701
|
+
StatusCode::TEMPORARY_REDIRECT | StatusCode::PERMANENT_REDIRECT
|
|
702
|
+
) {
|
|
703
|
+
return Ok(resp);
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// Accept a path-only `Location`, which is what HF sends, or one
|
|
707
|
+
// already on the endpoint (a bare origin, normalized by
|
|
708
|
+
// `HfBuilder`). What remains after dropping the endpoint must be
|
|
709
|
+
// a single-slash path, which also refuses a protocol-relative
|
|
710
|
+
// `//host/...` naming another host.
|
|
711
|
+
let Some(location) = parse_location(resp.headers())? else {
|
|
712
|
+
return Ok(resp);
|
|
713
|
+
};
|
|
714
|
+
let path = location
|
|
715
|
+
.strip_prefix(self.endpoint.as_str())
|
|
716
|
+
.unwrap_or(location);
|
|
717
|
+
if !path.starts_with('/') || path.starts_with("//") {
|
|
718
|
+
return Ok(resp);
|
|
719
|
+
}
|
|
720
|
+
format!("{}{path}", self.endpoint)
|
|
721
|
+
};
|
|
626
722
|
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
723
|
+
let target = HttpUri::new(target);
|
|
724
|
+
debug!("hf request redirected: url={}", target.redacted_uri());
|
|
725
|
+
let (mut parts, body) = retry.into_parts();
|
|
726
|
+
parts.uri = target
|
|
727
|
+
.original_uri()
|
|
728
|
+
.parse()
|
|
729
|
+
.map_err(new_http_uri_invalid_error)?;
|
|
730
|
+
ctx.http_transport()
|
|
731
|
+
.fetch(Request::from_parts(parts, body))
|
|
732
|
+
.await
|
|
631
733
|
}
|
|
632
734
|
|
|
633
735
|
/// Convert an operator-relative path to a repo-absolute path
|
|
@@ -639,7 +741,7 @@ impl HfCore {
|
|
|
639
741
|
}
|
|
640
742
|
|
|
641
743
|
pub(super) async fn path_info(&self, ctx: &OperationContext, path: &str) -> Result<PathInfo> {
|
|
642
|
-
let uri = self.
|
|
744
|
+
let uri = self.repo.uri(&self.root, path);
|
|
643
745
|
let url = uri.paths_info_url(&self.endpoint);
|
|
644
746
|
let form_body = format!("paths={}&expand=True", percent_encode_path(&uri.path));
|
|
645
747
|
|
|
@@ -648,7 +750,7 @@ impl HfCore {
|
|
|
648
750
|
.header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
|
649
751
|
.body(Buffer::from(Bytes::from(form_body)))
|
|
650
752
|
.map_err(new_request_build_error)?;
|
|
651
|
-
let resp =
|
|
753
|
+
let resp = self.send(ctx, req).await?;
|
|
652
754
|
if !resp.status().is_success() {
|
|
653
755
|
let (parts, _) = resp.into_parts();
|
|
654
756
|
return Err(parse_error(
|
|
@@ -669,6 +771,125 @@ impl HfCore {
|
|
|
669
771
|
Ok(files.remove(0))
|
|
670
772
|
}
|
|
671
773
|
|
|
774
|
+
pub(super) async fn read(
|
|
775
|
+
&self,
|
|
776
|
+
ctx: &OperationContext,
|
|
777
|
+
path: &str,
|
|
778
|
+
range: BytesRange,
|
|
779
|
+
) -> Result<HfReadResponse> {
|
|
780
|
+
let entry = if self.enable_resolve_cache {
|
|
781
|
+
self.resolved_files.entry(path)
|
|
782
|
+
} else {
|
|
783
|
+
None
|
|
784
|
+
};
|
|
785
|
+
let Some(entry) = entry else {
|
|
786
|
+
let resp = self.resolve(ctx, path, range, self.download_mode).await?;
|
|
787
|
+
return HfReadResponse::from_response(resp, self.download_mode).await;
|
|
788
|
+
};
|
|
789
|
+
|
|
790
|
+
let mut rejected: Option<Arc<HttpDownload>> = None;
|
|
791
|
+
loop {
|
|
792
|
+
let destination = {
|
|
793
|
+
let mut state = entry.lock().await;
|
|
794
|
+
match &*state {
|
|
795
|
+
ResolvedFile::Xet(info) => return Ok(HfReadResponse::Xet(info.clone())),
|
|
796
|
+
ResolvedFile::NotXet | ResolvedFile::Bypass => {
|
|
797
|
+
drop(state);
|
|
798
|
+
let resp = self.resolve(ctx, path, range, HfDownloadMode::Http).await?;
|
|
799
|
+
return Ok(HfReadResponse::Http(resp));
|
|
800
|
+
}
|
|
801
|
+
ResolvedFile::Http(current)
|
|
802
|
+
if current.valid_until > Instant::now()
|
|
803
|
+
&& !rejected
|
|
804
|
+
.as_ref()
|
|
805
|
+
.is_some_and(|old| Arc::ptr_eq(old, current)) =>
|
|
806
|
+
{
|
|
807
|
+
current.clone()
|
|
808
|
+
}
|
|
809
|
+
_ => {
|
|
810
|
+
let mode = if matches!(&*state, ResolvedFile::Http(_)) {
|
|
811
|
+
HfDownloadMode::Http
|
|
812
|
+
} else {
|
|
813
|
+
self.download_mode
|
|
814
|
+
};
|
|
815
|
+
// Serialize resolution and refresh. XET metadata is
|
|
816
|
+
// independent of the range; a one-byte probe bounds the
|
|
817
|
+
// body discarded when the file turns out to use HTTP.
|
|
818
|
+
*state = ResolvedFile::Empty;
|
|
819
|
+
let resolve_range = if mode == HfDownloadMode::Xet {
|
|
820
|
+
BytesRange::new(0, Some(1))
|
|
821
|
+
} else {
|
|
822
|
+
range
|
|
823
|
+
};
|
|
824
|
+
let resp = self.resolve(ctx, path, resolve_range, mode).await?;
|
|
825
|
+
let resp = match HfReadResponse::from_response(resp, mode).await? {
|
|
826
|
+
HfReadResponse::Xet(info) => {
|
|
827
|
+
*state = ResolvedFile::Xet(info.clone());
|
|
828
|
+
return Ok(HfReadResponse::Xet(info));
|
|
829
|
+
}
|
|
830
|
+
HfReadResponse::Http(resp) => resp,
|
|
831
|
+
};
|
|
832
|
+
*state = match HttpDownload::from_response(&resp) {
|
|
833
|
+
Some(download) => ResolvedFile::Http(Arc::new(download)),
|
|
834
|
+
None if self.download_mode == HfDownloadMode::Xet => {
|
|
835
|
+
ResolvedFile::NotXet
|
|
836
|
+
}
|
|
837
|
+
None => {
|
|
838
|
+
// This live slot cannot have been evicted or
|
|
839
|
+
// replaced while we hold an Arc to it.
|
|
840
|
+
self.resolved_files
|
|
841
|
+
.entries
|
|
842
|
+
.lock()
|
|
843
|
+
.expect("resolved file cache lock poisoned")
|
|
844
|
+
.remove(path);
|
|
845
|
+
ResolvedFile::Bypass
|
|
846
|
+
}
|
|
847
|
+
};
|
|
848
|
+
if mode == HfDownloadMode::Http {
|
|
849
|
+
// The response already contains the requested range.
|
|
850
|
+
return Ok(HfReadResponse::Http(resp));
|
|
851
|
+
}
|
|
852
|
+
// The XET probe returned one byte of HTTP content.
|
|
853
|
+
// Reuse its destination to fetch the caller's range.
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
};
|
|
858
|
+
|
|
859
|
+
let url = self
|
|
860
|
+
.repo
|
|
861
|
+
.uri(&self.root, path)
|
|
862
|
+
.resolve_url(&self.endpoint, self.repo.revision());
|
|
863
|
+
let mut req = self
|
|
864
|
+
.request(http::Method::GET, &url, Operation::Read, "Download")?
|
|
865
|
+
.extension(destination.redirect.clone());
|
|
866
|
+
if !range.is_full() {
|
|
867
|
+
req = req.header(header::RANGE, range.to_header());
|
|
868
|
+
}
|
|
869
|
+
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
|
|
870
|
+
let resp = ctx.http_transport().fetch(req).await?;
|
|
871
|
+
if resp.status().is_success() {
|
|
872
|
+
return Ok(HfReadResponse::Http(resp));
|
|
873
|
+
}
|
|
874
|
+
if rejected.is_none()
|
|
875
|
+
&& matches!(
|
|
876
|
+
resp.status(),
|
|
877
|
+
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN
|
|
878
|
+
)
|
|
879
|
+
{
|
|
880
|
+
// A late rejection must not discard another caller's refreshed
|
|
881
|
+
// destination. The next iteration compares the Arc generation.
|
|
882
|
+
rejected = Some(destination);
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
let (parts, _) = resp.into_parts();
|
|
886
|
+
return Err(parse_error(
|
|
887
|
+
ErrorContext::new(ServiceOperation("Download")),
|
|
888
|
+
parts,
|
|
889
|
+
));
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
|
|
672
893
|
/// Send `GET /resolve` and return the raw streaming response.
|
|
673
894
|
///
|
|
674
895
|
/// In `Xet` mode adds `Accept: application/vnd.xet-fileinfo+json` so the
|
|
@@ -683,7 +904,7 @@ impl HfCore {
|
|
|
683
904
|
range: BytesRange,
|
|
684
905
|
mode: HfDownloadMode,
|
|
685
906
|
) -> Result<Response<HttpBody>> {
|
|
686
|
-
let uri = self.
|
|
907
|
+
let uri = self.repo.uri(&self.root, path);
|
|
687
908
|
let url = uri.resolve_url(&self.endpoint, self.repo.revision());
|
|
688
909
|
|
|
689
910
|
let mut req = self.request(http::Method::GET, &url, Operation::Read, "Resolve")?;
|
|
@@ -697,7 +918,7 @@ impl HfCore {
|
|
|
697
918
|
}
|
|
698
919
|
|
|
699
920
|
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
|
|
700
|
-
let resp =
|
|
921
|
+
let resp = self.send(ctx, req).await?;
|
|
701
922
|
|
|
702
923
|
if !resp.status().is_success() {
|
|
703
924
|
let status = resp.status();
|
|
@@ -726,10 +947,7 @@ impl HfCore {
|
|
|
726
947
|
deleted_files: Vec<DeletedFile>,
|
|
727
948
|
deleted_folders: Vec<DeletedFolder>,
|
|
728
949
|
) -> Result<CommitResponse> {
|
|
729
|
-
let url = self
|
|
730
|
-
.canonical_repo(ctx)
|
|
731
|
-
.await?
|
|
732
|
-
.git_commit_url(&self.endpoint);
|
|
950
|
+
let url = self.repo.git_commit_url(&self.endpoint);
|
|
733
951
|
|
|
734
952
|
let payload = MixedCommitPayload {
|
|
735
953
|
summary: "Commit via OpenDAL".to_string(),
|
|
@@ -748,7 +966,7 @@ impl HfCore {
|
|
|
748
966
|
.body(Buffer::from(json_body))
|
|
749
967
|
.map_err(new_request_build_error)?;
|
|
750
968
|
|
|
751
|
-
let resp =
|
|
969
|
+
let resp = self.send(ctx, req).await?;
|
|
752
970
|
if !resp.status().is_success() {
|
|
753
971
|
let (parts, _) = resp.into_parts();
|
|
754
972
|
return Err(parse_error(
|
|
@@ -792,7 +1010,7 @@ impl HfCore {
|
|
|
792
1010
|
.body(Buffer::from(Bytes::from(body)))
|
|
793
1011
|
.map_err(new_request_build_error)?;
|
|
794
1012
|
|
|
795
|
-
let resp =
|
|
1013
|
+
let resp = self.send(ctx, req).await?;
|
|
796
1014
|
if !resp.status().is_success() {
|
|
797
1015
|
let (parts, _) = resp.into_parts();
|
|
798
1016
|
return Err(parse_error(
|
|
@@ -806,7 +1024,7 @@ impl HfCore {
|
|
|
806
1024
|
|
|
807
1025
|
#[cfg(test)]
|
|
808
1026
|
pub(crate) mod test_utils {
|
|
809
|
-
use http::{Request, Response
|
|
1027
|
+
use http::{Request, Response};
|
|
810
1028
|
use std::sync::{Arc, Mutex};
|
|
811
1029
|
|
|
812
1030
|
use super::super::core::HfRepoType;
|
|
@@ -826,10 +1044,8 @@ pub(crate) mod test_utils {
|
|
|
826
1044
|
/// mocked [`XetFileResponse`] body instead of plain bytes, so tests
|
|
827
1045
|
/// can exercise the XET classification path without real network.
|
|
828
1046
|
xet_file: Arc<Mutex<Option<XetFileResponse>>>,
|
|
829
|
-
/// `Range` header of the most recent XET
|
|
830
|
-
///
|
|
831
|
-
/// tell which racing caller's range a single-flighted classification
|
|
832
|
-
/// actually sent.
|
|
1047
|
+
/// `Range` header of the most recent XET metadata probe, so tests can
|
|
1048
|
+
/// check it sends a fixed single byte rather than the caller's range.
|
|
833
1049
|
classify_range_header: Arc<Mutex<Option<String>>>,
|
|
834
1050
|
}
|
|
835
1051
|
|
|
@@ -880,9 +1096,6 @@ pub(crate) mod test_utils {
|
|
|
880
1096
|
*self.request_count.lock().unwrap()
|
|
881
1097
|
}
|
|
882
1098
|
|
|
883
|
-
/// `Range` header of the most recent XET-classifying `/resolve/`
|
|
884
|
-
/// request, or `None` if that range was full (no `Range` header) or
|
|
885
|
-
/// no such request has happened yet.
|
|
886
1099
|
pub(crate) fn get_captured_classify_range_header(&self) -> Option<String> {
|
|
887
1100
|
self.classify_range_header.lock().unwrap().clone()
|
|
888
1101
|
}
|
|
@@ -973,16 +1186,6 @@ pub(crate) mod test_utils {
|
|
|
973
1186
|
HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
|
|
974
1187
|
size,
|
|
975
1188
|
)
|
|
976
|
-
} else if let Some(rest) = req.uri().path().strip_prefix("/api/") {
|
|
977
|
-
// Repo-info: echo the requested id back as the canonical id.
|
|
978
|
-
let id = rest.split_once('/').map(|(_, id)| id).unwrap_or(rest);
|
|
979
|
-
let data = Bytes::from(format!(r#"{{"id":"{id}"}}"#));
|
|
980
|
-
let size = data.len() as u64;
|
|
981
|
-
let buffer = Buffer::from(data);
|
|
982
|
-
(
|
|
983
|
-
HttpBody::new(futures::stream::iter(vec![Ok(buffer)]), Some(size)),
|
|
984
|
-
size,
|
|
985
|
-
)
|
|
986
1189
|
} else {
|
|
987
1190
|
let data = Bytes::from_static(b"hello");
|
|
988
1191
|
let size = data.len() as u64;
|
|
@@ -1007,8 +1210,25 @@ pub(crate) mod test_utils {
|
|
|
1007
1210
|
revision: &str,
|
|
1008
1211
|
endpoint: &str,
|
|
1009
1212
|
) -> (HfCore, OperationContext, MockHttpTransport) {
|
|
1010
|
-
|
|
1011
|
-
|
|
1213
|
+
create_test_core_with(
|
|
1214
|
+
MockHttpTransport::new(),
|
|
1215
|
+
repo_type,
|
|
1216
|
+
repo_id,
|
|
1217
|
+
revision,
|
|
1218
|
+
endpoint,
|
|
1219
|
+
)
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
/// Like [`create_test_core`] but with a caller-supplied transport, for
|
|
1223
|
+
/// tests that need to script responses the shared mock doesn't.
|
|
1224
|
+
pub(crate) fn create_test_core_with<T: HttpTransport + Clone>(
|
|
1225
|
+
transport: T,
|
|
1226
|
+
repo_type: HfRepoType,
|
|
1227
|
+
repo_id: &str,
|
|
1228
|
+
revision: &str,
|
|
1229
|
+
endpoint: &str,
|
|
1230
|
+
) -> (HfCore, OperationContext, T) {
|
|
1231
|
+
let http_transport = HttpTransporter::new(transport.clone());
|
|
1012
1232
|
let ctx = OperationContext::from_parts(http_transport, Executor::default());
|
|
1013
1233
|
|
|
1014
1234
|
let info = ServiceInfo::new("hf", "", "");
|
|
@@ -1028,7 +1248,7 @@ pub(crate) mod test_utils {
|
|
|
1028
1248
|
HfDownloadMode::Xet,
|
|
1029
1249
|
);
|
|
1030
1250
|
|
|
1031
|
-
(core, ctx,
|
|
1251
|
+
(core, ctx, transport)
|
|
1032
1252
|
}
|
|
1033
1253
|
}
|
|
1034
1254
|
|
|
@@ -1037,10 +1257,9 @@ mod tests {
|
|
|
1037
1257
|
use std::sync::{Arc, Mutex};
|
|
1038
1258
|
|
|
1039
1259
|
use http::Response;
|
|
1040
|
-
use http::StatusCode;
|
|
1041
1260
|
|
|
1042
1261
|
use super::super::core::HfRepoType;
|
|
1043
|
-
use super::test_utils::create_test_core;
|
|
1262
|
+
use super::test_utils::{create_test_core, create_test_core_with};
|
|
1044
1263
|
use super::*;
|
|
1045
1264
|
|
|
1046
1265
|
#[tokio::test]
|
|
@@ -1123,15 +1342,26 @@ mod tests {
|
|
|
1123
1342
|
Ok(())
|
|
1124
1343
|
}
|
|
1125
1344
|
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
requests: Arc<Mutex<Vec<(String, String, String)>>>,
|
|
1345
|
+
struct SeenRequest {
|
|
1346
|
+
method: String,
|
|
1347
|
+
uri: String,
|
|
1348
|
+
auth: String,
|
|
1349
|
+
body: String,
|
|
1132
1350
|
}
|
|
1133
1351
|
|
|
1134
|
-
|
|
1352
|
+
/// A scripted transport mirroring how HF serves a repo whose configured
|
|
1353
|
+
/// id is not canonically cased: it answers that URL with `status` and a
|
|
1354
|
+
/// `Location` of `location`, and only the canonically cased URL accepts
|
|
1355
|
+
/// the commit. Tests set both, so one transport covers
|
|
1356
|
+
/// a followed redirect, a refused one, and a non-307 status.
|
|
1357
|
+
#[derive(Clone)]
|
|
1358
|
+
struct RedirectingTransport {
|
|
1359
|
+
status: StatusCode,
|
|
1360
|
+
location: &'static str,
|
|
1361
|
+
requests: Arc<Mutex<Vec<SeenRequest>>>,
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
impl HttpTransport for RedirectingTransport {
|
|
1135
1365
|
async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
|
|
1136
1366
|
let uri = req.uri().to_string();
|
|
1137
1367
|
let auth = req
|
|
@@ -1140,91 +1370,200 @@ mod tests {
|
|
|
1140
1370
|
.and_then(|v| v.to_str().ok())
|
|
1141
1371
|
.unwrap_or_default()
|
|
1142
1372
|
.to_string();
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
.
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1373
|
+
let body = String::from_utf8(req.body().to_bytes().to_vec()).unwrap();
|
|
1374
|
+
self.requests.lock().unwrap().push(SeenRequest {
|
|
1375
|
+
method: req.method().to_string(),
|
|
1376
|
+
uri: uri.clone(),
|
|
1377
|
+
auth,
|
|
1378
|
+
body,
|
|
1379
|
+
});
|
|
1380
|
+
|
|
1381
|
+
let resp = match uri.as_str() {
|
|
1382
|
+
"https://huggingface.co/api/models/test-user/uppercase-repo/commit/main" => {
|
|
1383
|
+
Response::builder()
|
|
1384
|
+
.status(self.status)
|
|
1385
|
+
.header(header::LOCATION, self.location)
|
|
1386
|
+
.header(header::CONTENT_LENGTH, 0)
|
|
1387
|
+
.body(HttpBody::new(futures::stream::empty(), Some(0)))
|
|
1151
1388
|
}
|
|
1152
1389
|
"https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main" => {
|
|
1153
|
-
|
|
1390
|
+
Response::builder()
|
|
1391
|
+
.status(StatusCode::OK)
|
|
1392
|
+
.header(header::CONTENT_LENGTH, 2)
|
|
1393
|
+
.body(HttpBody::new(
|
|
1394
|
+
futures::stream::iter(vec![Ok(Buffer::from(Bytes::from_static(
|
|
1395
|
+
b"{}",
|
|
1396
|
+
)))]),
|
|
1397
|
+
Some(2),
|
|
1398
|
+
))
|
|
1154
1399
|
}
|
|
1155
1400
|
other => panic!("unexpected request to {other}"),
|
|
1156
1401
|
};
|
|
1157
|
-
|
|
1158
|
-
Ok(Response::builder()
|
|
1159
|
-
.status(http::StatusCode::OK)
|
|
1160
|
-
.header(header::CONTENT_LENGTH, len)
|
|
1161
|
-
.body(HttpBody::new(
|
|
1162
|
-
futures::stream::iter(vec![Ok(Buffer::from(body))]),
|
|
1163
|
-
Some(len),
|
|
1164
|
-
))
|
|
1165
|
-
.unwrap())
|
|
1402
|
+
Ok(resp.unwrap())
|
|
1166
1403
|
}
|
|
1167
1404
|
}
|
|
1168
1405
|
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
HfRepo::new(
|
|
1184
|
-
HfRepoType::Model,
|
|
1185
|
-
"test-user/uppercase-repo".to_string(),
|
|
1186
|
-
Some("main".to_string()),
|
|
1187
|
-
),
|
|
1188
|
-
"/".to_string(),
|
|
1189
|
-
Some("hf_dummy".to_string()),
|
|
1190
|
-
"https://huggingface.co".to_string(),
|
|
1191
|
-
xet_session,
|
|
1192
|
-
HfDownloadMode::Xet,
|
|
1406
|
+
fn redirecting_core(
|
|
1407
|
+
status: StatusCode,
|
|
1408
|
+
location: &'static str,
|
|
1409
|
+
) -> (HfCore, OperationContext, RedirectingTransport) {
|
|
1410
|
+
let (mut core, ctx, transport) = create_test_core_with(
|
|
1411
|
+
RedirectingTransport {
|
|
1412
|
+
status,
|
|
1413
|
+
location,
|
|
1414
|
+
requests: Arc::default(),
|
|
1415
|
+
},
|
|
1416
|
+
HfRepoType::Model,
|
|
1417
|
+
"test-user/uppercase-repo",
|
|
1418
|
+
"main",
|
|
1419
|
+
"https://huggingface.co",
|
|
1193
1420
|
);
|
|
1421
|
+
core.token = Some("hf_dummy".to_string());
|
|
1422
|
+
(core, ctx, transport)
|
|
1423
|
+
}
|
|
1194
1424
|
|
|
1195
|
-
|
|
1425
|
+
fn lfs_file(path: &str) -> LfsFile {
|
|
1426
|
+
LfsFile {
|
|
1196
1427
|
path: path.to_string(),
|
|
1197
1428
|
oid: "deadbeef".to_string(),
|
|
1198
1429
|
algo: "sha256".to_string(),
|
|
1199
1430
|
size: 2812,
|
|
1200
|
-
}
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
/// `send` re-issues a bodied `POST` answered with a 307 to the
|
|
1435
|
+
/// canonically cased URL with the same method, headers and body,
|
|
1436
|
+
/// regardless of whether the transport replays redirects itself. HF
|
|
1437
|
+
/// sends the `Location` as an absolute path, as in the report for #8107.
|
|
1438
|
+
#[tokio::test]
|
|
1439
|
+
async fn test_commit_follows_case_redirect() -> Result<()> {
|
|
1440
|
+
let (core, ctx, transport) = redirecting_core(
|
|
1441
|
+
StatusCode::TEMPORARY_REDIRECT,
|
|
1442
|
+
"/api/models/test-user/Uppercase-Repo/commit/main",
|
|
1443
|
+
);
|
|
1444
|
+
|
|
1201
1445
|
core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1202
1446
|
.await?;
|
|
1203
|
-
core.commit_git(&ctx, vec![], vec![lfs_file("b.md")], vec![], vec![])
|
|
1204
|
-
.await?;
|
|
1205
1447
|
|
|
1206
1448
|
let requests = transport.requests.lock().unwrap();
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1449
|
+
assert_eq!(requests.len(), 2);
|
|
1450
|
+
let (first, redirected) = (&requests[0], &requests[1]);
|
|
1451
|
+
assert_eq!(first.method, "POST");
|
|
1452
|
+
assert_eq!(
|
|
1453
|
+
first.uri,
|
|
1454
|
+
"https://huggingface.co/api/models/test-user/uppercase-repo/commit/main"
|
|
1455
|
+
);
|
|
1456
|
+
assert_eq!(first.auth, "Bearer hf_dummy");
|
|
1457
|
+
assert_eq!(redirected.method, "POST");
|
|
1458
|
+
assert_eq!(
|
|
1459
|
+
redirected.uri,
|
|
1460
|
+
"https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main"
|
|
1461
|
+
);
|
|
1462
|
+
assert_eq!(redirected.auth, "Bearer hf_dummy");
|
|
1463
|
+
assert_eq!(redirected.body, first.body);
|
|
1464
|
+
assert!(first.body.contains("a.md"));
|
|
1465
|
+
|
|
1466
|
+
Ok(())
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
/// `send` also follows an absolute `Location` on the configured endpoint,
|
|
1470
|
+
/// and a 308 like a 307.
|
|
1471
|
+
#[tokio::test]
|
|
1472
|
+
async fn test_commit_follows_absolute_same_endpoint_redirect() -> Result<()> {
|
|
1473
|
+
let (core, ctx, transport) = redirecting_core(
|
|
1474
|
+
StatusCode::PERMANENT_REDIRECT,
|
|
1475
|
+
"https://huggingface.co/api/models/test-user/Uppercase-Repo/commit/main",
|
|
1476
|
+
);
|
|
1477
|
+
|
|
1478
|
+
core.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1479
|
+
.await?;
|
|
1480
|
+
|
|
1481
|
+
assert_eq!(transport.requests.lock().unwrap().len(), 2);
|
|
1482
|
+
Ok(())
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/// `send` must not follow a redirect off the configured endpoint: the
|
|
1486
|
+
/// re-issued request would carry the bearer token to a foreign host.
|
|
1487
|
+
/// The 307 surfaces as an error instead.
|
|
1488
|
+
#[tokio::test]
|
|
1489
|
+
async fn test_commit_does_not_follow_foreign_redirect() -> Result<()> {
|
|
1490
|
+
let (core, ctx, transport) = redirecting_core(
|
|
1491
|
+
StatusCode::TEMPORARY_REDIRECT,
|
|
1492
|
+
"https://evil.example.com/api/models/test-user/Uppercase-Repo/commit/main",
|
|
1493
|
+
);
|
|
1494
|
+
|
|
1495
|
+
let err = core
|
|
1496
|
+
.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1497
|
+
.await
|
|
1498
|
+
.expect_err("a foreign redirect must not be followed");
|
|
1499
|
+
assert_eq!(err.kind(), ErrorKind::Unexpected);
|
|
1500
|
+
assert!(
|
|
1501
|
+
err.to_string()
|
|
1502
|
+
.contains("redirect to https://evil.example.com/api/models/test-user/Uppercase-Repo/commit/main not followed"),
|
|
1503
|
+
"the refused redirect must be named in the error: {err}"
|
|
1504
|
+
);
|
|
1505
|
+
assert_eq!(transport.requests.lock().unwrap().len(), 1);
|
|
1506
|
+
|
|
1507
|
+
Ok(())
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
/// A protocol-relative `Location` names another host, so it is refused
|
|
1511
|
+
/// rather than treated as a path on ours.
|
|
1512
|
+
#[tokio::test]
|
|
1513
|
+
async fn test_commit_does_not_follow_protocol_relative_redirect() -> Result<()> {
|
|
1514
|
+
let (core, ctx, transport) =
|
|
1515
|
+
redirecting_core(StatusCode::TEMPORARY_REDIRECT, "//evil.example.com/api/x");
|
|
1516
|
+
|
|
1517
|
+
let err = core
|
|
1518
|
+
.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1519
|
+
.await
|
|
1520
|
+
.expect_err("a protocol-relative redirect must not be followed");
|
|
1521
|
+
assert!(err.to_string().contains("not followed"), "{err}");
|
|
1522
|
+
assert_eq!(transport.requests.lock().unwrap().len(), 1);
|
|
1523
|
+
|
|
1524
|
+
Ok(())
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
/// A `Location` without a leading slash is a relative reference this
|
|
1528
|
+
/// service does not resolve; it is refused rather than guessed at.
|
|
1529
|
+
#[tokio::test]
|
|
1530
|
+
async fn test_commit_does_not_follow_relative_redirect() -> Result<()> {
|
|
1531
|
+
let (core, ctx, transport) = redirecting_core(
|
|
1532
|
+
StatusCode::TEMPORARY_REDIRECT,
|
|
1533
|
+
"api/models/test-user/Uppercase-Repo/commit/main",
|
|
1534
|
+
);
|
|
1535
|
+
|
|
1536
|
+
let err = core
|
|
1537
|
+
.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1538
|
+
.await
|
|
1539
|
+
.expect_err("a relative redirect must not be followed");
|
|
1540
|
+
assert!(err.to_string().contains("not followed"), "{err}");
|
|
1541
|
+
assert_eq!(transport.requests.lock().unwrap().len(), 1);
|
|
1542
|
+
|
|
1543
|
+
Ok(())
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/// `send` only replays 307/308. A 302 is passed through untouched: in
|
|
1547
|
+
/// http download mode the transport follows the resolve 302 to the CDN
|
|
1548
|
+
/// itself, and `send` must not re-issue it on the API host.
|
|
1549
|
+
#[tokio::test]
|
|
1550
|
+
async fn test_send_leaves_302_to_the_transport() -> Result<()> {
|
|
1551
|
+
let (core, ctx, transport) = redirecting_core(
|
|
1552
|
+
StatusCode::FOUND,
|
|
1553
|
+
"/api/models/test-user/Uppercase-Repo/commit/main",
|
|
1554
|
+
);
|
|
1555
|
+
|
|
1556
|
+
let err = core
|
|
1557
|
+
.commit_git(&ctx, vec![], vec![lfs_file("a.md")], vec![], vec![])
|
|
1558
|
+
.await
|
|
1559
|
+
.expect_err("an unfollowed 302 must surface as an error");
|
|
1560
|
+
assert!(
|
|
1561
|
+
err.to_string().contains(
|
|
1562
|
+
"redirect to /api/models/test-user/Uppercase-Repo/commit/main not followed"
|
|
1220
1563
|
),
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
assert_eq!(method, exp_method);
|
|
1225
|
-
assert_eq!(uri, exp_uri);
|
|
1226
|
-
assert_eq!(auth, "Bearer hf_dummy");
|
|
1227
|
-
}
|
|
1564
|
+
"{err}"
|
|
1565
|
+
);
|
|
1566
|
+
assert_eq!(transport.requests.lock().unwrap().len(), 1);
|
|
1228
1567
|
|
|
1229
1568
|
Ok(())
|
|
1230
1569
|
}
|
|
@@ -1239,22 +1578,15 @@ mod tests {
|
|
|
1239
1578
|
);
|
|
1240
1579
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1241
1580
|
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
let repo = core.canonical_repo(&ctx).await?;
|
|
1245
|
-
let first = core
|
|
1246
|
-
.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1247
|
-
.await?;
|
|
1248
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
1581
|
+
let first = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1582
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1249
1583
|
assert_eq!(first.access_token, "mock-token");
|
|
1250
1584
|
assert_eq!(first.cas_url, "https://cas.example.com");
|
|
1251
1585
|
|
|
1252
1586
|
// A second call with the cached token nowhere near expiry must not
|
|
1253
|
-
// hit the network again
|
|
1254
|
-
let second = core
|
|
1255
|
-
|
|
1256
|
-
.await?;
|
|
1257
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
1587
|
+
// hit the network again.
|
|
1588
|
+
let second = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1589
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1258
1590
|
assert_eq!(second.access_token, first.access_token);
|
|
1259
1591
|
|
|
1260
1592
|
Ok(())
|
|
@@ -1271,19 +1603,13 @@ mod tests {
|
|
|
1271
1603
|
// Already within the refresh buffer of "now" -- immediately stale.
|
|
1272
1604
|
mock_client.set_xet_token_expires_at(0);
|
|
1273
1605
|
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
let repo = core.canonical_repo(&ctx).await?;
|
|
1277
|
-
core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1278
|
-
.await?;
|
|
1279
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
1606
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1607
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1280
1608
|
|
|
1281
|
-
|
|
1282
|
-
core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1283
|
-
.await?;
|
|
1609
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1284
1610
|
assert_eq!(
|
|
1285
1611
|
mock_client.request_count(),
|
|
1286
|
-
|
|
1612
|
+
2,
|
|
1287
1613
|
"a token that's always stale must be refreshed on every call, not cached"
|
|
1288
1614
|
);
|
|
1289
1615
|
|
|
@@ -1304,21 +1630,15 @@ mod tests {
|
|
|
1304
1630
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1305
1631
|
mock_client.fail_next_requests(1);
|
|
1306
1632
|
|
|
1307
|
-
|
|
1308
|
-
// request resolving it makes.
|
|
1309
|
-
match core.canonical_repo(&ctx).await {
|
|
1633
|
+
match core.cached_xet_token(&ctx, XetTokenScope::Read).await {
|
|
1310
1634
|
Err(err) => assert!(err.to_string().contains("mock injected failure")),
|
|
1311
|
-
Ok(_) => panic!("a failed
|
|
1635
|
+
Ok(_) => panic!("a failed token fetch must surface as an error"),
|
|
1312
1636
|
}
|
|
1313
1637
|
assert_eq!(mock_client.request_count(), 1);
|
|
1314
1638
|
|
|
1315
|
-
|
|
1316
|
-
let repo = core.canonical_repo(&ctx).await?;
|
|
1317
|
-
let token = core
|
|
1318
|
-
.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1319
|
-
.await?;
|
|
1639
|
+
let token = core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1320
1640
|
assert_eq!(token.access_token, "mock-token");
|
|
1321
|
-
assert_eq!(mock_client.request_count(),
|
|
1641
|
+
assert_eq!(mock_client.request_count(), 2);
|
|
1322
1642
|
|
|
1323
1643
|
Ok(())
|
|
1324
1644
|
}
|
|
@@ -1331,32 +1651,24 @@ mod tests {
|
|
|
1331
1651
|
"main",
|
|
1332
1652
|
"https://huggingface.co",
|
|
1333
1653
|
);
|
|
1334
|
-
//
|
|
1335
|
-
//
|
|
1654
|
+
// Write-scope token fetches are tagged `Operation::Write`, which
|
|
1655
|
+
// requires a token.
|
|
1336
1656
|
core.token = Some("hf_dummy".to_string());
|
|
1337
1657
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1338
1658
|
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
let repo = core.canonical_repo(&ctx).await?;
|
|
1342
|
-
core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1343
|
-
.await?;
|
|
1344
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
1659
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1660
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1345
1661
|
|
|
1346
1662
|
// A write token request must not be satisfied by the read token's
|
|
1347
|
-
// cache slot -- read and write are distinct HF API scopes.
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
.await?;
|
|
1351
|
-
assert_eq!(mock_client.request_count(), 3);
|
|
1663
|
+
// cache slot -- read and write are distinct HF API scopes.
|
|
1664
|
+
core.cached_xet_token(&ctx, XetTokenScope::Write).await?;
|
|
1665
|
+
assert_eq!(mock_client.request_count(), 2);
|
|
1352
1666
|
assert!(mock_client.get_captured_url().contains("write"));
|
|
1353
1667
|
|
|
1354
1668
|
// Both are now warm; neither call should hit the network again.
|
|
1355
|
-
core.cached_xet_token(&ctx,
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
.await?;
|
|
1359
|
-
assert_eq!(mock_client.request_count(), 3);
|
|
1669
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1670
|
+
core.cached_xet_token(&ctx, XetTokenScope::Write).await?;
|
|
1671
|
+
assert_eq!(mock_client.request_count(), 2);
|
|
1360
1672
|
|
|
1361
1673
|
Ok(())
|
|
1362
1674
|
}
|
|
@@ -1375,15 +1687,13 @@ mod tests {
|
|
|
1375
1687
|
);
|
|
1376
1688
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1377
1689
|
|
|
1378
|
-
// 2 requests: one to resolve the canonical repo id, one for the
|
|
1379
|
-
// read token itself.
|
|
1380
1690
|
core.xet_download_group(&ctx).await?;
|
|
1381
|
-
assert_eq!(mock_client.request_count(),
|
|
1691
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1382
1692
|
|
|
1383
|
-
// A second group build must reuse the cached read token
|
|
1384
|
-
//
|
|
1693
|
+
// A second group build must reuse the cached read token rather than
|
|
1694
|
+
// fetching its own.
|
|
1385
1695
|
core.xet_download_group(&ctx).await?;
|
|
1386
|
-
assert_eq!(mock_client.request_count(),
|
|
1696
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1387
1697
|
|
|
1388
1698
|
Ok(())
|
|
1389
1699
|
}
|
|
@@ -1399,23 +1709,20 @@ mod tests {
|
|
|
1399
1709
|
core.token = Some("hf_dummy".to_string());
|
|
1400
1710
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1401
1711
|
|
|
1402
|
-
// 2 requests: one to resolve the canonical repo id, one for the
|
|
1403
|
-
// write token itself.
|
|
1404
1712
|
core.xet_upload_commit(&ctx).await?;
|
|
1405
|
-
assert_eq!(mock_client.request_count(),
|
|
1713
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1406
1714
|
assert!(mock_client.get_captured_url().contains("write"));
|
|
1407
1715
|
|
|
1408
|
-
// A second commit build must reuse the cached write token
|
|
1409
|
-
//
|
|
1716
|
+
// A second commit build must reuse the cached write token rather
|
|
1717
|
+
// than fetching its own.
|
|
1410
1718
|
core.xet_upload_commit(&ctx).await?;
|
|
1411
|
-
assert_eq!(mock_client.request_count(),
|
|
1719
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1412
1720
|
|
|
1413
1721
|
Ok(())
|
|
1414
1722
|
}
|
|
1415
1723
|
|
|
1416
|
-
/// A write-token fetch must be tagged `Operation::Write
|
|
1417
|
-
///
|
|
1418
|
-
/// fast-path instead of firing a real unauthenticated network call.
|
|
1724
|
+
/// A write-token fetch must be tagged `Operation::Write` so a
|
|
1725
|
+
/// token-less writer is rejected locally instead of hitting the network.
|
|
1419
1726
|
#[tokio::test]
|
|
1420
1727
|
async fn test_xet_upload_commit_without_token_fails_locally() -> Result<()> {
|
|
1421
1728
|
let (core, ctx, mock_client) = create_test_core(
|
|
@@ -1430,10 +1737,7 @@ mod tests {
|
|
|
1430
1737
|
Err(err) => assert_eq!(err.kind(), ErrorKind::PermissionDenied),
|
|
1431
1738
|
Ok(_) => panic!("an upload commit without a token must fail locally"),
|
|
1432
1739
|
}
|
|
1433
|
-
|
|
1434
|
-
// write-token fetch itself must be rejected before it reaches the
|
|
1435
|
-
// network.
|
|
1436
|
-
assert_eq!(mock_client.request_count(), 1);
|
|
1740
|
+
assert_eq!(mock_client.request_count(), 0);
|
|
1437
1741
|
|
|
1438
1742
|
Ok(())
|
|
1439
1743
|
}
|
|
@@ -1456,31 +1760,25 @@ mod tests {
|
|
|
1456
1760
|
.as_secs();
|
|
1457
1761
|
|
|
1458
1762
|
// Exactly at the buffer boundary: `expires_at > now + BUFFER` is
|
|
1459
|
-
// false, so this must be treated as stale.
|
|
1460
|
-
// for the one-time canonical repo id lookup.
|
|
1763
|
+
// false, so this must be treated as stale.
|
|
1461
1764
|
mock_client.set_xet_token_expires_at(now + XET_TOKEN_REFRESH_BUFFER_SECS);
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
1466
|
-
core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1467
|
-
.await?;
|
|
1765
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1766
|
+
assert_eq!(mock_client.request_count(), 1);
|
|
1767
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1468
1768
|
assert_eq!(
|
|
1469
1769
|
mock_client.request_count(),
|
|
1470
|
-
|
|
1770
|
+
2,
|
|
1471
1771
|
"a token expiring exactly at the refresh buffer must be refetched, not reused"
|
|
1472
1772
|
);
|
|
1473
1773
|
|
|
1474
1774
|
// One second past the boundary: now fresh, so cached and reused.
|
|
1475
1775
|
mock_client.set_xet_token_expires_at(now + XET_TOKEN_REFRESH_BUFFER_SECS + 1);
|
|
1476
|
-
core.cached_xet_token(&ctx,
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
core.cached_xet_token(&ctx, &repo, XetTokenScope::Read)
|
|
1480
|
-
.await?;
|
|
1776
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1777
|
+
assert_eq!(mock_client.request_count(), 3);
|
|
1778
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read).await?;
|
|
1481
1779
|
assert_eq!(
|
|
1482
1780
|
mock_client.request_count(),
|
|
1483
|
-
|
|
1781
|
+
3,
|
|
1484
1782
|
"a token expiring past the refresh buffer must be cached and reused"
|
|
1485
1783
|
);
|
|
1486
1784
|
|
|
@@ -1500,11 +1798,10 @@ mod tests {
|
|
|
1500
1798
|
);
|
|
1501
1799
|
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
1502
1800
|
|
|
1503
|
-
let repo = core.canonical_repo(&ctx).await?;
|
|
1504
1801
|
let (r1, r2, r3) = futures::join!(
|
|
1505
|
-
core.cached_xet_token(&ctx,
|
|
1506
|
-
core.cached_xet_token(&ctx,
|
|
1507
|
-
core.cached_xet_token(&ctx,
|
|
1802
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read),
|
|
1803
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read),
|
|
1804
|
+
core.cached_xet_token(&ctx, XetTokenScope::Read),
|
|
1508
1805
|
);
|
|
1509
1806
|
r1?;
|
|
1510
1807
|
r2?;
|
|
@@ -1512,9 +1809,8 @@ mod tests {
|
|
|
1512
1809
|
|
|
1513
1810
|
assert_eq!(
|
|
1514
1811
|
mock_client.request_count(),
|
|
1515
|
-
|
|
1516
|
-
"concurrent callers racing on a cold/stale token must share one refresh
|
|
1517
|
-
(plus the one-time canonical repo id lookup)"
|
|
1812
|
+
1,
|
|
1813
|
+
"concurrent callers racing on a cold/stale token must share one refresh"
|
|
1518
1814
|
);
|
|
1519
1815
|
|
|
1520
1816
|
Ok(())
|
|
@@ -1554,8 +1850,6 @@ mod tests {
|
|
|
1554
1850
|
}
|
|
1555
1851
|
}
|
|
1556
1852
|
|
|
1557
|
-
use http::StatusCode;
|
|
1558
|
-
|
|
1559
1853
|
/// Context needed to classify an error from this service.
|
|
1560
1854
|
#[derive(Clone, Copy, Debug)]
|
|
1561
1855
|
pub(crate) struct ErrorContext {
|
|
@@ -1569,16 +1863,23 @@ impl ErrorContext {
|
|
|
1569
1863
|
}
|
|
1570
1864
|
|
|
1571
1865
|
/// Parse an error response using its service request context.
|
|
1572
|
-
pub(crate) fn parse_error(ctx: ErrorContext, parts: http::response::Parts) -> Error {
|
|
1866
|
+
pub(crate) fn parse_error(ctx: ErrorContext, mut parts: http::response::Parts) -> Error {
|
|
1867
|
+
let location = HttpUri::from_response_location(&mut parts).cloned();
|
|
1573
1868
|
// HF sets x-error-message on every error response with a short human-readable
|
|
1574
1869
|
// description. Using the header avoids reading the response body, which can be
|
|
1575
1870
|
// a large HTML error page (e.g. 52 KB on 404s from the /resolve/ endpoint).
|
|
1576
|
-
let
|
|
1871
|
+
let error_message = parts
|
|
1577
1872
|
.headers
|
|
1578
1873
|
.get("x-error-message")
|
|
1579
|
-
.and_then(|v| v.to_str().ok())
|
|
1580
|
-
|
|
1581
|
-
.to_string()
|
|
1874
|
+
.and_then(|v| v.to_str().ok());
|
|
1875
|
+
let message = match (error_message, location) {
|
|
1876
|
+
(Some(message), _) => message.to_string(),
|
|
1877
|
+
(None, Some(location)) if parts.status.is_redirection() => {
|
|
1878
|
+
let location = location.redacted_uri();
|
|
1879
|
+
format!("redirect to {location} not followed")
|
|
1880
|
+
}
|
|
1881
|
+
_ => "unknown error".to_string(),
|
|
1882
|
+
};
|
|
1582
1883
|
|
|
1583
1884
|
// HF git-style commit APIs reject stale branch snapshots with 412.
|
|
1584
1885
|
// Treat this specific conflict as temporary so RetryLayer can replay
|
|
@@ -1633,7 +1934,11 @@ mod uri {
|
|
|
1633
1934
|
Self {
|
|
1634
1935
|
repo_type,
|
|
1635
1936
|
repo_id,
|
|
1636
|
-
revision
|
|
1937
|
+
// An empty revision is no revision: `hf://datasets/user/repo@`
|
|
1938
|
+
// and `revision=""` via options both reach here, and leaving
|
|
1939
|
+
// `Some("")` in place would build URLs with an empty revision
|
|
1940
|
+
// segment instead of falling back to `main`.
|
|
1941
|
+
revision: revision.filter(|revision| !revision.is_empty()),
|
|
1637
1942
|
}
|
|
1638
1943
|
}
|
|
1639
1944
|
|
|
@@ -1779,11 +2084,7 @@ mod uri {
|
|
|
1779
2084
|
}
|
|
1780
2085
|
} else if let Some((repo_id, rev)) = path.split_once('@') {
|
|
1781
2086
|
let rev = rev.replace("%2F", "/");
|
|
1782
|
-
(
|
|
1783
|
-
repo_id.to_string(),
|
|
1784
|
-
if rev.is_empty() { None } else { Some(rev) },
|
|
1785
|
-
String::new(),
|
|
1786
|
-
)
|
|
2087
|
+
(repo_id.to_string(), Some(rev), String::new())
|
|
1787
2088
|
} else {
|
|
1788
2089
|
(path, None, String::new())
|
|
1789
2090
|
};
|
|
@@ -1872,21 +2173,27 @@ mod uri {
|
|
|
1872
2173
|
recursive: bool,
|
|
1873
2174
|
cursor: Option<&str>,
|
|
1874
2175
|
) -> String {
|
|
2176
|
+
// HF answers a trailing slash with a 302 to the slash-less URL, so
|
|
2177
|
+
// the separator lives in the segment rather than the template.
|
|
2178
|
+
let path_segment = if self.path.is_empty() {
|
|
2179
|
+
String::new()
|
|
2180
|
+
} else {
|
|
2181
|
+
format!("/{}", percent_encode_path(&self.path))
|
|
2182
|
+
};
|
|
2183
|
+
|
|
1875
2184
|
let mut url = if self.repo.is_bucket() {
|
|
1876
2185
|
format!(
|
|
1877
|
-
"{}/api/buckets/{}/tree
|
|
1878
|
-
endpoint,
|
|
1879
|
-
self.repo.repo_id,
|
|
1880
|
-
percent_encode_path(&self.path),
|
|
2186
|
+
"{}/api/buckets/{}/tree{}?expand=True",
|
|
2187
|
+
endpoint, self.repo.repo_id, path_segment,
|
|
1881
2188
|
)
|
|
1882
2189
|
} else {
|
|
1883
2190
|
format!(
|
|
1884
|
-
"{}/api/{}/{}/tree/{}
|
|
2191
|
+
"{}/api/{}/{}/tree/{}{}?expand=True",
|
|
1885
2192
|
endpoint,
|
|
1886
2193
|
self.repo.repo_type.as_plural_str(),
|
|
1887
2194
|
self.repo.repo_id,
|
|
1888
2195
|
percent_encode_revision(self.revision()),
|
|
1889
|
-
|
|
2196
|
+
path_segment,
|
|
1890
2197
|
)
|
|
1891
2198
|
};
|
|
1892
2199
|
|
|
@@ -2156,6 +2463,76 @@ mod uri {
|
|
|
2156
2463
|
let url = p.repo.bucket_batch_url("https://huggingface.co");
|
|
2157
2464
|
assert_eq!(url, "https://huggingface.co/api/buckets/user/bucket/batch");
|
|
2158
2465
|
}
|
|
2466
|
+
|
|
2467
|
+
#[test]
|
|
2468
|
+
fn test_file_tree_url_root_has_no_trailing_slash() {
|
|
2469
|
+
let p = resolve("datasets/user/repo");
|
|
2470
|
+
assert_eq!(
|
|
2471
|
+
p.file_tree_url("https://huggingface.co", false, None),
|
|
2472
|
+
"https://huggingface.co/api/datasets/user/repo/tree/main?expand=True"
|
|
2473
|
+
);
|
|
2474
|
+
}
|
|
2475
|
+
|
|
2476
|
+
#[test]
|
|
2477
|
+
fn test_file_tree_url_subdir_keeps_path_separator() {
|
|
2478
|
+
let p = resolve("datasets/user/repo/data");
|
|
2479
|
+
assert_eq!(
|
|
2480
|
+
p.file_tree_url("https://huggingface.co", false, None),
|
|
2481
|
+
"https://huggingface.co/api/datasets/user/repo/tree/main/data?expand=True"
|
|
2482
|
+
);
|
|
2483
|
+
}
|
|
2484
|
+
|
|
2485
|
+
#[test]
|
|
2486
|
+
fn test_file_tree_url_encoded_revision_root() {
|
|
2487
|
+
let p = resolve("datasets/user/repo@refs/convert/parquet");
|
|
2488
|
+
assert_eq!(
|
|
2489
|
+
p.file_tree_url("https://huggingface.co", false, None),
|
|
2490
|
+
"https://huggingface.co/api/datasets/user/repo/tree/refs%2Fconvert%2Fparquet?expand=True"
|
|
2491
|
+
);
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2494
|
+
/// Every construction site funnels through `HfRepo::new`, so an empty
|
|
2495
|
+
/// revision from a URI or from `revision=""` in options is unset.
|
|
2496
|
+
#[test]
|
|
2497
|
+
fn test_empty_revision_is_unset() {
|
|
2498
|
+
assert!(resolve("datasets/user/repo@").repo.revision.is_none());
|
|
2499
|
+
assert!(
|
|
2500
|
+
HfRepo::new(
|
|
2501
|
+
HfRepoType::Dataset,
|
|
2502
|
+
"user/repo".to_string(),
|
|
2503
|
+
Some(String::new()),
|
|
2504
|
+
)
|
|
2505
|
+
.revision
|
|
2506
|
+
.is_none()
|
|
2507
|
+
);
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2510
|
+
#[test]
|
|
2511
|
+
fn test_file_tree_url_recursive_and_cursor() {
|
|
2512
|
+
let p = resolve("datasets/user/repo");
|
|
2513
|
+
assert_eq!(
|
|
2514
|
+
p.file_tree_url("https://huggingface.co", true, Some("abc123")),
|
|
2515
|
+
"https://huggingface.co/api/datasets/user/repo/tree/main?expand=True&recursive=True&cursor=abc123"
|
|
2516
|
+
);
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2519
|
+
#[test]
|
|
2520
|
+
fn test_bucket_file_tree_url_root_ends_at_tree() {
|
|
2521
|
+
let p = resolve("buckets/user/bucket");
|
|
2522
|
+
assert_eq!(
|
|
2523
|
+
p.file_tree_url("https://huggingface.co", false, None),
|
|
2524
|
+
"https://huggingface.co/api/buckets/user/bucket/tree?expand=True&recursive=false"
|
|
2525
|
+
);
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
#[test]
|
|
2529
|
+
fn test_bucket_file_tree_url_subdir_keeps_path_separator() {
|
|
2530
|
+
let p = resolve("buckets/user/bucket/data");
|
|
2531
|
+
assert_eq!(
|
|
2532
|
+
p.file_tree_url("https://huggingface.co", false, None),
|
|
2533
|
+
"https://huggingface.co/api/buckets/user/bucket/tree/data?expand=True&recursive=false"
|
|
2534
|
+
);
|
|
2535
|
+
}
|
|
2159
2536
|
}
|
|
2160
2537
|
}
|
|
2161
2538
|
|