opendal 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +24 -24
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +24 -24
- data/core/CHANGELOG.md +9 -1
- data/core/Cargo.lock +110 -110
- data/core/Cargo.toml +94 -94
- data/core/DEPENDENCIES.rust.tsv +12 -12
- data/core/benches/types/buffer.rs +61 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/CHANGELOG.md +9 -1
- data/core/core/src/raw/futures_util.rs +5 -0
- data/core/core/src/raw/oio/copy/block_copy.rs +1 -1
- data/core/core/src/raw/oio/copy/multipart_copy.rs +1 -1
- data/core/core/src/raw/oio/write/block_write.rs +1 -1
- data/core/core/src/raw/oio/write/multipart_write.rs +1 -1
- data/core/core/src/raw/oio/write/position_write.rs +41 -1
- data/core/core/src/types/buffer.rs +131 -0
- data/core/core/src/types/delete/deleter.rs +13 -0
- data/core/core/src/types/mod.rs +1 -0
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/concurrent-limit/Cargo.toml +2 -2
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/Cargo.toml +1 -1
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/services/aliyun-drive/Cargo.toml +1 -1
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azdls/Cargo.toml +2 -2
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +1 -1
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dropbox/Cargo.toml +1 -1
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/ftp/Cargo.toml +1 -1
- data/core/services/gcs/Cargo.toml +1 -1
- data/core/services/gcs-grpc/Cargo.toml +1 -1
- data/core/services/gdrive/Cargo.toml +1 -1
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +43 -16
- data/core/services/goosefs/src/config.rs +4 -1
- data/core/services/goosefs/src/docs.md +1 -1
- data/core/services/gridfs/Cargo.toml +1 -1
- data/core/services/hdfs/Cargo.toml +2 -2
- data/core/services/hdfs/src/backend.rs +15 -7
- data/core/services/hdfs/src/core.rs +55 -16
- data/core/services/hdfs/src/docs.md +9 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hf/Cargo.toml +5 -2
- data/core/services/hf/src/backend.rs +8 -3
- data/core/services/hf/src/config.rs +4 -2
- data/core/services/hf/src/docs.md +6 -2
- data/core/services/hf/src/reader.rs +163 -68
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/koofr/Cargo.toml +1 -1
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/mongodb/Cargo.toml +1 -1
- data/core/services/monoiofs/Cargo.toml +1 -1
- data/core/services/mysql/Cargo.toml +1 -1
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/onedrive/Cargo.toml +1 -1
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +143 -2
- data/core/services/oss/src/docs.md +12 -0
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/postgresql/Cargo.toml +1 -1
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/s3/Cargo.toml +1 -1
- data/core/services/seafile/Cargo.toml +1 -1
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sqlite/Cargo.toml +1 -1
- data/core/services/surrealdb/Cargo.toml +1 -1
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/tikv/Cargo.toml +1 -1
- data/core/services/tos/Cargo.toml +1 -1
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/webdav/Cargo.toml +1 -1
- data/core/services/webdav/src/backend.rs +10 -21
- data/core/services/webdav/src/config.rs +5 -14
- data/core/services/webhdfs/Cargo.toml +1 -1
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/testkit/Cargo.toml +4 -4
- metadata +2 -2
|
@@ -16,9 +16,8 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use super::backend::*;
|
|
19
|
-
#[cfg(test)]
|
|
20
19
|
use super::core::HfDownloadMode;
|
|
21
|
-
use super::core::{HfCore, HfReadResponse};
|
|
20
|
+
use super::core::{HfCore, HfReadResponse, XetFileResponse};
|
|
22
21
|
use asyncband::once::OnceCell;
|
|
23
22
|
use http::Response;
|
|
24
23
|
use opendal_core::raw::*;
|
|
@@ -103,6 +102,10 @@ pub struct HfReader {
|
|
|
103
102
|
backend: HfBackend,
|
|
104
103
|
ctx: OperationContext,
|
|
105
104
|
path: String,
|
|
105
|
+
// Keep one XET file version for this reader, independently of the backend's
|
|
106
|
+
// optional cross-reader cache. Concurrent ranges share initialization;
|
|
107
|
+
// failed resolutions remain retryable. None means the first read used HTTP.
|
|
108
|
+
resolved: OnceCell<Option<XetFileResponse>>,
|
|
106
109
|
// Built once per reader and reused for every later XET range, seeded
|
|
107
110
|
// from `HfCore`'s cached CAS token. Reuse across concurrent ranges
|
|
108
111
|
// relies on the `xet` crate's documented (not type-enforced) support
|
|
@@ -116,6 +119,7 @@ impl HfReader {
|
|
|
116
119
|
backend,
|
|
117
120
|
ctx,
|
|
118
121
|
path: path.to_string(),
|
|
122
|
+
resolved: OnceCell::new(),
|
|
119
123
|
xet_group: OnceCell::new(),
|
|
120
124
|
}
|
|
121
125
|
}
|
|
@@ -142,7 +146,33 @@ impl oio::StreamRead for HfReader {
|
|
|
142
146
|
let core = &self.backend.core;
|
|
143
147
|
let path = self.path.as_str();
|
|
144
148
|
|
|
145
|
-
let
|
|
149
|
+
let mut first_http_response = None;
|
|
150
|
+
let resolved = if core.download_mode == HfDownloadMode::Xet {
|
|
151
|
+
self.resolved
|
|
152
|
+
.get_or_try_init(async || {
|
|
153
|
+
match core.read(&self.ctx, path, range).await? {
|
|
154
|
+
HfReadResponse::Xet(info) => Ok(Some(info)),
|
|
155
|
+
HfReadResponse::Http(resp) => {
|
|
156
|
+
// This response already contains the initializing caller's
|
|
157
|
+
// range. Keep its bytes instead of probing and discarding them.
|
|
158
|
+
first_http_response = Some(resp);
|
|
159
|
+
Ok(None)
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
})
|
|
163
|
+
.await?
|
|
164
|
+
.as_ref()
|
|
165
|
+
} else {
|
|
166
|
+
None
|
|
167
|
+
};
|
|
168
|
+
let response = match resolved {
|
|
169
|
+
Some(info) => HfReadResponse::Xet(info.clone()),
|
|
170
|
+
None => match first_http_response {
|
|
171
|
+
Some(resp) => HfReadResponse::Http(resp),
|
|
172
|
+
None => core.read(&self.ctx, path, range).await?,
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
let (rp, stream) = match response {
|
|
146
176
|
HfReadResponse::Xet(info) => {
|
|
147
177
|
self.xet_stream(core, &info.hash, info.size, range).await?
|
|
148
178
|
}
|
|
@@ -313,38 +343,68 @@ mod tests {
|
|
|
313
343
|
Ok(())
|
|
314
344
|
}
|
|
315
345
|
|
|
316
|
-
/// Same guarantee as the non-XET version below, but for a file that
|
|
317
|
-
/// classifies as XET-backed: concurrent opens must share both the one
|
|
318
|
-
/// classifying resolve and the one `xet_group` build (and, via that
|
|
319
|
-
/// group, the one CAS read-token fetch) rather than each independently
|
|
320
|
-
/// racing to build its own group.
|
|
321
346
|
#[tokio::test]
|
|
322
|
-
async fn
|
|
323
|
-
|
|
347
|
+
async fn test_concurrent_xet_ranges_reuse_reader_resolution() -> Result<()> {
|
|
348
|
+
for enabled in [false, true] {
|
|
349
|
+
let (mut core, ctx, mock_client) = create_test_core(
|
|
350
|
+
HfRepoType::Model,
|
|
351
|
+
"test-user/test-repo",
|
|
352
|
+
"main",
|
|
353
|
+
"https://huggingface.co",
|
|
354
|
+
);
|
|
355
|
+
core.enable_resolve_cache = enabled;
|
|
356
|
+
mock_client.set_xet_backed(&"00".repeat(32), 64);
|
|
357
|
+
let core = Arc::new(core);
|
|
358
|
+
let reader = hf_reader_from_arc(core.clone(), ctx.clone(), "xet-file.bin");
|
|
359
|
+
|
|
360
|
+
let (r1, r2, r3) = futures::join!(
|
|
361
|
+
reader.open(BytesRange::new(0, Some(1))),
|
|
362
|
+
reader.open(BytesRange::new(1, Some(1))),
|
|
363
|
+
reader.open(BytesRange::new(2, Some(1))),
|
|
364
|
+
);
|
|
365
|
+
r1?;
|
|
366
|
+
r2?;
|
|
367
|
+
r3?;
|
|
368
|
+
// One resolve and one CAS token request; actual CAS bytes are
|
|
369
|
+
// covered by the live multiple-range test below.
|
|
370
|
+
assert_eq!(mock_client.request_count(), 2);
|
|
371
|
+
|
|
372
|
+
mock_client.set_xet_backed(&"11".repeat(32), 128);
|
|
373
|
+
let (rp, _) = reader.open(BytesRange::new(3, Some(1))).await?;
|
|
374
|
+
assert_eq!(rp.metadata().unwrap().content_length(), 64);
|
|
375
|
+
assert_eq!(
|
|
376
|
+
reader.resolved.get().unwrap().as_ref().unwrap().hash,
|
|
377
|
+
"00".repeat(32)
|
|
378
|
+
);
|
|
379
|
+
assert_eq!(mock_client.request_count(), 2);
|
|
380
|
+
|
|
381
|
+
let fresh = hf_reader_from_arc(core, ctx, "xet-file.bin");
|
|
382
|
+
let (rp, _) = fresh.open(BytesRange::new(0, Some(1))).await?;
|
|
383
|
+
assert_eq!(
|
|
384
|
+
rp.metadata().unwrap().content_length(),
|
|
385
|
+
if enabled { 64 } else { 128 }
|
|
386
|
+
);
|
|
387
|
+
assert_eq!(mock_client.request_count(), if enabled { 2 } else { 3 });
|
|
388
|
+
}
|
|
389
|
+
Ok(())
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
#[tokio::test]
|
|
393
|
+
async fn test_reader_resolution_retries_after_failure() -> Result<()> {
|
|
394
|
+
let (core, ctx, mock_client) = create_test_core(
|
|
324
395
|
HfRepoType::Model,
|
|
325
396
|
"test-user/test-repo",
|
|
326
397
|
"main",
|
|
327
398
|
"https://huggingface.co",
|
|
328
399
|
);
|
|
329
|
-
|
|
330
|
-
mock_client.
|
|
331
|
-
mock_client.set_xet_token_expires_at(u64::MAX);
|
|
400
|
+
mock_client.set_xet_backed(&"00".repeat(32), 64);
|
|
401
|
+
mock_client.fail_next_requests(1);
|
|
332
402
|
let reader = hf_reader(core, ctx, "xet-file.bin");
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
);
|
|
339
|
-
r1?;
|
|
340
|
-
r2?;
|
|
341
|
-
r3?;
|
|
342
|
-
|
|
343
|
-
// 1 shared classifying resolve + 1 shared xet-read-token fetch for
|
|
344
|
-
// the shared group build. Fetching actual bytes from the group would
|
|
345
|
-
// need a real CAS server, so this test only covers open().
|
|
346
|
-
assert_eq!(mock_client.request_count(), 2);
|
|
347
|
-
|
|
403
|
+
assert!(reader.open(BytesRange::new(0, Some(1))).await.is_err());
|
|
404
|
+
assert!(reader.resolved.get().is_none());
|
|
405
|
+
reader.open(BytesRange::new(0, Some(1))).await?;
|
|
406
|
+
reader.open(BytesRange::new(1, Some(1))).await?;
|
|
407
|
+
assert_eq!(mock_client.request_count(), 3);
|
|
348
408
|
Ok(())
|
|
349
409
|
}
|
|
350
410
|
|
|
@@ -406,33 +466,30 @@ mod tests {
|
|
|
406
466
|
Ok(())
|
|
407
467
|
}
|
|
408
468
|
|
|
409
|
-
/// Concurrent opens on a cold reader share exactly one classifying
|
|
410
|
-
/// resolve (`object_store::get_ranges` drives up to 8 by default) --
|
|
411
|
-
/// they must not each independently probe the path before finding out
|
|
412
|
-
/// it isn't XET-backed. Each still fetches its own range afterward.
|
|
413
469
|
#[tokio::test]
|
|
414
|
-
async fn
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
let (r1, r2, r3) = futures::join!(
|
|
425
|
-
reader.open(BytesRange::new(0, Some(1))),
|
|
426
|
-
reader.open(BytesRange::new(1, Some(1))),
|
|
427
|
-
reader.open(BytesRange::new(2, Some(1))),
|
|
428
|
-
);
|
|
429
|
-
r1?.1.read().await?;
|
|
430
|
-
r2?.1.read().await?;
|
|
431
|
-
r3?.1.read().await?;
|
|
432
|
-
|
|
433
|
-
// 1 shared classifying resolve + 3 individual fetches.
|
|
434
|
-
assert_eq!(mock_client.request_count(), 4);
|
|
470
|
+
async fn test_concurrent_non_xet_ranges_keep_initial_response() -> Result<()> {
|
|
471
|
+
for enabled in [false, true] {
|
|
472
|
+
let (mut core, ctx, mock_client) = create_test_core(
|
|
473
|
+
HfRepoType::Model,
|
|
474
|
+
"test-user/test-repo",
|
|
475
|
+
"main",
|
|
476
|
+
"https://huggingface.co",
|
|
477
|
+
);
|
|
478
|
+
core.enable_resolve_cache = enabled;
|
|
479
|
+
let reader = hf_reader(core, ctx, "plain.txt");
|
|
435
480
|
|
|
481
|
+
let (r1, r2, r3) = futures::join!(
|
|
482
|
+
reader.open(BytesRange::new(0, Some(1))),
|
|
483
|
+
reader.open(BytesRange::new(1, Some(1))),
|
|
484
|
+
reader.open(BytesRange::new(2, Some(1))),
|
|
485
|
+
);
|
|
486
|
+
for result in [r1, r2, r3] {
|
|
487
|
+
assert_eq!(result?.1.read().await?.to_vec(), b"hello");
|
|
488
|
+
}
|
|
489
|
+
// Each range fetches its bytes once. Only the backend cache's
|
|
490
|
+
// optional classification adds a one-byte probe.
|
|
491
|
+
assert_eq!(mock_client.request_count(), if enabled { 4 } else { 3 });
|
|
492
|
+
}
|
|
436
493
|
Ok(())
|
|
437
494
|
}
|
|
438
495
|
|
|
@@ -559,25 +616,63 @@ mod tests {
|
|
|
559
616
|
assert_eq!(&bytes, PARQUET_MAGIC);
|
|
560
617
|
}
|
|
561
618
|
|
|
562
|
-
///
|
|
563
|
-
/// the same `Reader` (mirroring `object_store::get_ranges`) must both
|
|
564
|
-
/// return correct bytes while each call resolves the XET hash and the
|
|
565
|
-
/// reader reuses its CAS download group.
|
|
619
|
+
/// Read separated ranges through one Reader against a public XET file.
|
|
566
620
|
#[tokio::test]
|
|
567
621
|
#[ignore = "requires network access"]
|
|
568
622
|
async fn test_read_xet_multiple_ranges_on_one_reader() {
|
|
569
|
-
|
|
570
|
-
let reader = op
|
|
571
|
-
.reader_with("full/train-00000-of-00001.parquet")
|
|
572
|
-
.await
|
|
573
|
-
.expect("opening a reader should succeed");
|
|
623
|
+
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
574
624
|
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
625
|
+
#[derive(Debug, Clone, Default)]
|
|
626
|
+
struct CountingTransport {
|
|
627
|
+
inner: opendal_http_transport_reqwest::ReqwestTransport,
|
|
628
|
+
resolves: Arc<AtomicUsize>,
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
impl HttpTransport for CountingTransport {
|
|
632
|
+
async fn fetch(&self, req: http::Request<Buffer>) -> Result<Response<HttpBody>> {
|
|
633
|
+
if req.uri().path().contains("/resolve/") {
|
|
634
|
+
self.resolves.fetch_add(1, Ordering::SeqCst);
|
|
635
|
+
}
|
|
636
|
+
self.inner.fetch(req).await
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
let transport = CountingTransport::default();
|
|
641
|
+
let op = Operator::new(
|
|
642
|
+
super::super::backend::HfBuilder::default()
|
|
643
|
+
.repo_type("dataset")
|
|
644
|
+
.repo_id("google-research-datasets/mbpp")
|
|
645
|
+
.download_mode("xet"),
|
|
646
|
+
)
|
|
647
|
+
.unwrap()
|
|
648
|
+
.with_context(
|
|
649
|
+
OperationContext::new().with_http_transport(HttpTransporter::new(transport.clone())),
|
|
650
|
+
);
|
|
651
|
+
let path = "full/train-00000-of-00001.parquet";
|
|
652
|
+
let reader = op.reader_with(path).concurrent(3).gap(0).await.unwrap();
|
|
653
|
+
let ranges = vec![0..4, 16..20, 32..36];
|
|
654
|
+
let bufs = reader.fetch(ranges.clone()).await.unwrap();
|
|
580
655
|
assert_eq!(bufs[0].to_vec(), PARQUET_MAGIC);
|
|
581
|
-
|
|
656
|
+
assert!(bufs.iter().all(|buf| buf.len() == 4));
|
|
657
|
+
let first_resolves = transport.resolves.load(Ordering::SeqCst);
|
|
658
|
+
let repeated = reader.fetch(ranges.clone()).await.unwrap();
|
|
659
|
+
for (first, second) in bufs.iter().zip(repeated) {
|
|
660
|
+
assert_eq!(first.to_vec(), second.to_vec());
|
|
661
|
+
}
|
|
662
|
+
let repeated_resolves = transport.resolves.load(Ordering::SeqCst);
|
|
663
|
+
let fresh = op.read_with(path).range(0..36).await.unwrap().to_vec();
|
|
664
|
+
for (range, buf) in ranges.iter().zip(&bufs) {
|
|
665
|
+
assert_eq!(
|
|
666
|
+
buf.to_vec(),
|
|
667
|
+
fresh[range.start as usize..range.end as usize]
|
|
668
|
+
);
|
|
669
|
+
}
|
|
670
|
+
let fresh_resolves = transport.resolves.load(Ordering::SeqCst);
|
|
671
|
+
eprintln!(
|
|
672
|
+
"resolve counts: first batch={first_resolves}, repeated batch={repeated_resolves}, fresh reader={fresh_resolves}"
|
|
673
|
+
);
|
|
674
|
+
assert_eq!(first_resolves, 1);
|
|
675
|
+
assert_eq!(repeated_resolves, 1);
|
|
676
|
+
assert_eq!(fresh_resolves, 2);
|
|
582
677
|
}
|
|
583
678
|
}
|
|
@@ -22,7 +22,7 @@ license = "Apache-2.0"
|
|
|
22
22
|
name = "opendal-service-http"
|
|
23
23
|
readme = "README.md"
|
|
24
24
|
repository = "https://github.com/apache/opendal"
|
|
25
|
-
version = "0.59.
|
|
25
|
+
version = "0.59.4"
|
|
26
26
|
|
|
27
27
|
[package.metadata.docs.rs]
|
|
28
28
|
all-features = true
|
|
@@ -30,9 +30,9 @@ all-features = true
|
|
|
30
30
|
[dependencies]
|
|
31
31
|
http = "1.4"
|
|
32
32
|
log = "0.4"
|
|
33
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
33
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
34
34
|
serde = { version = "1", features = ["derive"] }
|
|
35
35
|
|
|
36
36
|
[dev-dependencies]
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
38
38
|
tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] }
|
|
@@ -35,7 +35,7 @@ all-features = true
|
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
37
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
39
39
|
prost = "0.14.3"
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
|
|
@@ -32,7 +32,7 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
35
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
36
36
|
|
|
37
37
|
bytes = { workspace = true }
|
|
38
38
|
http = { workspace = true }
|
|
@@ -36,7 +36,7 @@ asyncband = { workspace = true, features = ["mutex", "once-cell"] }
|
|
|
36
36
|
bytes = { workspace = true }
|
|
37
37
|
http = { workspace = true }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
serde_json = { workspace = true }
|
|
42
42
|
|
|
@@ -35,7 +35,7 @@ all-features = true
|
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
37
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
39
39
|
serde = { workspace = true, features = ["derive"] }
|
|
40
40
|
serde_json = { workspace = true }
|
|
41
41
|
|
|
@@ -33,7 +33,7 @@ all-features = true
|
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
asyncband = { workspace = true, features = ["pool"] }
|
|
36
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
36
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
37
37
|
serde = { workspace = true, features = ["derive"] }
|
|
38
38
|
tokio = { workspace = true, features = ["io-util", "macros", "net", "time"] }
|
|
39
39
|
url = { workspace = true }
|
|
@@ -34,5 +34,5 @@ all-features = true
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
log = { workspace = true }
|
|
36
36
|
mini-moka = "0.10"
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
38
38
|
serde = { workspace = true, features = ["derive"] }
|
|
@@ -34,7 +34,7 @@ all-features = true
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
log = { workspace = true }
|
|
36
36
|
moka = { version = "0.12", features = ["future", "sync"] }
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
38
38
|
serde = { workspace = true, features = ["derive"] }
|
|
39
39
|
|
|
40
40
|
[dev-dependencies]
|
|
@@ -34,7 +34,7 @@ all-features = true
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
asyncband = { workspace = true, features = ["once-cell"] }
|
|
36
36
|
mongodb = { version = "3.3.0" }
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
38
38
|
serde = { workspace = true, features = ["derive"] }
|
|
39
39
|
|
|
40
40
|
[dev-dependencies]
|
|
@@ -41,7 +41,7 @@ monoio = { version = "0.2.4", features = [
|
|
|
41
41
|
"unlinkat",
|
|
42
42
|
"renameat",
|
|
43
43
|
] }
|
|
44
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
44
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
45
45
|
serde = { workspace = true, features = ["derive"] }
|
|
46
46
|
|
|
47
47
|
[dev-dependencies]
|
|
@@ -33,7 +33,7 @@ all-features = true
|
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
asyncband = { workspace = true, features = ["once-cell"] }
|
|
36
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
36
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
37
37
|
serde = { workspace = true, features = ["derive"] }
|
|
38
38
|
sqlx = { version = "0.8.0", features = ["runtime-tokio-rustls", "mysql"] }
|
|
39
39
|
|
|
@@ -35,7 +35,7 @@ all-features = true
|
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
37
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false, features = [
|
|
39
39
|
"reqsign",
|
|
40
40
|
] }
|
|
41
41
|
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
|
|
@@ -36,7 +36,7 @@ asyncband = { workspace = true, features = ["mutex"] }
|
|
|
36
36
|
bytes = { workspace = true }
|
|
37
37
|
http = { workspace = true }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
serde_json = { workspace = true }
|
|
42
42
|
tokio = { workspace = true, features = ["time"] }
|
|
@@ -35,7 +35,7 @@ targets = ["wasm32-unknown-unknown"]
|
|
|
35
35
|
|
|
36
36
|
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
|
37
37
|
js-sys = "0.3.77"
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
39
39
|
send_wrapper = "0.6"
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
wasm-bindgen = "0.2.100"
|
|
@@ -35,7 +35,7 @@ all-features = true
|
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
37
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false, features = [
|
|
39
39
|
"reqsign",
|
|
40
40
|
] }
|
|
41
41
|
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
|
|
@@ -523,8 +523,8 @@ impl Builder for OssBuilder {
|
|
|
523
523
|
}
|
|
524
524
|
let mut provider = provider
|
|
525
525
|
.push(EnvCredentialProvider::new())
|
|
526
|
-
.push(
|
|
527
|
-
.push(
|
|
526
|
+
.push(assume_role)
|
|
527
|
+
.push(EcsRamRoleCredentialProvider::new());
|
|
528
528
|
|
|
529
529
|
if let Some(role_arn) = &self.config.role_arn {
|
|
530
530
|
let mut assume_role_with_ak = AssumeRoleCredentialProvider::new()
|
|
@@ -859,3 +859,144 @@ impl Service for OssBackend {
|
|
|
859
859
|
)))
|
|
860
860
|
}
|
|
861
861
|
}
|
|
862
|
+
|
|
863
|
+
#[cfg(test)]
|
|
864
|
+
mod tests {
|
|
865
|
+
use std::collections::HashMap;
|
|
866
|
+
use std::sync::Mutex;
|
|
867
|
+
|
|
868
|
+
use bytes::Bytes;
|
|
869
|
+
use reqsign_core::{FileRead, HttpSend};
|
|
870
|
+
|
|
871
|
+
use super::*;
|
|
872
|
+
|
|
873
|
+
#[derive(Debug)]
|
|
874
|
+
struct OidcTokenFile;
|
|
875
|
+
|
|
876
|
+
impl FileRead for OidcTokenFile {
|
|
877
|
+
async fn file_read(&self, path: &str) -> reqsign_core::Result<Vec<u8>> {
|
|
878
|
+
assert_eq!(path, "/test/oidc-token");
|
|
879
|
+
Ok(b"test-oidc-token".to_vec())
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
#[derive(Clone, Debug, Default)]
|
|
884
|
+
struct CredentialHttpSend {
|
|
885
|
+
requests: Arc<Mutex<Vec<String>>>,
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
impl HttpSend for CredentialHttpSend {
|
|
889
|
+
async fn http_send(
|
|
890
|
+
&self,
|
|
891
|
+
req: http::Request<Bytes>,
|
|
892
|
+
) -> reqsign_core::Result<http::Response<Bytes>> {
|
|
893
|
+
let host = req.uri().host().unwrap();
|
|
894
|
+
let path = req.uri().path();
|
|
895
|
+
self.requests.lock().unwrap().push(format!("{host}{path}"));
|
|
896
|
+
let body = match (host, path) {
|
|
897
|
+
("sts.aliyuncs.com", "/") => {
|
|
898
|
+
assert!(
|
|
899
|
+
req.uri()
|
|
900
|
+
.query()
|
|
901
|
+
.unwrap()
|
|
902
|
+
.contains("Action=AssumeRoleWithOIDC")
|
|
903
|
+
);
|
|
904
|
+
r#"{"Credentials":{"AccessKeyId":"oidc-ak","AccessKeySecret":"oidc-secret","SecurityToken":"oidc-token","Expiration":"2099-01-01T00:00:00Z"}}"#
|
|
905
|
+
}
|
|
906
|
+
("100.100.100.200", "/latest/api/token") => "metadata-token",
|
|
907
|
+
("100.100.100.200", "/latest/meta-data/ram/security-credentials/") => "node-role",
|
|
908
|
+
("100.100.100.200", "/latest/meta-data/ram/security-credentials/node-role") => {
|
|
909
|
+
r#"{"Code":"Success","AccessKeyId":"ecs-ak","AccessKeySecret":"ecs-secret","SecurityToken":"ecs-token","Expiration":"2099-01-01T00:00:00Z"}"#
|
|
910
|
+
}
|
|
911
|
+
_ => panic!("unexpected credential request: {}", req.uri()),
|
|
912
|
+
};
|
|
913
|
+
Ok(http::Response::new(Bytes::from_static(body.as_bytes())))
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
#[tokio::test]
|
|
918
|
+
async fn test_credential_precedence() {
|
|
919
|
+
for (static_keys, env_keys, oidc, expected_key, expected_token) in [
|
|
920
|
+
(true, true, true, "static-ak", "static-token"),
|
|
921
|
+
(false, true, true, "env-ak", "env-token"),
|
|
922
|
+
(false, false, true, "oidc-ak", "oidc-token"),
|
|
923
|
+
(false, false, false, "ecs-ak", "ecs-token"),
|
|
924
|
+
] {
|
|
925
|
+
let mut builder = OssBuilder::default()
|
|
926
|
+
.bucket("test-bucket")
|
|
927
|
+
.endpoint("https://oss-cn-hangzhou.aliyuncs.com");
|
|
928
|
+
if static_keys {
|
|
929
|
+
builder = builder
|
|
930
|
+
.access_key_id("static-ak")
|
|
931
|
+
.access_key_secret("static-secret")
|
|
932
|
+
.security_token("static-token");
|
|
933
|
+
}
|
|
934
|
+
let backend = builder.build().unwrap();
|
|
935
|
+
let backend = (&backend as &dyn std::any::Any)
|
|
936
|
+
.downcast_ref::<OssBackend>()
|
|
937
|
+
.unwrap();
|
|
938
|
+
let mut envs = HashMap::new();
|
|
939
|
+
if env_keys {
|
|
940
|
+
envs.extend([
|
|
941
|
+
("ALIBABA_CLOUD_ACCESS_KEY_ID".into(), "env-ak".into()),
|
|
942
|
+
(
|
|
943
|
+
"ALIBABA_CLOUD_ACCESS_KEY_SECRET".into(),
|
|
944
|
+
"env-secret".into(),
|
|
945
|
+
),
|
|
946
|
+
("ALIBABA_CLOUD_SECURITY_TOKEN".into(), "env-token".into()),
|
|
947
|
+
]);
|
|
948
|
+
}
|
|
949
|
+
if oidc {
|
|
950
|
+
envs.extend([
|
|
951
|
+
(
|
|
952
|
+
"ALIBABA_CLOUD_ROLE_ARN".into(),
|
|
953
|
+
"acs:ram::123456789012:role/pod-role".into(),
|
|
954
|
+
),
|
|
955
|
+
(
|
|
956
|
+
"ALIBABA_CLOUD_OIDC_PROVIDER_ARN".into(),
|
|
957
|
+
"acs:ram::123456789012:oidc-provider/test".into(),
|
|
958
|
+
),
|
|
959
|
+
(
|
|
960
|
+
"ALIBABA_CLOUD_OIDC_TOKEN_FILE".into(),
|
|
961
|
+
"/test/oidc-token".into(),
|
|
962
|
+
),
|
|
963
|
+
]);
|
|
964
|
+
}
|
|
965
|
+
let http = CredentialHttpSend::default();
|
|
966
|
+
let ctx = Context::new()
|
|
967
|
+
.with_env(StaticEnv {
|
|
968
|
+
home_dir: None,
|
|
969
|
+
envs,
|
|
970
|
+
})
|
|
971
|
+
.with_file_read(OidcTokenFile)
|
|
972
|
+
.with_http_send(http.clone());
|
|
973
|
+
let signer = backend.core.signer.clone().with_context(ctx);
|
|
974
|
+
let (mut parts, _) = http::Request::builder()
|
|
975
|
+
.method(http::Method::PUT)
|
|
976
|
+
.uri("https://test-bucket.oss-cn-hangzhou.aliyuncs.com/manifest.json")
|
|
977
|
+
.body(())
|
|
978
|
+
.unwrap()
|
|
979
|
+
.into_parts();
|
|
980
|
+
signer.sign(&mut parts, None).await.unwrap();
|
|
981
|
+
assert!(
|
|
982
|
+
parts.headers[http::header::AUTHORIZATION]
|
|
983
|
+
.to_str()
|
|
984
|
+
.unwrap()
|
|
985
|
+
.starts_with(&format!("OSS {expected_key}:"))
|
|
986
|
+
);
|
|
987
|
+
assert_eq!(parts.headers["x-oss-security-token"], expected_token);
|
|
988
|
+
let expected_requests = if static_keys || env_keys {
|
|
989
|
+
vec![]
|
|
990
|
+
} else if oidc {
|
|
991
|
+
vec!["sts.aliyuncs.com/"]
|
|
992
|
+
} else {
|
|
993
|
+
vec![
|
|
994
|
+
"100.100.100.200/latest/api/token",
|
|
995
|
+
"100.100.100.200/latest/meta-data/ram/security-credentials/",
|
|
996
|
+
"100.100.100.200/latest/meta-data/ram/security-credentials/node-role",
|
|
997
|
+
]
|
|
998
|
+
};
|
|
999
|
+
assert_eq!(*http.requests.lock().unwrap(), expected_requests);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
@@ -21,6 +21,18 @@ Use [`crate::OssConfig`] for serializable configuration and this builder's
|
|
|
21
21
|
methods for direct construction. The field and method documentation defines
|
|
22
22
|
accepted values, defaults, and environment interaction.
|
|
23
23
|
|
|
24
|
+
### Credential resolution
|
|
25
|
+
|
|
26
|
+
OSS tries explicit access keys, environment credentials, OIDC assume-role
|
|
27
|
+
credentials, and ECS instance RAM role credentials, in that order. The first
|
|
28
|
+
provider that returns credentials is used. In ACK RRSA deployments, OIDC
|
|
29
|
+
credentials therefore take precedence over the node's ECS metadata credentials.
|
|
30
|
+
If a provider returns no credentials or fails, the chain tries the next provider.
|
|
31
|
+
An OSS permission error does not trigger a switch to another credential provider.
|
|
32
|
+
|
|
33
|
+
When `role_arn` is set in the service configuration, these providers supply the
|
|
34
|
+
base credentials for the additional AK-based AssumeRole request.
|
|
35
|
+
|
|
24
36
|
## Example
|
|
25
37
|
|
|
26
38
|
### Via Builder
|
|
@@ -35,7 +35,7 @@ all-features = true
|
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
37
|
log = { workspace = true }
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
39
39
|
serde = { workspace = true, features = ["derive"] }
|
|
40
40
|
serde_json = { workspace = true }
|
|
41
41
|
|
|
@@ -32,7 +32,7 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
35
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
36
36
|
persy = "1.7.1"
|
|
37
37
|
serde = { workspace = true, features = ["derive"] }
|
|
38
38
|
|
|
@@ -33,7 +33,7 @@ all-features = true
|
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
asyncband = { workspace = true, features = ["once-cell"] }
|
|
36
|
-
opendal-core = { path = "../../core", version = "0.59.
|
|
36
|
+
opendal-core = { path = "../../core", version = "0.59.4", default-features = false }
|
|
37
37
|
serde = { workspace = true, features = ["derive"] }
|
|
38
38
|
sqlx = { version = "0.8.0", features = ["runtime-tokio-rustls", "postgres"] }
|
|
39
39
|
|