opendal 0.1.11 → 0.1.13

data/core/services/goosefs/src/reader.rs

@@ -16,94 +16,25 @@
 // under the License.
 
 use super::backend::*;
-use super::core::GoosefsCore;
 use super::core::{ErrorContext, parse_error};
 use goosefs_sdk::io::GoosefsFileReader as SdkReader;
 use opendal_core::raw::*;
 use opendal_core::*;
-use std::sync::Arc;
 
 pub struct GoosefsReadStream {
-    core: Arc<GoosefsCore>,
-    path: String,
-    range: BytesRange,
-    content_length: Option<u64>,
-    /// Lazily-opened SDK reader. `None` until the first `read()` call.
-    inner: Option<SdkReader>,
-    /// Terminal flag: once the underlying stream has returned `None`,
-    /// subsequent `read()` calls must keep returning an empty `Buffer`
-    /// without re-entering the SDK (which would try to open a fresh
-    /// stream and re-read the file).
+    inner: SdkReader,
+    // The SDK may reopen its stream after EOF; keep OpenDAL's EOF terminal.
     done: bool,
 }
 
-impl GoosefsReadStream {
-    pub fn new(
-        core: Arc<GoosefsCore>,
-        path: String,
-        range: BytesRange,
-        content_length: Option<u64>,
-    ) -> Self {
-        GoosefsReadStream {
-            core,
-            path,
-            range,
-            content_length,
-            inner: None,
-            done: false,
-        }
-    }
-
-    /// Open the underlying SDK reader, picking between the full-file
-    /// and ranged variants based on `self.range`.
-    async fn open(&self) -> Result<SdkReader> {
-        let offset = self.range.offset();
-        let size = self.range.size();
-
-        // Three cases:
-        //   1. No offset and no size      → full-file stream.
-        //   2. Offset+size both set       → ranged stream.
-        //   3. Offset set, size unbounded → resolve tail length via
-        //      get_status and fall through to the ranged opener, so we
-        //      still get per-block streaming (rather than buffering the
-        //      whole tail in one call).
-        match (offset, size) {
-            (0, None) => self.core.open_reader(&self.path).await,
-            (off, Some(len)) => self.core.open_range_reader(&self.path, off, len).await,
-            (off, None) => {
-                let content_length = self.content_length.ok_or_else(|| {
-                    Error::new(
-                        ErrorKind::Unexpected,
-                        "content length must be known for offset reads",
-                    )
-                })?;
-                let len = content_length.saturating_sub(off);
-                if len == 0 {
-                    // Empty tail — short-circuit with a zero-length
-                    // ranged open so the very next `read_next_block`
-                    // call returns `None` and we EOF cleanly.
-                    self.core.open_range_reader(&self.path, off, 0).await
-                } else {
-                    self.core.open_range_reader(&self.path, off, len).await
-                }
-            }
-        }
-    }
-}
-
 impl oio::ReadStream for GoosefsReadStream {
     async fn read(&mut self) -> Result<Buffer> {
         if self.done {
             return Ok(Buffer::new());
         }
 
-        // Lazy open on first call.
-        if self.inner.is_none() {
-            self.inner = Some(self.open().await?);
-        }
-        let reader = self.inner.as_mut().expect("inner was just set");
-
-        match reader
+        match self
+            .inner
             .read_next_block()
             .await
             .map_err(|err| parse_error(ErrorContext::new(ServiceOperation("ReadNextBlock")), err))?
@@ -137,25 +68,27 @@ impl oio::StreamRead for GoosefsReader {
         let backend = &self.backend;
         let path = self.path.as_str();
 
-        let content_length = if range.offset() != 0 && range.size().is_none() {
-            let file_info = backend.core.get_status(path).await?;
-            Some(
-                backend
+        let inner = match (range.offset(), range.size()) {
+            (0, None) => backend.core.open_reader(path).await?,
+            (offset, Some(length)) => backend.core.open_range_reader(path, offset, length).await?,
+            (offset, None) => {
+                let file_info = backend.core.get_status(path).await?;
+                let content_length = backend
                     .core
                     .file_info_to_metadata(&file_info)?
-                    .content_length(),
-            )
-        } else {
-            None
+                    .content_length();
+                backend
+                    .core
+                    .open_range_reader(path, offset, content_length.saturating_sub(offset))
+                    .await?
+            }
         };
-        let rp = RpRead::default();
-        let stream = GoosefsReadStream::new(
-            backend.core.clone(),
-            path.to_string(),
-            range,
-            content_length,
-        );
+        let metadata = backend.core.file_info_to_metadata(inner.file_info())?;
+        let stream = GoosefsReadStream { inner, done: false };
 
-        Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
+        Ok((
+            RpRead::new(metadata),
+            Box::new(stream) as Box<dyn oio::ReadStreamDyn>,
+        ))
     }
 }

data/core/services/goosefs/tests/master_addr_resolution.rs

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! End-to-end master-address resolution against a live GooseFS master.
+//!
+//! Requires a live GooseFS master. Skip unless `OPENDAL_GOOSEFS_MASTER_ADDR`
+//! is set (same env the behavior CI fixture exports).
+//!
+//! The whole file is one test because it mutates process-global `GOOSEFS_*`
+//! environment variables.
+
+use std::time::Duration;
+
+use opendal::Operator;
+
+/// A dummy address from TEST-NET-1 (RFC 5737): guaranteed not to host a
+/// GooseFS master.
+const DUMMY_AUTHORITY: &str = "192.0.2.5:9999";
+
+/// Bound for an operation expected to fail against [`DUMMY_AUTHORITY`]. The
+/// SDK retries master inquiry for far longer than a test should wait, so a
+/// timeout counts as "did not reach a master".
+const DIAL_FAILURE_TIMEOUT: Duration = Duration::from_secs(20);
+
+/// `goosefs-site.properties` supplies the master address even when the URI
+/// authority points somewhere unreachable.
+///
+/// Reported failure: a client configured through `$GOOSEFS_CONFIG_FILE` still
+/// dialed the URI authority and failed with `gRPC transport error`, so a
+/// deployment's HA master list never took effect.
+#[tokio::test]
+async fn site_properties_supply_the_master_address() {
+    let Ok(master) = std::env::var("OPENDAL_GOOSEFS_MASTER_ADDR") else {
+        eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR unset");
+        return;
+    };
+    if master.trim().is_empty() {
+        eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR is empty");
+        return;
+    }
+    let root = std::env::var("OPENDAL_GOOSEFS_ROOT").unwrap_or_else(|_| "/".to_string());
+    let uri = format!("goosefs://{DUMMY_AUTHORITY}{root}");
+
+    // `GOOSEFS_MASTER_ADDR` outranks both sources under test, and a site file
+    // deployed on the host would decide the control case below.
+    unsafe {
+        std::env::remove_var("GOOSEFS_MASTER_ADDR");
+        std::env::set_var(
+            "GOOSEFS_CONFIG_FILE",
+            "/nonexistent/goosefs-site.properties",
+        );
+    }
+
+    // Control: without a site file the URI authority is used, and writing
+    // through it must not reach any master.
+    let op = Operator::from_uri(uri.as_str()).expect("build operator from dummy URI");
+    let control = tokio::time::timeout(DIAL_FAILURE_TIMEOUT, op.write("resolution-control", "x"))
+        .await
+        .map_err(|_| "timed out");
+    assert!(
+        !matches!(control, Ok(Ok(_))),
+        "{DUMMY_AUTHORITY} must not serve a master, otherwise this test proves nothing"
+    );
+
+    // A site file that lists the live master must win over the URI authority.
+    let site_file = std::env::temp_dir().join(format!(
+        "opendal_goosefs_resolution_{}_site.properties",
+        std::process::id()
+    ));
+    std::fs::write(
+        &site_file,
+        format!("goosefs.master.rpc.addresses={master}\n"),
+    )
+    .expect("write goosefs-site.properties");
+    unsafe { std::env::set_var("GOOSEFS_CONFIG_FILE", &site_file) };
+
+    let op =
+        Operator::from_uri(uri.as_str()).expect("build operator from dummy URI with a site file");
+    let path = format!("resolution-{}", std::process::id());
+    op.write(&path, "site-properties-win")
+        .await
+        .expect("write must reach the master listed in goosefs-site.properties");
+    let body = op.read(&path).await.expect("read back").to_bytes();
+    assert_eq!(&body[..], b"site-properties-win");
+
+    let _ = op.delete(&path).await;
+    let _ = std::fs::remove_file(&site_file);
+    unsafe { std::env::remove_var("GOOSEFS_CONFIG_FILE") };
+}

data/core/services/gridfs/Cargo.toml

@@ -32,10 +32,10 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-asyncband = { workspace = true }
+asyncband = { workspace = true, features = ["once-cell"] }
 futures = { workspace = true }
 mongodb = "3.3.0"
-opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]

data/core/services/gridfs/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hdfs/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 
 bytes = { workspace = true }
 futures = { workspace = true }

data/core/services/hdfs/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hdfs-native/Cargo.toml

@@ -36,5 +36,5 @@ bytes = { workspace = true }
 futures = { workspace = true }
 hdfs-native = { version = "0.14" }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 serde = { workspace = true, features = ["derive"] }

data/core/services/hdfs-native/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hf/Cargo.toml

@@ -32,12 +32,12 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-asyncband = { workspace = true }
+asyncband = { workspace = true, features = ["mutex", "once-cell"] }
 bytes = { workspace = true }
 hf-xet = "1.6.0"
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 percent-encoding = "2"
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
@@ -45,6 +45,6 @@ serde_json = { workspace = true }
 [dev-dependencies]
 base64 = { workspace = true }
 futures = { workspace = true }
-opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.0" }
+opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.2" }
 serde_json = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

data/core/services/hf/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hf/src/backend.rs

@@ -133,6 +133,22 @@ impl HfBuilder {
         self
     }
 
+    /// Enable caching of resolved HTTP download addresses and XET file metadata.
+    ///
+    /// Defaults to `false`: every range resolves through the Hub in either
+    /// download mode. Set to `true` to share resolve results across readers on
+    /// the same backend. HTTP addresses refresh near expiry.
+    ///
+    /// Enable this only when previously written files are not modified. Changed
+    /// files can remain invisible while cached results are reused, including
+    /// changes from other clients or a floating repository revision. Issued
+    /// download URLs can remain usable until expiry after Hub permissions change.
+    /// Separate authorization identities must use separately constructed backends.
+    pub fn enable_resolve_cache(mut self, enabled: bool) -> Self {
+        self.config.enable_resolve_cache = enabled;
+        self
+    }
+
     /// Set the Hub base URL.
     ///
     /// Configure this when your organization uses a
@@ -146,12 +162,18 @@ impl HfBuilder {
         self
     }
 
+    /// Resolve the Hub base URL: an explicit config value wins, then
+    /// `HF_ENDPOINT`, then the public Hub. A trailing slash is trimmed
+    /// because every URL is built by appending `/api/...` to this, and HF
+    /// answers the resulting `//api/...` with a 404.
     fn hf_endpoint(&self) -> String {
         self.config
             .endpoint
             .clone()
             .or_else(|| std::env::var("HF_ENDPOINT").ok())
             .unwrap_or_else(|| "https://huggingface.co".to_string())
+            .trim_end_matches('/')
+            .to_string()
     }
 
     /// Resolve the download mode: an explicit config value wins; otherwise a set,
@@ -260,16 +282,10 @@ impl Builder for HfBuilder {
         let repo = HfRepo::new(repo_type, repo_id, Some(revision.clone()));
         debug!("backend repo uri: {:?}", repo.uri(&root, ""));
 
+        let mut core = HfCore::build(info, capability, repo, root, token, endpoint, download_mode)?;
+        core.enable_resolve_cache = self.config.enable_resolve_cache;
         Ok(HfBackend {
-            core: Arc::new(HfCore::build(
-                info,
-                capability,
-                repo,
-                root,
-                token,
-                endpoint,
-                download_mode,
-            )?),
+            core: Arc::new(core),
         })
     }
 }
@@ -435,6 +451,19 @@ pub(super) mod test_utils {
         finish_operator(op)
     }
 
+    /// Same public dataset as [`mbpp_operator`], but with the repo id cased
+    /// differently from the canonical `google-research-datasets/mbpp`. HF
+    /// answers every request for it with a `307` to the canonical URL.
+    pub fn miscased_mbpp_operator() -> Operator {
+        let op = Operator::new(
+            HfBuilder::default()
+                .repo_type("dataset")
+                .repo_id("Google-Research-Datasets/MBPP"),
+        )
+        .unwrap();
+        finish_operator(op)
+    }
+
     pub fn testing_dataset_core() -> Arc<HfCore> {
         let repo_id = std::env::var("HF_OPENDAL_DATASET").expect("HF_OPENDAL_DATASET must be set");
         let token = std::env::var("HF_OPENDAL_TOKEN").expect("HF_OPENDAL_TOKEN must be set");
@@ -538,6 +567,18 @@ mod tests {
         assert_eq!(result.as_deref(), Some("file-token"));
     }
 
+    #[test]
+    fn hf_endpoint_trims_trailing_slash() {
+        let _guard = ENV_LOCK.lock().unwrap();
+        unsafe { std::env::remove_var("HF_ENDPOINT") };
+        assert_eq!(
+            HfBuilder::default()
+                .endpoint("https://hub.example.com/")
+                .hf_endpoint(),
+            "https://hub.example.com"
+        );
+    }
+
     #[test]
     fn hf_endpoint_returns_default() {
         let _guard = ENV_LOCK.lock().unwrap();
@@ -600,6 +641,33 @@ mod tests {
         assert_eq!(mode, HfDownloadMode::Xet);
     }
 
+    #[tokio::test]
+    async fn build_resolve_cache_requires_opt_in() -> Result<()> {
+        use super::super::core::test_utils::MockHttpTransport;
+
+        for enabled in [None, Some(false), Some(true)] {
+            let mut builder = HfBuilder::default()
+                .repo_type("model")
+                .repo_id("org/repo")
+                .download_mode("xet");
+            if let Some(enabled) = enabled {
+                builder = builder.enable_resolve_cache(enabled);
+            }
+            let transport = MockHttpTransport::new();
+            let ctx = OperationContext::new()
+                .with_http_transport(HttpTransporter::new(transport.clone()));
+            let op = Operator::new(builder)?.with_context(ctx);
+            for reads in 1..=2 {
+                assert_eq!(op.read("plain.txt").await?.to_vec(), b"hello");
+                assert_eq!(
+                    transport.request_count(),
+                    reads + usize::from(enabled == Some(true))
+                );
+            }
+        }
+        Ok(())
+    }
+
     #[test]
     fn build_accepts_datasets_alias() {
         HfBuilder::default()

data/core/services/hf/src/config.rs

@@ -60,6 +60,13 @@ pub struct HfConfig {
     ///
     /// See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
     pub download_mode: Option<HfDownloadMode>,
+    /// Enable caching of resolved HTTP download addresses and XET file metadata.
+    ///
+    /// Defaults to `false`. Set to `true` to share resolve results across readers
+    /// on the same backend. Changed files may remain invisible while cached
+    /// results are reused. See [`HfBuilder::enable_resolve_cache`] for freshness
+    /// semantics.
+    pub enable_resolve_cache: bool,
 }
 
 impl Debug for HfConfig {
@@ -73,6 +80,7 @@ impl Debug for HfConfig {
             .field("revision", &self.revision)
             .field("root", &self.root)
             .field("download_mode", &self.download_mode)
+            .field("enable_resolve_cache", &self.enable_resolve_cache)
             .finish_non_exhaustive()
     }
 }
@@ -106,6 +114,19 @@ impl opendal_core::Configurator for HfConfig {
             .map(|s| HfDownloadMode::parse(s))
             .transpose()?;
 
+        let enable_resolve_cache = opts
+            .get("enable_resolve_cache")
+            .map(|value| value.parse::<bool>())
+            .transpose()
+            .map_err(|err| {
+                opendal_core::Error::new(
+                    opendal_core::ErrorKind::ConfigInvalid,
+                    "enable_resolve_cache must be true or false",
+                )
+                .set_source(err)
+            })?
+            .unwrap_or_default();
+
         if !path.is_empty() {
             // Full URI like "hf://datasets/user/repo@rev/path"
             let parsed = HfUri::parse(&path)?;
@@ -117,6 +138,7 @@ impl opendal_core::Configurator for HfConfig {
                 token: opts.get("token").cloned(),
                 endpoint: opts.get("endpoint").cloned(),
                 download_mode,
+                enable_resolve_cache,
             })
         } else {
             // Bare scheme from via_iter, all config is in options.
@@ -138,6 +160,7 @@ impl opendal_core::Configurator for HfConfig {
                 token: opts.get("token").cloned(),
                 endpoint: opts.get("endpoint").cloned(),
                 download_mode,
+                enable_resolve_cache,
             })
         }
     }
@@ -222,4 +245,39 @@ mod tests {
         let cfg = HfConfig::from_uri(&uri).unwrap();
         assert_eq!(cfg.download_mode.unwrap_or_default(), HfDownloadMode::Xet);
     }
+
+    #[test]
+    fn from_uri_resolve_cache_options() {
+        for scheme in ["hf://datasets/user/repo", "huggingface"] {
+            for (value, expected) in [(None, false), (Some("false"), false), (Some("true"), true)] {
+                let mut options = vec![("repo_type", "dataset"), ("repo_id", "user/repo")];
+                if let Some(value) = value {
+                    options.push(("enable_resolve_cache", value));
+                }
+                let uri = OperatorUri::new(
+                    scheme,
+                    options
+                        .into_iter()
+                        .map(|(key, value)| (key.to_string(), value.to_string())),
+                )
+                .unwrap();
+                assert_eq!(
+                    HfConfig::from_uri(&uri).unwrap().enable_resolve_cache,
+                    expected
+                );
+            }
+
+            let uri = OperatorUri::new(
+                scheme,
+                [("enable_resolve_cache".to_string(), "invalid".to_string())],
+            )
+            .unwrap();
+            let err = HfConfig::from_uri(&uri).unwrap_err();
+            assert_eq!(err.kind(), opendal_core::ErrorKind::ConfigInvalid);
+            assert!(
+                err.to_string()
+                    .contains("enable_resolve_cache must be true or false")
+            );
+        }
+    }
 }