opendal 0.1.12 → 0.1.13

data/core/services/hdfs/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 
 bytes = { workspace = true }
 futures = { workspace = true }

data/core/services/hdfs/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hdfs-native/Cargo.toml

@@ -36,5 +36,5 @@ bytes = { workspace = true }
 futures = { workspace = true }
 hdfs-native = { version = "0.14" }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 serde = { workspace = true, features = ["derive"] }

data/core/services/hdfs-native/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hf/Cargo.toml

@@ -32,12 +32,12 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-asyncband = { workspace = true }
+asyncband = { workspace = true, features = ["mutex", "once-cell"] }
 bytes = { workspace = true }
 hf-xet = "1.6.0"
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.2", default-features = false }
 percent-encoding = "2"
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
@@ -45,6 +45,6 @@ serde_json = { workspace = true }
 [dev-dependencies]
 base64 = { workspace = true }
 futures = { workspace = true }
-opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.1" }
+opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.2" }
 serde_json = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

data/core/services/hf/LICENSE

@@ -199,3 +199,19 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+
+------------------------------------------------------------------------------------
+This product bundles third-party components under other open source licenses.
+This section summarizes those components and their licenses. See the referenced
+license files for text of these licenses.
+
+
+Creative Commons Zero v1.0 Universal
+------------------------------------
+
+bindings/dotnet/.gitignore
+
+The above file is derived from VisualStudio.gitignore in
+https://github.com/github/gitignore. Its license text is in
+bindings/dotnet/LICENSE-CC0-1.0.txt.

data/core/services/hf/src/backend.rs

@@ -133,6 +133,22 @@ impl HfBuilder {
         self
     }
 
+    /// Enable caching of resolved HTTP download addresses and XET file metadata.
+    ///
+    /// Defaults to `false`: every range resolves through the Hub in either
+    /// download mode. Set to `true` to share resolve results across readers on
+    /// the same backend. HTTP addresses refresh near expiry.
+    ///
+    /// Enable this only when previously written files are not modified. Changed
+    /// files can remain invisible while cached results are reused, including
+    /// changes from other clients or a floating repository revision. Issued
+    /// download URLs can remain usable until expiry after Hub permissions change.
+    /// Separate authorization identities must use separately constructed backends.
+    pub fn enable_resolve_cache(mut self, enabled: bool) -> Self {
+        self.config.enable_resolve_cache = enabled;
+        self
+    }
+
     /// Set the Hub base URL.
     ///
     /// Configure this when your organization uses a
@@ -146,12 +162,18 @@ impl HfBuilder {
         self
     }
 
+    /// Resolve the Hub base URL: an explicit config value wins, then
+    /// `HF_ENDPOINT`, then the public Hub. A trailing slash is trimmed
+    /// because every URL is built by appending `/api/...` to this, and HF
+    /// answers the resulting `//api/...` with a 404.
     fn hf_endpoint(&self) -> String {
         self.config
             .endpoint
             .clone()
             .or_else(|| std::env::var("HF_ENDPOINT").ok())
             .unwrap_or_else(|| "https://huggingface.co".to_string())
+            .trim_end_matches('/')
+            .to_string()
     }
 
     /// Resolve the download mode: an explicit config value wins; otherwise a set,
@@ -260,16 +282,10 @@ impl Builder for HfBuilder {
         let repo = HfRepo::new(repo_type, repo_id, Some(revision.clone()));
         debug!("backend repo uri: {:?}", repo.uri(&root, ""));
 
+        let mut core = HfCore::build(info, capability, repo, root, token, endpoint, download_mode)?;
+        core.enable_resolve_cache = self.config.enable_resolve_cache;
         Ok(HfBackend {
-            core: Arc::new(HfCore::build(
-                info,
-                capability,
-                repo,
-                root,
-                token,
-                endpoint,
-                download_mode,
-            )?),
+            core: Arc::new(core),
         })
     }
 }
@@ -435,6 +451,19 @@ pub(super) mod test_utils {
         finish_operator(op)
     }
 
+    /// Same public dataset as [`mbpp_operator`], but with the repo id cased
+    /// differently from the canonical `google-research-datasets/mbpp`. HF
+    /// answers every request for it with a `307` to the canonical URL.
+    pub fn miscased_mbpp_operator() -> Operator {
+        let op = Operator::new(
+            HfBuilder::default()
+                .repo_type("dataset")
+                .repo_id("Google-Research-Datasets/MBPP"),
+        )
+        .unwrap();
+        finish_operator(op)
+    }
+
     pub fn testing_dataset_core() -> Arc<HfCore> {
         let repo_id = std::env::var("HF_OPENDAL_DATASET").expect("HF_OPENDAL_DATASET must be set");
         let token = std::env::var("HF_OPENDAL_TOKEN").expect("HF_OPENDAL_TOKEN must be set");
@@ -538,6 +567,18 @@ mod tests {
         assert_eq!(result.as_deref(), Some("file-token"));
     }
 
+    #[test]
+    fn hf_endpoint_trims_trailing_slash() {
+        let _guard = ENV_LOCK.lock().unwrap();
+        unsafe { std::env::remove_var("HF_ENDPOINT") };
+        assert_eq!(
+            HfBuilder::default()
+                .endpoint("https://hub.example.com/")
+                .hf_endpoint(),
+            "https://hub.example.com"
+        );
+    }
+
     #[test]
     fn hf_endpoint_returns_default() {
         let _guard = ENV_LOCK.lock().unwrap();
@@ -600,6 +641,33 @@ mod tests {
         assert_eq!(mode, HfDownloadMode::Xet);
     }
 
+    #[tokio::test]
+    async fn build_resolve_cache_requires_opt_in() -> Result<()> {
+        use super::super::core::test_utils::MockHttpTransport;
+
+        for enabled in [None, Some(false), Some(true)] {
+            let mut builder = HfBuilder::default()
+                .repo_type("model")
+                .repo_id("org/repo")
+                .download_mode("xet");
+            if let Some(enabled) = enabled {
+                builder = builder.enable_resolve_cache(enabled);
+            }
+            let transport = MockHttpTransport::new();
+            let ctx = OperationContext::new()
+                .with_http_transport(HttpTransporter::new(transport.clone()));
+            let op = Operator::new(builder)?.with_context(ctx);
+            for reads in 1..=2 {
+                assert_eq!(op.read("plain.txt").await?.to_vec(), b"hello");
+                assert_eq!(
+                    transport.request_count(),
+                    reads + usize::from(enabled == Some(true))
+                );
+            }
+        }
+        Ok(())
+    }
+
     #[test]
     fn build_accepts_datasets_alias() {
         HfBuilder::default()

data/core/services/hf/src/config.rs

@@ -60,6 +60,13 @@ pub struct HfConfig {
     ///
     /// See <https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
     pub download_mode: Option<HfDownloadMode>,
+    /// Enable caching of resolved HTTP download addresses and XET file metadata.
+    ///
+    /// Defaults to `false`. Set to `true` to share resolve results across readers
+    /// on the same backend. Changed files may remain invisible while cached
+    /// results are reused. See [`HfBuilder::enable_resolve_cache`] for freshness
+    /// semantics.
+    pub enable_resolve_cache: bool,
 }
 
 impl Debug for HfConfig {
@@ -73,6 +80,7 @@ impl Debug for HfConfig {
             .field("revision", &self.revision)
             .field("root", &self.root)
             .field("download_mode", &self.download_mode)
+            .field("enable_resolve_cache", &self.enable_resolve_cache)
             .finish_non_exhaustive()
     }
 }
@@ -106,6 +114,19 @@ impl opendal_core::Configurator for HfConfig {
             .map(|s| HfDownloadMode::parse(s))
             .transpose()?;
 
+        let enable_resolve_cache = opts
+            .get("enable_resolve_cache")
+            .map(|value| value.parse::<bool>())
+            .transpose()
+            .map_err(|err| {
+                opendal_core::Error::new(
+                    opendal_core::ErrorKind::ConfigInvalid,
+                    "enable_resolve_cache must be true or false",
+                )
+                .set_source(err)
+            })?
+            .unwrap_or_default();
+
         if !path.is_empty() {
             // Full URI like "hf://datasets/user/repo@rev/path"
             let parsed = HfUri::parse(&path)?;
@@ -117,6 +138,7 @@ impl opendal_core::Configurator for HfConfig {
                 token: opts.get("token").cloned(),
                 endpoint: opts.get("endpoint").cloned(),
                 download_mode,
+                enable_resolve_cache,
             })
         } else {
             // Bare scheme from via_iter, all config is in options.
@@ -138,6 +160,7 @@ impl opendal_core::Configurator for HfConfig {
                 token: opts.get("token").cloned(),
                 endpoint: opts.get("endpoint").cloned(),
                 download_mode,
+                enable_resolve_cache,
             })
         }
     }
@@ -222,4 +245,39 @@ mod tests {
         let cfg = HfConfig::from_uri(&uri).unwrap();
         assert_eq!(cfg.download_mode.unwrap_or_default(), HfDownloadMode::Xet);
     }
+
+    #[test]
+    fn from_uri_resolve_cache_options() {
+        for scheme in ["hf://datasets/user/repo", "huggingface"] {
+            for (value, expected) in [(None, false), (Some("false"), false), (Some("true"), true)] {
+                let mut options = vec![("repo_type", "dataset"), ("repo_id", "user/repo")];
+                if let Some(value) = value {
+                    options.push(("enable_resolve_cache", value));
+                }
+                let uri = OperatorUri::new(
+                    scheme,
+                    options
+                        .into_iter()
+                        .map(|(key, value)| (key.to_string(), value.to_string())),
+                )
+                .unwrap();
+                assert_eq!(
+                    HfConfig::from_uri(&uri).unwrap().enable_resolve_cache,
+                    expected
+                );
+            }
+
+            let uri = OperatorUri::new(
+                scheme,
+                [("enable_resolve_cache".to_string(), "invalid".to_string())],
+            )
+            .unwrap();
+            let err = HfConfig::from_uri(&uri).unwrap_err();
+            assert_eq!(err.kind(), opendal_core::ErrorKind::ConfigInvalid);
+            assert!(
+                err.to_string()
+                    .contains("enable_resolve_cache must be true or false")
+            );
+        }
+    }
 }