opendal 0.1.14 → 0.1.16

data/core/services/fs/src/core.rs

@@ -420,7 +420,7 @@ mod tests {
 
     #[cfg(unix)]
     #[tokio::test]
-    async fn test_delete_iter_rejects_absolute_path_outside_root() {
+    async fn test_delete_iter_normalizes_absolute_path_under_root() {
         use opendal_core::Operator;
 
         let root_dir = tempfile::TempDir::new().unwrap();
@@ -429,16 +429,34 @@ mod tests {
         let outside = outside_dir.path().join("outside.txt");
         std::fs::write(&outside, b"content").unwrap();
 
+        let inside = root_dir.path().join(outside.strip_prefix("/").unwrap());
+        std::fs::create_dir_all(inside.parent().unwrap()).unwrap();
+        std::fs::write(&inside, b"content").unwrap();
+
         let op =
             Operator::new(crate::Fs::default().root(root_dir.path().to_str().unwrap())).unwrap();
 
-        // `Deleter` takes the caller's key verbatim, so an absolute key reaches
-        // `root_join` without `normalize_path` having stripped the leading `/`.
-        let err = op
-            .delete_iter([outside.to_str().unwrap().to_string()])
+        op.delete_iter([outside.to_str().unwrap().to_string()])
             .await
-            .unwrap_err();
+            .unwrap();
+        assert!(!inside.exists());
+        assert_eq!(std::fs::read(&outside).unwrap(), b"content");
+    }
+
+    #[tokio::test]
+    async fn test_delete_iter_rejects_parent_path_outside_root() {
+        use opendal_core::Operator;
+
+        let temp_dir = tempfile::TempDir::new().unwrap();
+        let root = temp_dir.path().join("root");
+        std::fs::create_dir(&root).unwrap();
+
+        let outside = temp_dir.path().join("outside.txt");
+        std::fs::write(&outside, b"content").unwrap();
+
+        let op = Operator::new(crate::Fs::default().root(root.to_str().unwrap())).unwrap();
+        let err = op.delete_iter(["../outside.txt"]).await.unwrap_err();
         assert_eq!(err.kind(), ErrorKind::NotFound);
-        assert!(outside.exists());
+        assert_eq!(std::fs::read(&outside).unwrap(), b"content");
     }
 }

data/core/services/ftp/Cargo.toml

@@ -36,7 +36,7 @@ asyncband = { workspace = true, features = ["pool"] }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 rustls-native-certs = "0.8"
 serde = { workspace = true, features = ["derive"] }
 suppaftp = { version = "10.0.2", default-features = false, features = [

data/core/services/gcs/Cargo.toml

@@ -36,7 +36,7 @@ async-trait = "0.1"
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false, features = [
   "reqsign",
 ] }
 percent-encoding = "2.3"

data/core/services/gcs-grpc/Cargo.toml

@@ -36,7 +36,7 @@ bytes = { workspace = true }
 futures = { workspace = true, features = ["async-await", "std"] }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false, features = [
   "reqsign",
 ] }
 percent-encoding = "2"

data/core/services/gdrive/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 
 asyncband = { workspace = true, features = ["mutex", "singleflight"] }
 bytes = { workspace = true }

data/core/services/ghac/Cargo.toml

@@ -36,8 +36,8 @@ bytes = { workspace = true }
 ghac = { version = "0.3.0", default-features = false }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
-opendal-service-azblob = { path = "../azblob", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
+opendal-service-azblob = { path = "../azblob", version = "0.59.5", default-features = false }
 prost = { version = "0.14.3", default-features = false }
 reqsign-azure-storage = { version = "3.1.1", default-features = false }
 reqsign-core = { version = "3.2.0", default-features = false }

data/core/services/github/Cargo.toml

@@ -36,7 +36,7 @@ base64 = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/goosefs/Cargo.toml

@@ -45,11 +45,11 @@ asyncband = { workspace = true, features = ["rwlock"] }
 bytes = { workspace = true }
 goosefs-sdk = { version = "0.2.1", default-features = false }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]
-opendal = { path = "../..", version = "0.59.3", features = [
+opendal = { path = "../..", version = "0.59.5", features = [
   "services-goosefs",
 ] }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

data/core/services/gridfs/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 asyncband = { workspace = true, features = ["once-cell"] }
 futures = { workspace = true }
 mongodb = "3.3.0"
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]

data/core/services/hdfs/Cargo.toml

@@ -32,11 +32,11 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 
 bytes = { workspace = true }
 futures = { workspace = true }
-hdrs = { version = "0.3.2", features = ["async_file"] }
+hdrs = { version = "0.3.3", features = ["async_file"] }
 log = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 tokio = { workspace = true, features = ["rt"] }

data/core/services/hdfs/src/backend.rs

@@ -169,6 +169,8 @@ impl Builder for HdfsBuilder {
 
                     list: true,
 
+                    copy: true,
+
                     rename: true,
                     rename_with_if_not_exists: true,
 
@@ -195,7 +197,7 @@ impl Service for HdfsBackend {
     type Writer = HdfsLazyWriter;
     type Lister = Option<HdfsLister>;
     type Deleter = oio::OneShotDeleter<HdfsDeleter>;
-    type Copier = ();
+    type Copier = oio::OneShotCopier;
     type Composer = ();
 
     fn info(&self) -> ServiceInfo {
@@ -258,14 +260,20 @@ impl Service for HdfsBackend {
     fn copy(
         &self,
         _ctx: &OperationContext,
-        _from: &str,
-        _to: &str,
+        from: &str,
+        to: &str,
         _args: OpCopy,
     ) -> Result<Self::Copier> {
-        Err(Error::new(
-            ErrorKind::Unsupported,
-            "operation is not supported",
-        ))
+        let core = self.core.clone();
+        let from = from.to_string();
+        let to = to.to_string();
+        // Recreate the future so retry layers can rerun the copy after a temporary error.
+        Ok(oio::OneShotCopier::new_with(move || {
+            let core = core.clone();
+            let from = from.clone();
+            let to = to.clone();
+            async move { core.hdfs_copy(&from, &to).await }
+        }))
     }
 
     async fn rename(

data/core/services/hdfs/src/core.rs

@@ -124,8 +124,7 @@ impl HdfsCore {
         });
 
         if !target_exists {
-            let parent = get_parent(&target_path);
-            self.client.create_dir(parent).map_err(new_std_io_error)?;
+            self.ensure_parent_dir(&target_path)?;
         }
         if !should_append {
             initial_size = 0;
@@ -175,20 +174,7 @@ impl HdfsCore {
                     return Err(new_std_io_error(err));
                 }
 
-                let parent = std::path::PathBuf::from(&to_path)
-                    .parent()
-                    .ok_or_else(|| {
-                        Error::new(
-                            ErrorKind::Unexpected,
-                            "path should have parent but not, it must be malformed",
-                        )
-                        .with_context("to", &to_path)
-                    })?
-                    .to_path_buf();
-
-                self.client
-                    .create_dir(&parent.to_string_lossy())
-                    .map_err(new_std_io_error)?;
+                self.ensure_parent_dir(&to_path)?;
             }
             Ok(metadata) => {
                 if metadata.is_file() {
@@ -215,6 +201,59 @@ impl HdfsCore {
 
         Ok(())
     }
+
+    pub async fn hdfs_copy(&self, from: &str, to: &str) -> Result<Metadata> {
+        let from_path = build_rooted_abs_path(&self.root, from);
+        // OpenDAL copy is file-to-file only. Reject directory sources before
+        // the HDFS API can recursively copy their contents.
+        let from_meta = self.client.metadata(&from_path).map_err(new_std_io_error)?;
+        if !from_meta.is_file() {
+            return Err(
+                Error::new(ErrorKind::IsADirectory, "from path should be a file")
+                    .with_context("from", &from_path),
+            );
+        }
+
+        let to_path = build_rooted_abs_path(&self.root, to);
+        match self.client.metadata(&to_path) {
+            Ok(meta) => {
+                // OpenDAL treats `to` as the exact destination file path. Reject
+                // an existing directory instead of copying the source into it.
+                if meta.is_dir() {
+                    return Err(
+                        Error::new(ErrorKind::IsADirectory, "to path should be a file")
+                            .with_context("to", &to_path),
+                    );
+                }
+                // The HDFS copy API does not replace an existing destination,
+                // so remove it first to preserve OpenDAL's overwrite semantics.
+                self.client
+                    .remove_file(&to_path)
+                    .map_err(new_std_io_error)?;
+            }
+            Err(err) if err.kind() == io::ErrorKind::NotFound => {
+                // hdrs copy_file requires the destination parent to already exist.
+                self.ensure_parent_dir(&to_path)?;
+            }
+            Err(err) => return Err(new_std_io_error(err)),
+        }
+
+        let client = self.client.clone();
+        let copy_from = from_path.clone();
+        let copy_to = to_path.clone();
+        tokio::task::spawn_blocking(move || client.copy_file(&copy_from, &copy_to))
+            .await
+            .map_err(|e| Error::new(ErrorKind::Unexpected, "tokio task join failed").set_source(e))?
+            .map_err(new_std_io_error)?;
+
+        Ok(MetadataBuilder::file(from_meta.len()).build())
+    }
+
+    fn ensure_parent_dir(&self, path: &str) -> Result<()> {
+        self.client
+            .create_dir(get_parent(path))
+            .map_err(new_std_io_error)
+    }
 }
 
 #[cfg(test)]

data/core/services/hdfs/src/docs.md

@@ -10,13 +10,21 @@ Depending on its configuration and the backing system, this service can expose:
 - [x] write
 - [x] delete
 - [x] list
-- [ ] copy
+- [x] copy
 - [x] rename
 - [ ] ~~presign~~
 
 Inspect the effective capability set with [`opendal_core::Operator::info`] and
 [`opendal_core::OperatorInfo::capability`] after building an operator.
 
+## Copy behavior
+
+The HDFS service copies files to exact destination file paths and rejects
+directory sources and destinations. Copy overwrites an existing destination by
+removing it before copying the source. This replacement is not atomic: if the
+copy fails after removing the destination, the destination can be missing or
+incomplete.
+
 ## Differences with webhdfs
 
 The [WebHDFS service](https://docs.rs/opendal-service-webhdfs) uses HDFS's

data/core/services/hdfs-native/Cargo.toml

@@ -36,5 +36,5 @@ bytes = { workspace = true }
 futures = { workspace = true }
 hdfs-native = { version = "0.14" }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }

data/core/services/hf/Cargo.toml

@@ -37,7 +37,7 @@ bytes = { workspace = true }
 hf-xet = "1.6.0"
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 percent-encoding = "2"
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
@@ -45,9 +45,9 @@ serde_json = { workspace = true }
 [dev-dependencies]
 base64 = { workspace = true }
 futures = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", features = [
+opendal-core = { path = "../../core", version = "0.59.5", features = [
   "executors-tokio",
 ] }
-opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.3" }
+opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.5" }
 serde_json = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

data/core/services/http/Cargo.toml

@@ -22,7 +22,7 @@ license = "Apache-2.0"
 name = "opendal-service-http"
 readme = "README.md"
 repository = "https://github.com/apache/opendal"
-version = "0.59.3"
+version = "0.59.5"
 
 [package.metadata.docs.rs]
 all-features = true
@@ -30,9 +30,9 @@ all-features = true
 [dependencies]
 http = "1.4"
 log = "0.4"
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { version = "1", features = ["derive"] }
 
 [dev-dependencies]
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] }

data/core/services/http/src/core.rs

@@ -177,7 +177,8 @@ pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
             (ErrorKind::ConditionNotMatch, false)
         }
         StatusCode::RANGE_NOT_SATISFIABLE => (ErrorKind::RangeNotSatisfied, false),
-        StatusCode::INTERNAL_SERVER_ERROR
+        StatusCode::TOO_MANY_REQUESTS
+        | StatusCode::INTERNAL_SERVER_ERROR
         | StatusCode::BAD_GATEWAY
         | StatusCode::SERVICE_UNAVAILABLE
         | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),

data/core/services/ipfs/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 prost = "0.14.3"
 serde = { workspace = true, features = ["derive"] }
 

data/core/services/ipmfs/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 
 bytes = { workspace = true }
 http = { workspace = true }

data/core/services/koofr/Cargo.toml

@@ -36,7 +36,7 @@ asyncband = { workspace = true, features = ["mutex", "once-cell"] }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/lakefs/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/memcached/Cargo.toml

@@ -33,7 +33,7 @@ all-features = true
 
 [dependencies]
 asyncband = { workspace = true, features = ["pool"] }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 tokio = { workspace = true, features = ["io-util", "macros", "net", "time"] }
 url = { workspace = true }

data/core/services/mini_moka/Cargo.toml

@@ -34,5 +34,5 @@ all-features = true
 [dependencies]
 log = { workspace = true }
 mini-moka = "0.10"
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }

data/core/services/moka/Cargo.toml

@@ -34,7 +34,7 @@ all-features = true
 [dependencies]
 log = { workspace = true }
 moka = { version = "0.12", features = ["future", "sync"] }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]

data/core/services/mongodb/Cargo.toml

@@ -34,7 +34,7 @@ all-features = true
 [dependencies]
 asyncband = { workspace = true, features = ["once-cell"] }
 mongodb = { version = "3.3.0" }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]

data/core/services/monoiofs/Cargo.toml

@@ -41,7 +41,7 @@ monoio = { version = "0.2.4", features = [
   "unlinkat",
   "renameat",
 ] }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 
 [dev-dependencies]

data/core/services/mysql/Cargo.toml

@@ -33,7 +33,7 @@ all-features = true
 
 [dependencies]
 asyncband = { workspace = true, features = ["once-cell"] }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 sqlx = { version = "0.8.0", features = ["runtime-tokio-rustls", "mysql"] }
 

data/core/services/obs/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false, features = [
   "reqsign",
 ] }
 quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }

data/core/services/onedrive/Cargo.toml

@@ -36,7 +36,7 @@ asyncband = { workspace = true, features = ["mutex"] }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 tokio = { workspace = true, features = ["time"] }

data/core/services/opfs/Cargo.toml

@@ -35,7 +35,7 @@ targets = ["wasm32-unknown-unknown"]
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 js-sys = "0.3.77"
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false }
 send_wrapper = "0.6"
 serde = { workspace = true, features = ["derive"] }
 wasm-bindgen = "0.2.100"

data/core/services/oss/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.59.3", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.5", default-features = false, features = [
   "reqsign",
 ] }
 quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }