opendal 0.1.9 → 0.1.11

data/core/services/webhdfs/src/core.rs

@@ -17,13 +17,13 @@
 
 use std::fmt::Debug;
 
+use asyncband::once::OnceCell;
 use bytes::Buf;
 use http::Request;
 use http::Response;
 use http::StatusCode;
 use http::header::CONTENT_LENGTH;
 use http::header::CONTENT_TYPE;
-use mea::once::OnceCell;
 use serde::Deserialize;
 
 use opendal_core::raw::*;
@@ -118,7 +118,10 @@ impl WebhdfsCore {
         let status = resp.status();
 
         if status != StatusCode::CREATED && status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("Create")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();
@@ -205,7 +208,10 @@ impl WebhdfsCore {
 
                 Ok(resp.location)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Append")),
+                resp,
+            )),
         }
     }
 
@@ -314,7 +320,7 @@ impl WebhdfsCore {
             percent_encode_path(&p),
         );
         if !start_after.is_empty() {
-            url += format!("&startAfter={start_after}").as_str();
+            url += format!("&startAfter={}", percent_encode_path(start_after)).as_str();
         }
         if let Some(user) = &self.user_name {
             url += format!("&user.name={user}").as_str();
@@ -430,82 +436,80 @@ pub(super) struct LocationResponse {
     pub location: String,
 }
 
-mod error {
-    use http::Response;
-    use http::StatusCode;
-    use http::response::Parts;
-    use serde::Deserialize;
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+struct WebHdfsErrorWrapper {
+    pub remote_exception: WebHdfsError,
+}
 
-    use opendal_core::raw::*;
-    use opendal_core::*;
+/// WebHdfsError is the error message returned by WebHdfs service
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+struct WebHdfsError {
+    exception: String,
+    message: String,
+    java_class_name: String,
+}
 
-    #[derive(Debug, Deserialize)]
-    #[serde(rename_all = "PascalCase")]
-    struct WebHdfsErrorWrapper {
-        pub remote_exception: WebHdfsError,
-    }
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
 
-    /// WebHdfsError is the error message returned by WebHdfs service
-    #[derive(Debug, Deserialize)]
-    #[serde(rename_all = "camelCase")]
-    #[allow(dead_code)]
-    struct WebHdfsError {
-        exception: String,
-        message: String,
-        java_class_name: String,
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
     }
+}
 
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
-        let s = String::from_utf8_lossy(&bs);
-        parse_error_msg(parts, &s)
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        // passing invalid arguments will return BAD_REQUEST
+        // should be un-retryable
+        StatusCode::BAD_REQUEST => (ErrorKind::Unexpected, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match serde_json::from_slice::<WebHdfsErrorWrapper>(&bs) {
+        Ok(wh_error) => format!("{:?}", wh_error.remote_exception),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
     }
 
-    pub(crate) fn parse_error_msg(parts: Parts, body: &str) -> Error {
-        let (kind, retryable) = match parts.status {
-            StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
-            StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
-                (ErrorKind::PermissionDenied, false)
-            }
-            // passing invalid arguments will return BAD_REQUEST
-            // should be un-retryable
-            StatusCode::BAD_REQUEST => (ErrorKind::Unexpected, false),
-            StatusCode::INTERNAL_SERVER_ERROR
-            | StatusCode::BAD_GATEWAY
-            | StatusCode::SERVICE_UNAVAILABLE
-            | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
-
-        let message = match serde_json::from_str::<WebHdfsErrorWrapper>(body) {
-            Ok(wh_error) => format!("{:?}", wh_error.remote_exception),
-            Err(_) => body.to_owned(),
-        };
-
-        let mut err = Error::new(kind, message);
-
-        err = with_error_response_context(err, parts);
-
-        if retryable {
-            err = err.set_temporary();
-        }
-
-        err
-    }
+    err
+}
 
-    #[cfg(test)]
-    mod tests {
-        use bytes::Buf;
-        use serde_json::from_reader;
+#[cfg(test)]
+mod tests {
+    use bytes::Buf;
+    use serde_json::from_reader;
 
-        use super::*;
+    use super::*;
 
-        /// Error response example from https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Error%20Responses
-        #[tokio::test]
-        async fn test_parse_error() -> Result<()> {
-            let ill_args = bytes::Bytes::from(
-                r#"
+    /// Error response example from https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Error%20Responses
+    #[tokio::test]
+    async fn test_parse_error() -> Result<()> {
+        let ill_args = bytes::Bytes::from(
+            r#"
 {
   "RemoteException":
   {
@@ -515,33 +519,30 @@ mod error {
   }
 }
     "#,
-            );
-            let body = Buffer::from(ill_args.clone());
-            let resp = Response::builder()
-                .status(StatusCode::BAD_REQUEST)
-                .body(body)
-                .unwrap();
-
-            let err = parse_error(resp);
-            assert_eq!(err.kind(), ErrorKind::Unexpected);
-            assert!(!err.is_temporary());
-
-            let err_msg: WebHdfsError = from_reader::<_, WebHdfsErrorWrapper>(ill_args.reader())
-                .expect("must success")
-                .remote_exception;
-            assert_eq!(err_msg.exception, "IllegalArgumentException");
-            assert_eq!(
-                err_msg.java_class_name,
-                "java.lang.IllegalArgumentException"
-            );
-            assert_eq!(
-                err_msg.message,
-                "Invalid value for webhdfs parameter \"permission\": ..."
-            );
-
-            Ok(())
-        }
+        );
+        let body = Buffer::from(ill_args.clone());
+        let resp = Response::builder()
+            .status(StatusCode::BAD_REQUEST)
+            .body(body)
+            .unwrap();
+
+        let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
+        assert_eq!(err.kind(), ErrorKind::Unexpected);
+        assert!(!err.is_temporary());
+
+        let err_msg: WebHdfsError = from_reader::<_, WebHdfsErrorWrapper>(ill_args.reader())
+            .expect("must success")
+            .remote_exception;
+        assert_eq!(err_msg.exception, "IllegalArgumentException");
+        assert_eq!(
+            err_msg.java_class_name,
+            "java.lang.IllegalArgumentException"
+        );
+        assert_eq!(
+            err_msg.message,
+            "Invalid value for webhdfs parameter \"permission\": ..."
+        );
+
+        Ok(())
     }
 }
-
-pub(super) use error::*;

data/core/services/webhdfs/src/deleter.rs

@@ -19,8 +19,8 @@ use std::sync::Arc;
 
 use http::StatusCode;
 
-use super::core::WebhdfsCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, WebhdfsCore};
 use opendal_core::raw::oio;
 use opendal_core::raw::*;
 use opendal_core::*;
@@ -42,7 +42,10 @@ impl oio::OneShotDelete for WebhdfsDeleter {
 
         match resp.status() {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Delete")),
+                resp,
+            )),
         }
     }
 }

data/core/services/webhdfs/src/lister.rs

@@ -20,8 +20,8 @@ use std::sync::Arc;
 use bytes::Buf;
 use http::StatusCode;
 
-use super::core::WebhdfsCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, WebhdfsCore};
 use super::message::*;
 use opendal_core::raw::oio;
 use opendal_core::raw::*;
@@ -52,7 +52,7 @@ impl oio::PageList for WebhdfsLister {
                     ctx.done = true;
                     ctx.entries.push_back(oio::Entry::new(
                         format!("{}/", self.path).as_str(),
-                        Metadata::new(EntryMode::DIR),
+                        MetadataBuilder::dir().build(),
                     ));
 
                     let bs = resp.into_body();
@@ -65,7 +65,12 @@ impl oio::PageList for WebhdfsLister {
                     ctx.done = true;
                     return Ok(());
                 }
-                _ => return Err(parse_error(resp)),
+                _ => {
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("ListStatus")),
+                        resp,
+                    ));
+                }
             }
         } else {
             let resp = self
@@ -83,7 +88,7 @@ impl oio::PageList for WebhdfsLister {
                     if directory_listing.remaining_entries == 0 {
                         ctx.entries.push_back(oio::Entry::new(
                             format!("{}/", self.path).as_str(),
-                            Metadata::new(EntryMode::DIR),
+                            MetadataBuilder::dir().build(),
                         ));
 
                         ctx.done = true;
@@ -98,7 +103,12 @@ impl oio::PageList for WebhdfsLister {
                     ctx.done = true;
                     return Ok(());
                 }
-                _ => return Err(parse_error(resp)),
+                _ => {
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("ListStatusBatch")),
+                        resp,
+                    ));
+                }
             }
         };
 
@@ -110,10 +120,12 @@ impl oio::PageList for WebhdfsLister {
             };
 
             let meta = match status.ty {
-                FileStatusType::Directory => Metadata::new(EntryMode::DIR),
-                FileStatusType::File => Metadata::new(EntryMode::FILE)
-                    .with_content_length(status.length)
-                    .with_last_modified(Timestamp::from_millisecond(status.modification_time)?),
+                FileStatusType::Directory => MetadataBuilder::dir().build(),
+                FileStatusType::File => {
+                    let mut metadata = MetadataBuilder::file(status.length);
+                    metadata.last_modified(Timestamp::from_millisecond(status.modification_time)?);
+                    metadata.build()
+                }
             };
 
             if meta.mode().is_file() {

data/core/services/webhdfs/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::oio;
@@ -60,7 +60,10 @@ impl oio::StreamRead for WebhdfsReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("Open")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/webhdfs/src/writer.rs

@@ -21,8 +21,8 @@ use bytes::Buf;
 use http::StatusCode;
 use uuid::Uuid;
 
-use super::core::WebhdfsCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, WebhdfsCore};
 use crate::message::FileStatusWrapper;
 use opendal_core::raw::oio;
 use opendal_core::raw::*;
@@ -59,8 +59,11 @@ impl oio::BlockWrite for WebhdfsWriter {
 
         let status = resp.status();
         match status {
-            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::CREATED | StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Create")),
+                resp,
+            )),
         }
     }
 
@@ -85,7 +88,10 @@ impl oio::BlockWrite for WebhdfsWriter {
         let status = resp.status();
         match status {
             StatusCode::CREATED | StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Create")),
+                resp,
+            )),
         }
     }
 
@@ -110,7 +116,10 @@ impl oio::BlockWrite for WebhdfsWriter {
 
             let status = resp.status();
             if status != StatusCode::OK {
-                return Err(parse_error(resp));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("Concat")),
+                    resp,
+                ));
             }
         }
         // delete the path file
@@ -118,7 +127,10 @@ impl oio::BlockWrite for WebhdfsWriter {
 
         let status = resp.status();
         if status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("Delete")),
+                resp,
+            ));
         }
 
         // rename concat file to path
@@ -129,20 +141,34 @@ impl oio::BlockWrite for WebhdfsWriter {
 
         let status = resp.status();
         match status {
-            StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Rename")),
+                resp,
+            )),
         }
     }
 
     async fn abort_block(&self, block_ids: Vec<Uuid>) -> Result<()> {
+        let Some(ref atomic_write_dir) = self.core.atomic_write_dir else {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "write multi is not supported when atomic is not set",
+            ));
+        };
         for block_id in block_ids {
             let resp = self
                 .core
-                .webhdfs_delete(&self.ctx, &block_id.to_string())
+                .webhdfs_delete(&self.ctx, &format!("{atomic_write_dir}{block_id}"))
                 .await?;
             match resp.status() {
                 StatusCode::OK => {}
-                _ => return Err(parse_error(resp)),
+                _ => {
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("Delete")),
+                        resp,
+                    ));
+                }
             }
         }
         Ok(())
@@ -175,10 +201,16 @@ impl oio::AppendWrite for WebhdfsWriter {
                 let status = resp.status();
                 match status {
                     StatusCode::CREATED | StatusCode::OK => Ok(0),
-                    _ => Err(parse_error(resp)),
+                    _ => Err(parse_error(
+                        ErrorContext::new(ServiceOperation("Create")),
+                        resp,
+                    )),
                 }
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetFileStatus")),
+                resp,
+            )),
         }
     }
 
@@ -190,8 +222,11 @@ impl oio::AppendWrite for WebhdfsWriter {
 
         let status = resp.status();
         match status {
-            StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Append")),
+                resp,
+            )),
         }
     }
 }

data/core/services/yandex-disk/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 
 bytes = { workspace = true }
 http = { workspace = true }

data/core/services/yandex-disk/src/backend.rs

@@ -137,6 +137,7 @@ impl Service for YandexDiskBackend {
     type Lister = oio::PageLister<YandexDiskLister>;
     type Deleter = oio::OneShotDeleter<YandexDiskDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -172,7 +173,10 @@ impl Service for YandexDiskBackend {
 
         match status {
             StatusCode::OK | StatusCode::CREATED => Ok(RpRename::default()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("MoveResource")),
+                resp,
+            )),
         }
     }
 
@@ -181,15 +185,25 @@ impl Service for YandexDiskBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _args: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
+        let backend = self.clone();
         let core = self.core.clone();
         let ctx = ctx.clone();
         let from = from.to_string();
         let to = to.to_string();
+        let source_content_length_hint = args.source_content_length_hint();
 
         Ok(oio::OneShotCopier::new(async move {
+            let source_size = match source_content_length_hint {
+                Some(size) => size,
+                None => backend
+                    .stat(&ctx, &from, OpStat::default())
+                    .await?
+                    .into_metadata()
+                    .content_length(),
+            };
+
             core.ensure_dir_exists(&ctx, &to).await?;
 
             let resp = core.copy(&ctx, &from, &to).await?;
@@ -197,8 +211,13 @@ impl Service for YandexDiskBackend {
             let status = resp.status();
 
             match status {
-                StatusCode::OK | StatusCode::CREATED => Ok(Metadata::default()),
-                _ => Err(parse_error(resp)),
+                StatusCode::OK | StatusCode::CREATED => {
+                    Ok(MetadataBuilder::file(source_size).build())
+                }
+                _ => Err(parse_error(
+                    ErrorContext::new(ServiceOperation("CopyResource")),
+                    resp,
+                )),
             }
         }))
     }
@@ -229,7 +248,10 @@ impl Service for YandexDiskBackend {
 
                 parse_info(mf).map(RpStat::new)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetMetainformation")),
+                resp,
+            )),
         }
     }