opendal 0.1.9 → 0.1.11

data/core/services/seafile/src/backend.rs

@@ -18,8 +18,8 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
+use asyncband::rwlock::RwLock;
 use log::debug;
-use mea::rwlock::RwLock;
 
 use super::SEAFILE_SCHEME;
 use super::config::SeafileConfig;
@@ -196,6 +196,7 @@ impl Service for SeafileBackend {
     type Lister = oio::PageLister<SeafileLister>;
     type Deleter = oio::OneShotDeleter<SeafileDeleter>;
     type Copier = ();
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -217,7 +218,7 @@ impl Service for SeafileBackend {
 
     async fn stat(&self, ctx: &OperationContext, path: &str, _args: OpStat) -> Result<RpStat> {
         if path == "/" {
-            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+            return Ok(RpStat::new(MetadataBuilder::dir().build()));
         }
 
         let metadata = if path.ends_with('/') {
@@ -281,7 +282,6 @@ impl Service for SeafileBackend {
         _from: &str,
         _to: &str,
         _args: OpCopy,
-        _opts: OpCopier,
     ) -> Result<Self::Copier> {
         Err(Error::new(
             ErrorKind::Unsupported,

data/core/services/seafile/src/core.rs

@@ -18,13 +18,13 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
+use asyncband::rwlock::RwLock;
 use bytes::Buf;
 use bytes::Bytes;
 use http::Request;
 use http::Response;
 use http::StatusCode;
 use http::header;
-use mea::rwlock::RwLock;
 use serde::Deserialize;
 
 use opendal_core::raw::*;
@@ -109,7 +109,10 @@ impl SeafileCore {
                     };
                 }
                 _ => {
-                    return Err(parse_error(resp));
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("AuthToken")),
+                        resp,
+                    ));
                 }
             }
 
@@ -150,7 +153,10 @@ impl SeafileCore {
                     }
                 }
                 _ => {
-                    return Err(parse_error(resp));
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("ListRepositories")),
+                        resp,
+                    ));
                 }
             }
             Ok(signer.auth_info.clone())
@@ -185,7 +191,10 @@ impl SeafileCore {
                     .map_err(new_json_deserialize_error)?;
                 Ok(upload_url)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetUploadLink")),
+                resp,
+            )),
         }
     }
 
@@ -256,7 +265,10 @@ impl SeafileCore {
 
         match resp.status() {
             StatusCode::OK | StatusCode::CREATED => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Mkdir")),
+                resp,
+            )),
         }
     }
 
@@ -290,7 +302,10 @@ impl SeafileCore {
 
                 Ok(download_url)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetDownloadLink")),
+                resp,
+            )),
         }
     }
 
@@ -344,7 +359,10 @@ impl SeafileCore {
                     .map_err(new_json_deserialize_error)?;
                 Ok(file_detail)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetFileDetail")),
+                resp,
+            )),
         }
     }
 
@@ -377,7 +395,10 @@ impl SeafileCore {
                     .map_err(new_json_deserialize_error)?;
                 Ok(dir_detail)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetDirDetail")),
+                resp,
+            )),
         }
     }
 
@@ -415,7 +436,10 @@ impl SeafileCore {
 
         match status {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Delete")),
+                resp,
+            )),
         }
     }
 
@@ -457,7 +481,10 @@ impl SeafileCore {
                 infos: None,
                 rooted_abs_path,
             }),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListDir")),
+                resp,
+            )),
         }
     }
 }
@@ -479,20 +506,18 @@ pub struct DirDetail {
 }
 
 pub fn parse_dir_detail(dir_detail: DirDetail) -> Result<Metadata> {
-    let mut md = Metadata::new(EntryMode::DIR);
+    let mut md = MetadataBuilder::dir();
 
-    md.set_last_modified(dir_detail.mtime.parse::<Timestamp>()?);
+    md.last_modified(dir_detail.mtime.parse::<Timestamp>()?);
 
-    Ok(md)
+    Ok(md.build())
 }
 
 pub fn parse_file_detail(file_detail: FileDetail) -> Result<Metadata> {
-    let mut md = Metadata::new(EntryMode::FILE);
+    let mut md = MetadataBuilder::file(file_detail.size);
+    md.last_modified(file_detail.last_modified.parse::<Timestamp>()?);
 
-    md.set_content_length(file_detail.size);
-    md.set_last_modified(file_detail.last_modified.parse::<Timestamp>()?);
-
-    Ok(md)
+    Ok(md.build())
 }
 
 #[derive(Clone, Default)]
@@ -528,77 +553,78 @@ pub struct ListResponse {
     pub rooted_abs_path: String,
 }
 
-mod error {
-    use bytes::Buf;
-    use http::Response;
-    use serde::Deserialize;
+/// the error response of seafile
+#[derive(Default, Debug, Deserialize)]
+#[allow(dead_code)]
+struct SeafileError {
+    error_msg: String,
+}
 
-    use opendal_core::raw::*;
-    use opendal_core::*;
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
 
-    /// the error response of seafile
-    #[derive(Default, Debug, Deserialize)]
-    #[allow(dead_code)]
-    struct SeafileError {
-        error_msg: String,
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
     }
+}
 
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
 
-        let (kind, _retryable) = match parts.status.as_u16() {
-            403 => (ErrorKind::PermissionDenied, false),
-            404 => (ErrorKind::NotFound, false),
-            520 => (ErrorKind::Unexpected, false),
-            _ => (ErrorKind::Unexpected, false),
-        };
+    let (kind, _retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        520 => (ErrorKind::Unexpected, false),
+        _ => (ErrorKind::Unexpected, false),
+    };
 
-        let (message, _seafile_err) =
-            serde_json::from_reader::<_, SeafileError>(bs.clone().reader())
-                .map(|seafile_err| (format!("{seafile_err:?}"), Some(seafile_err)))
-                .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+    let (message, _seafile_err) = serde_json::from_reader::<_, SeafileError>(bs.clone().reader())
+        .map(|seafile_err| (format!("{seafile_err:?}"), Some(seafile_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
 
-        let mut err = Error::new(kind, message);
+    let mut err = Error::new(kind, message);
 
-        err = with_error_response_context(err, parts);
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
 
-        err
-    }
+    err
+}
 
-    #[cfg(test)]
-    mod test {
-        use http::StatusCode;
-
-        use super::*;
-
-        #[tokio::test]
-        async fn test_parse_error() {
-            let err_res = vec![
-                (
-                    r#"{"error_msg": "Permission denied"}"#,
-                    ErrorKind::PermissionDenied,
-                    StatusCode::FORBIDDEN,
-                ),
-                (
-                    r#"{"error_msg": "Folder /e982e75a-fead-487c-9f41-63094d9bf0de/a9d867b9-778d-4612-b674-47e674c14c28/ not found."}"#,
-                    ErrorKind::NotFound,
-                    StatusCode::NOT_FOUND,
-                ),
-            ];
-
-            for res in err_res {
-                let bs = bytes::Bytes::from(res.0);
-                let body = Buffer::from(bs);
-                let resp = Response::builder().status(res.2).body(body).unwrap();
-
-                let err = parse_error(resp);
-
-                assert_eq!(err.kind(), res.1);
-            }
+#[cfg(test)]
+mod tests {
+    use http::StatusCode;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"error_msg": "Permission denied"}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::FORBIDDEN,
+            ),
+            (
+                r#"{"error_msg": "Folder /e982e75a-fead-487c-9f41-63094d9bf0de/a9d867b9-778d-4612-b674-47e674c14c28/ not found."}"#,
+                ErrorKind::NotFound,
+                StatusCode::NOT_FOUND,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
+
+            let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
+
+            assert_eq!(err.kind(), res.1);
         }
     }
 }
-
-pub(super) use error::*;

data/core/services/seafile/src/lister.rs

@@ -47,7 +47,7 @@ impl oio::PageList for SeafileLister {
                 // add path itself
                 ctx.entries.push_back(Entry::new(
                     self.path.as_str(),
-                    Metadata::new(EntryMode::DIR),
+                    MetadataBuilder::dir().build(),
                 ));
 
                 for info in infos {
@@ -58,15 +58,18 @@ impl oio::PageList for SeafileLister {
                         );
 
                         let entry = if info.type_field == "file" {
-                            let mut meta = Metadata::new(EntryMode::FILE)
-                                .with_last_modified(Timestamp::from_second(info.mtime)?);
-                            if let Some(size) = info.size {
-                                meta.set_content_length(size);
-                            }
-                            Entry::new(&rel_path, meta)
+                            let size = info.size.ok_or_else(|| {
+                                Error::new(
+                                    ErrorKind::Unexpected,
+                                    "seafile list response does not contain file size",
+                                )
+                            })?;
+                            let mut meta = MetadataBuilder::file(size);
+                            meta.last_modified(Timestamp::from_second(info.mtime)?);
+                            Entry::new(&rel_path, meta.build())
                         } else {
                             let path = format!("{rel_path}/");
-                            Entry::new(&path, Metadata::new(EntryMode::DIR))
+                            Entry::new(&path, MetadataBuilder::dir().build())
                         };
 
                         ctx.entries.push_back(entry);

data/core/services/seafile/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -60,7 +60,10 @@ impl oio::StreamRead for SeafileReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("DownloadFile")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/seafile/src/writer.rs

@@ -19,8 +19,8 @@ use std::sync::Arc;
 
 use http::StatusCode;
 
-use super::core::SeafileCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, SeafileCore};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -50,8 +50,11 @@ impl oio::OneShotWrite for SeafileWriter {
 
         let status = resp.status();
         match status {
-            StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadFile")),
+                resp,
+            )),
         }
     }
 }

data/core/services/sftp/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 
 bytes = { workspace = true }
 fastpool = "1.0.2"

data/core/services/sftp/src/backend.rs

@@ -208,6 +208,7 @@ impl Service for SftpBackend {
     type Lister = SftpLazyLister;
     type Deleter = oio::OneShotDeleter<SftpDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -250,7 +251,7 @@ impl Service for SftpBackend {
         let mut fs = client.fs();
         fs.set_cwd(&self.core.root);
 
-        let meta: Metadata = to_metadata(fs.metadata(path).await.map_err(parse_sftp_error)?);
+        let meta: Metadata = to_metadata(fs.metadata(path).await.map_err(parse_sftp_error)?)?;
 
         Ok(RpStat::new(meta))
     }
@@ -289,8 +290,7 @@ impl Service for SftpBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
         let backend = self.clone();
         let ctx = ctx.clone();
@@ -313,12 +313,27 @@ impl Service for SftpBackend {
             let mut src_file = client.open(&src).await.map_err(parse_sftp_error)?;
             let mut dst_file = client.create(dst).await.map_err(parse_sftp_error)?;
 
+            let source_size = match args.source_content_length_hint() {
+                Some(size) => size,
+                None => src_file
+                    .metadata()
+                    .await
+                    .map_err(parse_sftp_error)?
+                    .len()
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "sftp source metadata does not contain file length",
+                        )
+                    })?,
+            };
+
             src_file
                 .copy_all_to(&mut dst_file)
                 .await
                 .map_err(parse_sftp_error)?;
 
-            Ok(Metadata::default())
+            Ok(MetadataBuilder::file(source_size).build())
         }))
     }
 

data/core/services/sftp/src/core.rs

@@ -160,63 +160,63 @@ impl ManageObject for Manager {
     }
 }
 
-mod error {
-    use openssh::Error as SshError;
-    use openssh_sftp_client::Error as SftpClientError;
-    use openssh_sftp_client::error::SftpErrorKind;
-
-    use opendal_core::Error;
-    use opendal_core::ErrorKind;
-
-    pub fn parse_sftp_error(e: SftpClientError) -> Error {
-        let kind = match &e {
-            SftpClientError::UnsupportedSftpProtocol { version: _ } => ErrorKind::Unsupported,
-            SftpClientError::SftpError(kind, _msg) => match kind {
-                SftpErrorKind::NoSuchFile => ErrorKind::NotFound,
-                SftpErrorKind::PermDenied => ErrorKind::PermissionDenied,
-                SftpErrorKind::OpUnsupported => ErrorKind::Unsupported,
-                _ => ErrorKind::Unexpected,
-            },
+use openssh::Error as SshError;
+use openssh_sftp_client::Error as SftpClientError;
+use openssh_sftp_client::error::SftpErrorKind;
+
+use opendal_core::Error;
+use opendal_core::ErrorKind;
+
+pub fn parse_sftp_error(e: SftpClientError) -> Error {
+    let kind = match &e {
+        SftpClientError::UnsupportedSftpProtocol { version: _ } => ErrorKind::Unsupported,
+        SftpClientError::SftpError(kind, _msg) => match kind {
+            SftpErrorKind::NoSuchFile => ErrorKind::NotFound,
+            SftpErrorKind::PermDenied => ErrorKind::PermissionDenied,
+            SftpErrorKind::OpUnsupported => ErrorKind::Unsupported,
             _ => ErrorKind::Unexpected,
-        };
+        },
+        _ => ErrorKind::Unexpected,
+    };
 
-        let mut err = Error::new(kind, "sftp error").set_source(e);
+    let mut err = Error::new(kind, "sftp error").set_source(e);
 
-        // Mark error as temporary if it's unexpected.
-        if kind == ErrorKind::Unexpected {
-            err = err.set_temporary();
-        }
-
-        err
+    // Mark error as temporary if it's unexpected.
+    if kind == ErrorKind::Unexpected {
+        err = err.set_temporary();
     }
 
-    pub fn parse_ssh_error(e: SshError) -> Error {
-        Error::new(ErrorKind::Unexpected, "ssh error").set_source(e)
-    }
+    err
+}
 
-    pub(crate) fn is_not_found(e: &SftpClientError) -> bool {
-        matches!(e, SftpClientError::SftpError(SftpErrorKind::NoSuchFile, _))
-    }
+pub fn parse_ssh_error(e: SshError) -> Error {
+    Error::new(ErrorKind::Unexpected, "ssh error").set_source(e)
+}
 
-    pub(crate) fn is_sftp_protocol_error(e: &SftpClientError) -> bool {
-        matches!(e, SftpClientError::SftpError(_, _))
-    }
+pub(crate) fn is_not_found(e: &SftpClientError) -> bool {
+    matches!(e, SftpClientError::SftpError(SftpErrorKind::NoSuchFile, _))
+}
 
-    pub(crate) fn is_sftp_failure(e: &SftpClientError) -> bool {
-        matches!(e, SftpClientError::SftpError(SftpErrorKind::Failure, _))
-    }
+pub(crate) fn is_sftp_protocol_error(e: &SftpClientError) -> bool {
+    matches!(e, SftpClientError::SftpError(_, _))
 }
 
-pub(super) use error::*;
+pub(crate) fn is_sftp_failure(e: &SftpClientError) -> bool {
+    matches!(e, SftpClientError::SftpError(SftpErrorKind::Failure, _))
+}
 
 mod utils {
     use openssh_sftp_client::metadata::MetaData as SftpMeta;
 
     use opendal_core::EntryMode;
+    use opendal_core::Error;
+    use opendal_core::ErrorKind;
     use opendal_core::Metadata;
+    use opendal_core::MetadataBuilder;
+    use opendal_core::Result;
     use opendal_core::raw::Timestamp;
 
-    pub fn to_metadata(meta: SftpMeta) -> Metadata {
+    pub fn to_metadata(meta: SftpMeta) -> Result<Metadata> {
         let mode = meta
             .file_type()
             .map(|filetype| {
@@ -230,19 +230,24 @@ mod utils {
             })
             .unwrap_or(EntryMode::Unknown);
 
-        let mut metadata = Metadata::new(mode);
-
-        if let Some(size) = meta.len() {
-            metadata.set_content_length(size);
-        }
+        let mut metadata = match mode {
+            EntryMode::FILE => MetadataBuilder::file(meta.len().ok_or_else(|| {
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "sftp metadata does not contain file length",
+                )
+            })?),
+            EntryMode::DIR => MetadataBuilder::dir(),
+            EntryMode::Unknown => MetadataBuilder::unknown(),
+        };
 
         if let Some(modified) = meta.modified()
             && let Ok(m) = Timestamp::try_from(modified.as_system_time())
         {
-            metadata.set_last_modified(m);
+            metadata.last_modified(m);
         }
 
-        metadata
+        Ok(metadata.build())
     }
 }
 

data/core/services/sftp/src/lister.rs

@@ -62,9 +62,9 @@ impl oio::List for SftpLister {
                         if self.prefix.is_empty() {
                             path = "/";
                         }
-                        return Ok(Some(Entry::new(path, to_metadata(e.metadata()))));
+                        return Ok(Some(Entry::new(path, to_metadata(e.metadata())?)));
                     } else {
-                        return Ok(Some(map_entry(self.prefix.as_str(), e)));
+                        return Ok(Some(map_entry(self.prefix.as_str(), e)?));
                     }
                 }
                 None => return Ok(None),
@@ -73,7 +73,7 @@ impl oio::List for SftpLister {
     }
 }
 
-fn map_entry(prefix: &str, value: DirEntry) -> Entry {
+fn map_entry(prefix: &str, value: DirEntry) -> Result<Entry> {
     let path = format!(
         "{}{}{}",
         prefix,
@@ -85,5 +85,5 @@ fn map_entry(prefix: &str, value: DirEntry) -> Entry {
         }
     );
 
-    Entry::new(path.as_str(), to_metadata(value.metadata()))
+    Ok(Entry::new(path.as_str(), to_metadata(value.metadata())?))
 }

data/core/services/sftp/src/writer.rs

@@ -55,7 +55,7 @@ impl oio::Write for SftpWriter {
     async fn close(&mut self) -> Result<Metadata> {
         self.file.shutdown().await.map_err(new_std_io_error)?;
 
-        Ok(Metadata::default())
+        Ok(MetadataBuilder::unknown().build())
     }
 
     async fn abort(&mut self) -> Result<()> {

data/core/services/sled/Cargo.toml

@@ -32,7 +32,7 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 sled = "0.34.7"