opendal 0.1.10 → 0.1.12

data/core/services/azfile/src/writer.rs

@@ -19,8 +19,8 @@ use std::sync::Arc;
 
 use http::StatusCode;
 
-use super::core::AzfileCore;
 use super::core::parse_error;
+use super::core::{AzfileCore, ErrorContext};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -50,17 +50,17 @@ impl AzfileWriter {
     }
 
     fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
-        let mut metadata = Metadata::default();
+        let mut metadata = MetadataBuilder::unknown();
 
         if let Some(last_modified) = parse_last_modified(headers)? {
-            metadata.set_last_modified(last_modified);
+            metadata.last_modified(last_modified);
         }
         let etag = parse_etag(headers)?;
         if let Some(etag) = etag {
-            metadata.set_etag(etag);
+            metadata.etag(etag);
         }
 
-        Ok(metadata)
+        Ok(metadata.build())
     }
 }
 
@@ -78,7 +78,10 @@ impl oio::OneShotWrite for AzfileWriter {
         match status {
             StatusCode::OK | StatusCode::CREATED => {}
             _ => {
-                return Err(parse_error(resp).with_operation("Backend::azfile_create_file"));
+                return Err(
+                    parse_error(ErrorContext::new(ServiceOperation("CreateFile")), resp)
+                        .with_operation("Backend::azfile_create_file"),
+                );
             }
         }
 
@@ -87,11 +90,14 @@ impl oio::OneShotWrite for AzfileWriter {
             .azfile_update(&self.ctx, &self.path, size as u64, 0, bs)
             .await?;
         let status = resp.status();
-        let mut meta = AzfileWriter::parse_metadata(resp.headers())?;
-        meta.set_content_length(size as u64);
+        let mut meta = AzfileWriter::parse_metadata(resp.headers())?.into_builder();
+        meta.set_file(size as u64);
         match status {
-            StatusCode::OK | StatusCode::CREATED => Ok(meta),
-            _ => Err(parse_error(resp).with_operation("Backend::azfile_update")),
+            StatusCode::OK | StatusCode::CREATED => Ok(meta.build()),
+            _ => Err(
+                parse_error(ErrorContext::new(ServiceOperation("PutRange")), resp)
+                    .with_operation("Backend::azfile_update"),
+            ),
         }
     }
 }
@@ -109,7 +115,10 @@ impl oio::AppendWrite for AzfileWriter {
 
         match status {
             StatusCode::OK => Ok(parse_content_length(resp.headers())?.unwrap_or_default()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetFileProperties")),
+                resp,
+            )),
         }
     }
 
@@ -120,11 +129,14 @@ impl oio::AppendWrite for AzfileWriter {
             .await?;
 
         let status = resp.status();
-        let mut meta = AzfileWriter::parse_metadata(resp.headers())?;
-        meta.set_content_length(offset + size);
+        let mut meta = AzfileWriter::parse_metadata(resp.headers())?.into_builder();
+        meta.set_file(offset + size);
         match status {
-            StatusCode::OK | StatusCode::CREATED => Ok(meta),
-            _ => Err(parse_error(resp).with_operation("Backend::azfile_update")),
+            StatusCode::OK | StatusCode::CREATED => Ok(meta.build()),
+            _ => Err(
+                parse_error(ErrorContext::new(ServiceOperation("PutRange")), resp)
+                    .with_operation("Backend::azfile_update"),
+            ),
         }
     }
 }

data/core/services/azure-common/Cargo.toml

@@ -33,4 +33,4 @@ all-features = true
 
 [dependencies]
 http = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.2", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.1", default-features = false }

data/core/services/b2/Cargo.toml

@@ -36,7 +36,7 @@ asyncband = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.2", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/b2/src/backend.rs

@@ -25,11 +25,11 @@ use log::debug;
 
 use super::B2_SCHEME;
 use super::config::B2Config;
-use super::core::B2Core;
 use super::core::B2Signer;
 use super::core::constants;
 use super::core::parse_error;
 use super::core::parse_file_info;
+use super::core::{B2Core, ErrorContext};
 use super::deleter::B2Deleter;
 use super::lister::B2Lister;
 use super::reader::*;
@@ -224,6 +224,7 @@ impl Service for B2Backend {
     type Lister = oio::PageLister<B2Lister>;
     type Deleter = oio::OneShotDeleter<B2Deleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -250,7 +251,7 @@ impl Service for B2Backend {
     async fn stat(&self, ctx: &OperationContext, path: &str, _args: OpStat) -> Result<RpStat> {
         // Stat root always returns a DIR.
         if path == "/" {
-            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+            return Ok(RpStat::new(MetadataBuilder::dir().build()));
         }
 
         let delimiter = if path.ends_with('/') { Some("/") } else { None };
@@ -317,7 +318,6 @@ impl Service for B2Backend {
         from: &str,
         to: &str,
         _args: OpCopy,
-        _opts: OpCopier,
     ) -> Result<Self::Copier> {
         let core = self.core.clone();
         let ctx = ctx.clone();
@@ -327,6 +327,7 @@ impl Service for B2Backend {
         Ok(oio::OneShotCopier::new(async move {
             let file_info = core.get_file_info(&ctx, &from, None).await?;
             let source_file_id = file_info.file_id;
+            let source_content_length = file_info.content_length;
 
             let Some(source_file_id) = source_file_id else {
                 return Err(Error::new(ErrorKind::IsADirectory, "is a directory"));
@@ -337,8 +338,14 @@ impl Service for B2Backend {
             let status = resp.status();
 
             match status {
-                StatusCode::OK => Ok(Metadata::default()),
-                _ => Err(parse_error(resp)),
+                StatusCode::OK => {
+                    let metadata = MetadataBuilder::file(source_content_length);
+                    Ok(metadata.build())
+                }
+                _ => Err(parse_error(
+                    ErrorContext::new(ServiceOperation("CopyFile")),
+                    resp,
+                )),
             }
         }))
     }
@@ -393,7 +400,7 @@ impl Service for B2Backend {
                     parts.headers,
                 )))
             }
-            PresignOperation::Read(_, _) => {
+            PresignOperation::Read(range, _) => {
                 let resp = self
                     .core
                     .get_download_authorization(ctx, path, args.expire())
@@ -410,7 +417,10 @@ impl Service for B2Backend {
                     resp.authorization_token
                 );
 
-                let req = Request::get(url);
+                let mut req = Request::get(url);
+                if !range.is_full() {
+                    req = req.header(http::header::RANGE, range.to_header());
+                }
 
                 let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 

data/core/services/b2/src/core.rs

@@ -120,7 +120,10 @@ impl B2Core {
                     };
                 }
                 _ => {
-                    return Err(parse_error(resp));
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("AuthorizeAccount")),
+                        resp,
+                    ));
                 }
             }
             Ok(signer.auth_info.clone())
@@ -196,7 +199,10 @@ impl B2Core {
                     .map_err(new_json_deserialize_error)?;
                 Ok(resp)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetUploadUrl")),
+                resp,
+            )),
         }
     }
 
@@ -241,7 +247,10 @@ impl B2Core {
                     .map_err(new_json_deserialize_error)?;
                 Ok(resp)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetDownloadAuthorization")),
+                resp,
+            )),
         }
     }
 
@@ -331,7 +340,7 @@ impl B2Core {
         // B2 uses fileInfo field for custom file info in start_large_file API.
         if let Some(user_metadata) = args.user_metadata() {
             let file_info: HashMap<String, String> = user_metadata
-                .iter()
+                .into_iter()
                 .map(|(k, v)| (k.to_string(), percent_encode_path(v)))
                 .collect();
             start_large_file_request.file_info = Some(file_info);
@@ -385,7 +394,10 @@ impl B2Core {
                     .map_err(new_json_deserialize_error)?;
                 Ok(resp)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetUploadPartUrl")),
+                resp,
+            )),
         }
     }
 
@@ -505,7 +517,10 @@ impl B2Core {
                 }
                 Ok(resp.files.swap_remove(0))
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListFileNames")),
+                resp,
+            )),
         }
     }
 
@@ -783,19 +798,17 @@ pub struct File {
 
 pub(super) fn parse_file_info(file: &File) -> Metadata {
     if file.file_name.ends_with('/') {
-        return Metadata::new(EntryMode::DIR);
+        return MetadataBuilder::dir().build();
     }
 
-    let mut metadata = Metadata::new(EntryMode::FILE);
-
-    metadata.set_content_length(file.content_length);
+    let mut metadata = MetadataBuilder::file(file.content_length);
 
     if let Some(content_md5) = &file.content_md5 {
-        metadata.set_content_md5(content_md5);
+        metadata.content_md5(content_md5);
     }
 
     if let Some(content_type) = &file.content_type {
-        metadata.set_content_type(content_type);
+        metadata.content_type(content_type);
     }
 
     // Parse user metadata from file_info
@@ -808,11 +821,11 @@ pub(super) fn parse_file_info(file: &File) -> Metadata {
             .map(|(k, v)| (k.to_lowercase(), percent_decode_path(v)))
             .collect();
         if !user_metadata.is_empty() {
-            metadata = metadata.with_user_metadata(user_metadata);
+            metadata.user_metadata(user_metadata);
         }
     }
 
-    metadata
+    metadata.build()
 }
 
 #[derive(Debug, Serialize)]
@@ -843,123 +856,124 @@ pub struct GetDownloadAuthorizationResponse {
     pub authorization_token: String,
 }
 
-mod error {
-    use bytes::Buf;
-    use http::Response;
-    use serde::Deserialize;
-
-    use opendal_core::raw::*;
-    use opendal_core::*;
-
-    /// the error response of b2
-    #[derive(Default, Debug, Deserialize)]
-    #[allow(dead_code)]
-    struct B2Error {
-        status: u32,
-        code: String,
-        message: String,
-    }
-
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
-
-        let (mut kind, mut retryable) = match parts.status.as_u16() {
-            403 => (ErrorKind::PermissionDenied, false),
-            404 => (ErrorKind::NotFound, false),
-            304 | 412 => (ErrorKind::ConditionNotMatch, false),
-            // Service b2 could return 403, show the authorization error
-            401 => (ErrorKind::PermissionDenied, true),
-            429 => (ErrorKind::RateLimited, true),
-            500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
-
-        let (message, b2_err) = serde_json::from_reader::<_, B2Error>(bs.clone().reader())
-            .map(|b2_err| (format!("{b2_err:?}"), Some(b2_err)))
-            .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
-
-        if let Some(b2_err) = b2_err {
-            (kind, retryable) =
-                parse_b2_error_code(b2_err.code.as_str()).unwrap_or((kind, retryable));
-        };
-
-        let mut err = Error::new(kind, message);
+/// the error response of b2
+#[derive(Default, Debug, Deserialize)]
+#[allow(dead_code)]
+struct B2Error {
+    status: u32,
+    code: String,
+    message: String,
+}
 
-        err = with_error_response_context(err, parts);
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
 
-        if retryable {
-            err = err.set_temporary();
-        }
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
+    }
+}
 
-        err
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // Service b2 could return 403, show the authorization error
+        401 => (ErrorKind::PermissionDenied, true),
+        429 => (ErrorKind::RateLimited, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, b2_err) = serde_json::from_reader::<_, B2Error>(bs.clone().reader())
+        .map(|b2_err| (format!("{b2_err:?}"), Some(b2_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(b2_err) = b2_err {
+        (kind, retryable) = parse_b2_error_code(b2_err.code.as_str()).unwrap_or((kind, retryable));
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
     }
 
-    /// Returns the `Error kind` of this code and whether the error is retryable.
-    pub(crate) fn parse_b2_error_code(code: &str) -> Option<(ErrorKind, bool)> {
-        match code {
-            "already_hidden" => Some((ErrorKind::AlreadyExists, false)),
-            "no_such_file" => Some((ErrorKind::NotFound, false)),
-            _ => None,
-        }
+    err
+}
+
+/// Returns the `Error kind` of this code and whether the error is retryable.
+pub(crate) fn parse_b2_error_code(code: &str) -> Option<(ErrorKind, bool)> {
+    match code {
+        "already_hidden" => Some((ErrorKind::AlreadyExists, false)),
+        "no_such_file" => Some((ErrorKind::NotFound, false)),
+        _ => None,
     }
+}
 
-    #[cfg(test)]
-    mod test {
-        use http::StatusCode;
+#[cfg(test)]
+mod tests {
+    use http::StatusCode;
 
-        use super::*;
+    use super::*;
 
-        #[test]
-        fn test_parse_b2_error_code() {
-            let code = "already_hidden";
-            assert_eq!(
-                parse_b2_error_code(code),
-                Some((opendal_core::ErrorKind::AlreadyExists, false))
-            );
+    #[test]
+    fn test_parse_b2_error_code() {
+        let code = "already_hidden";
+        assert_eq!(
+            parse_b2_error_code(code),
+            Some((opendal_core::ErrorKind::AlreadyExists, false))
+        );
 
-            let code = "no_such_file";
-            assert_eq!(
-                parse_b2_error_code(code),
-                Some((opendal_core::ErrorKind::NotFound, false))
-            );
+        let code = "no_such_file";
+        assert_eq!(
+            parse_b2_error_code(code),
+            Some((opendal_core::ErrorKind::NotFound, false))
+        );
 
-            let code = "not_found";
-            assert_eq!(parse_b2_error_code(code), None);
-        }
+        let code = "not_found";
+        assert_eq!(parse_b2_error_code(code), None);
+    }
 
-        #[tokio::test]
-        async fn test_parse_error() {
-            let err_res = vec![
-                (
-                    r#"{"status": 403, "code": "access_denied", "message":"The provided customer-managed encryption key is wrong."}"#,
-                    ErrorKind::PermissionDenied,
-                    StatusCode::FORBIDDEN,
-                ),
-                (
-                    r#"{"status": 404, "code": "not_found", "message":"File is not in B2 Cloud Storage."}"#,
-                    ErrorKind::NotFound,
-                    StatusCode::NOT_FOUND,
-                ),
-                (
-                    r#"{"status": 401, "code": "bad_auth_token", "message":"The auth token used is not valid. Call b2_authorize_account again to either get a new one, or an error message describing the problem."}"#,
-                    ErrorKind::PermissionDenied,
-                    StatusCode::UNAUTHORIZED,
-                ),
-            ];
-
-            for res in err_res {
-                let bs = bytes::Bytes::from(res.0);
-                let body = Buffer::from(bs);
-                let resp = Response::builder().status(res.2).body(body).unwrap();
-
-                let err = parse_error(resp);
-
-                assert_eq!(err.kind(), res.1);
-            }
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"status": 403, "code": "access_denied", "message":"The provided customer-managed encryption key is wrong."}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::FORBIDDEN,
+            ),
+            (
+                r#"{"status": 404, "code": "not_found", "message":"File is not in B2 Cloud Storage."}"#,
+                ErrorKind::NotFound,
+                StatusCode::NOT_FOUND,
+            ),
+            (
+                r#"{"status": 401, "code": "bad_auth_token", "message":"The auth token used is not valid. Call b2_authorize_account again to either get a new one, or an error message describing the problem."}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::UNAUTHORIZED,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
+
+            let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
+
+            assert_eq!(err.kind(), res.1);
         }
     }
 }
-
-pub(super) use error::*;

data/core/services/b2/src/deleter.rs

@@ -44,7 +44,7 @@ impl oio::OneShotDelete for B2Deleter {
         match status {
             StatusCode::OK => Ok(()),
             _ => {
-                let err = parse_error(resp);
+                let err = parse_error(ErrorContext::new(ServiceOperation("HideFile")), resp);
                 match err.kind() {
                     ErrorKind::NotFound => Ok(()),
                     // Representative deleted

data/core/services/b2/src/lister.rs

@@ -19,10 +19,10 @@ use std::sync::Arc;
 
 use bytes::Buf;
 
-use super::core::B2Core;
 use super::core::ListFileNamesResponse;
 use super::core::parse_error;
 use super::core::parse_file_info;
+use super::core::{B2Core, ErrorContext};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -80,7 +80,10 @@ impl oio::PageList for B2Lister {
             .await?;
 
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListFileNames")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();

data/core/services/b2/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -60,7 +60,10 @@ impl oio::StreamRead for B2Reader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("DownloadFileByName")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/b2/src/writer.rs

@@ -20,11 +20,11 @@ use std::sync::Arc;
 use bytes::Buf;
 use http::StatusCode;
 
-use super::core::B2Core;
 use super::core::StartLargeFileResponse;
 use super::core::UploadPartResponse;
 use super::core::UploadResponse;
 use super::core::parse_error;
+use super::core::{B2Core, ErrorContext};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -49,19 +49,21 @@ impl B2Writer {
     }
 
     pub fn parse_body_into_meta(path: &str, resp: UploadResponse) -> Metadata {
-        let mut meta = Metadata::new(EntryMode::from_path(path));
+        let mut meta = if path.ends_with('/') {
+            MetadataBuilder::dir()
+        } else {
+            MetadataBuilder::file(resp.content_length)
+        };
 
         if let Some(md5) = resp.content_md5 {
-            meta.set_content_md5(&md5);
+            meta.content_md5(&md5);
         }
 
         if let Some(content_type) = resp.content_type {
-            meta.set_content_type(&content_type);
+            meta.content_type(&content_type);
         }
 
-        meta.set_content_length(resp.content_length);
-
-        meta
+        meta.build()
     }
 }
 
@@ -85,7 +87,10 @@ impl oio::MultipartWrite for B2Writer {
 
                 Ok(meta)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadFile")),
+                resp,
+            )),
         }
     }
 
@@ -106,7 +111,10 @@ impl oio::MultipartWrite for B2Writer {
 
                 Ok(result.file_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("StartLargeFile")),
+                resp,
+            )),
         }
     }
 
@@ -141,7 +149,10 @@ impl oio::MultipartWrite for B2Writer {
                     size: None,
                 })
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadPart")),
+                resp,
+            )),
         }
     }
 
@@ -180,7 +191,10 @@ impl oio::MultipartWrite for B2Writer {
 
                 Ok(meta)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FinishLargeFile")),
+                resp,
+            )),
         }
     }
 
@@ -189,7 +203,10 @@ impl oio::MultipartWrite for B2Writer {
         match resp.status() {
             // b2 returns code 200 if abort succeeds.
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CancelLargeFile")),
+                resp,
+            )),
         }
     }
 }