opendal 0.1.9 → 0.1.11

data/core/services/oss/src/core.rs

@@ -240,13 +240,13 @@ impl OssCore {
     ///
     /// before return the user defined metadata, we'll strip the user_metadata_prefix from the key
     pub fn parse_metadata(&self, path: &str, headers: &HeaderMap) -> Result<Metadata> {
-        let mut m = parse_into_metadata(path, headers)?;
+        let mut m = parse_into_metadata(path, headers)?.into_builder();
         let user_meta = parse_prefixed_headers(headers, X_OSS_META_PREFIX);
         if !user_meta.is_empty() {
-            m = m.with_user_metadata(user_meta);
+            m.user_metadata(user_meta);
         }
 
-        Ok(m)
+        Ok(m.build())
     }
 }
 
@@ -657,33 +657,36 @@ impl OssCore {
         }
     }
 
-    #[allow(clippy::too_many_arguments)]
     pub async fn oss_initiate_upload(
         &self,
         ctx: &OperationContext,
         path: &str,
-        content_type: Option<&str>,
-        content_disposition: Option<&str>,
-        cache_control: Option<&str>,
-        content_encoding: Option<&str>,
+        args: &OpWrite,
         is_presign: bool,
     ) -> Result<Response<Buffer>> {
         let path = build_abs_path(&self.root, path);
         let endpoint = self.get_endpoint(is_presign);
         let url = format!("{}/{}?uploads", endpoint, percent_encode_path(&path));
         let mut req = Request::post(&url);
-        if let Some(mime) = content_type {
+        if let Some(mime) = args.content_type() {
             req = req.header(CONTENT_TYPE, mime);
         }
-        if let Some(disposition) = content_disposition {
+        if let Some(disposition) = args.content_disposition() {
             req = req.header(CONTENT_DISPOSITION, disposition);
         }
-        if let Some(cache_control) = cache_control {
+        if let Some(cache_control) = args.cache_control() {
             req = req.header(CACHE_CONTROL, cache_control);
         }
-        if let Some(encoding) = content_encoding {
+        if let Some(encoding) = args.content_encoding() {
             req = req.header(CONTENT_ENCODING, encoding);
         }
+        // OSS evaluates x-oss-forbid-overwrite on both InitiateMultipartUpload and
+        // CompleteMultipartUpload. Setting it only on one of them is not enough.
+        //
+        // ref: https://www.alibabacloud.com/help/en/oss/developer-reference/initiatemultipartupload
+        if args.if_not_exists() {
+            req = req.header(X_OSS_FORBID_OVERWRITE, "true");
+        }
         req = self.insert_sse_headers(req);
 
         let req = req
@@ -737,6 +740,7 @@ impl OssCore {
         upload_id: &str,
         is_presign: bool,
         parts: Vec<MultipartUploadPart>,
+        args: &OpWrite,
     ) -> Result<Response<Buffer>> {
         let p = build_abs_path(&self.root, path);
         let endpoint = self.get_endpoint(is_presign);
@@ -747,16 +751,23 @@ impl OssCore {
             percent_encode_path(upload_id)
         );
 
-        let req = Request::post(&url);
+        let mut req = Request::post(&url);
 
         let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest {
             part: parts.to_vec(),
         })
         .map_err(new_xml_serialize_error)?;
         // Make sure content length has been set to avoid post with chunked encoding.
-        let req = req.header(CONTENT_LENGTH, content.len());
+        req = req.header(CONTENT_LENGTH, content.len());
         // Set content-type to `application/xml` to avoid mixed with form post.
-        let req = req.header(CONTENT_TYPE, "application/xml");
+        req = req.header(CONTENT_TYPE, "application/xml");
+        // CompleteMultipartUpload is the request that commits the object, so
+        // if_not_exists must also be enforced here.
+        //
+        // ref: https://www.alibabacloud.com/help/en/oss/developer-reference/completemultipartupload
+        if args.if_not_exists() {
+            req = req.header(X_OSS_FORBID_OVERWRITE, "true");
+        }
 
         let req = req
             .extension(Operation::Write)
@@ -1133,71 +1144,12 @@ mod tests {
             ]
         )
     }
-}
-
-mod error {
-    use bytes::Buf;
-    use http::Response;
-    use http::StatusCode;
-    use quick_xml::de;
-    use serde::Deserialize;
-
-    use opendal_core::raw::*;
-    use opendal_core::*;
-
-    /// OssError is the error returned by oss service.
-    #[derive(Default, Debug, Deserialize)]
-    #[serde(default, rename_all = "PascalCase")]
-    struct OssError {
-        code: String,
-        message: String,
-        request_id: String,
-        host_id: String,
-    }
-
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
 
-        let (kind, retryable) = match parts.status {
-            StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
-            StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
-            StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED | StatusCode::CONFLICT => {
-                (ErrorKind::ConditionNotMatch, false)
-            }
-            StatusCode::INTERNAL_SERVER_ERROR
-            | StatusCode::BAD_GATEWAY
-            | StatusCode::SERVICE_UNAVAILABLE
-            | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
-
-        let message = match de::from_reader::<_, OssError>(bs.clone().reader()) {
-            Ok(oss_err) => format!("{oss_err:?}"),
-            Err(_) => String::from_utf8_lossy(&bs).into_owned(),
-        };
-
-        let mut err = Error::new(kind, message);
-
-        err = with_error_response_context(err, parts);
-
-        if retryable {
-            err = err.set_temporary();
-        }
-
-        err
-    }
-
-    #[cfg(test)]
-    mod tests {
-        use super::*;
-
-        /// Error response example is from https://www.alibabacloud.com/help/en/object-storage-service/latest/error-responses
-        #[test]
-        fn test_parse_error() {
-            let bs = bytes::Bytes::from(
-                r#"
+    /// Error response example is from https://www.alibabacloud.com/help/en/object-storage-service/latest/error-responses
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
 <?xml version="1.0" ?>
 <Error xmlns="http://doc.oss-cn-hangzhou.aliyuncs.com">
     <Code>
@@ -1214,20 +1166,168 @@ mod error {
     </HostId>
 </Error>
 "#,
-            );
-
-            let out: OssError = de::from_reader(bs.reader()).expect("must success");
-            println!("{out:?}");
-
-            assert_eq!(out.code.trim(), "AccessDenied");
-            assert_eq!(
-                out.message.trim(),
-                "Query-string authentication requires the Signature, Expires and OSSAccessKeyId parameters"
-            );
-            assert_eq!(out.request_id.trim(), "1D842BC54255****");
-            assert_eq!(out.host_id.trim(), "oss-cn-hangzhou.aliyuncs.com");
+        );
+
+        let out: OssError = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code.trim(), "AccessDenied");
+        assert_eq!(
+            out.message.trim(),
+            "Query-string authentication requires the Signature, Expires and OSSAccessKeyId parameters"
+        );
+        assert_eq!(out.request_id.trim(), "1D842BC54255****");
+        assert_eq!(out.host_id.trim(), "oss-cn-hangzhou.aliyuncs.com");
+    }
+}
+
+use bytes::Buf;
+use http::StatusCode;
+use quick_xml::de;
+
+/// OssError is the error returned by oss service.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct OssError {
+    code: String,
+    message: String,
+    request_id: String,
+    host_id: String,
+}
+
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+    caller_condition: bool,
+    if_not_exists: bool,
+}
+
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self {
+            service_operation,
+            caller_condition: false,
+            if_not_exists: false,
         }
     }
+
+    pub(crate) const fn with_caller_condition(mut self, caller_condition: bool) -> Self {
+        self.caller_condition = caller_condition;
+        self
+    }
+
+    pub(crate) const fn with_if_not_exists(mut self, if_not_exists: bool) -> Self {
+        self.caller_condition = self.caller_condition || if_not_exists;
+        self.if_not_exists = if_not_exists;
+        self
+    }
+}
+
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let oss_error = de::from_reader::<_, OssError>(bs.clone().reader()).ok();
+
+    let (mut kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED if ctx.caller_condition => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    if let Some(oss_error) = &oss_error {
+        match oss_error.code.as_str() {
+            "PreconditionFailed" if ctx.caller_condition => {
+                (kind, retryable) = (ErrorKind::ConditionNotMatch, false);
+            }
+            "FileAlreadyExists" if ctx.if_not_exists => {
+                (kind, retryable) = (ErrorKind::ConditionNotMatch, false);
+            }
+            "FileAlreadyExists" | "FileImmutable" => {
+                (kind, retryable) = (ErrorKind::Conflict, false);
+            }
+            _ if matches!(
+                parts.status,
+                StatusCode::CONFLICT | StatusCode::PRECONDITION_FAILED
+            ) =>
+            {
+                (kind, retryable) = (ErrorKind::Unexpected, false);
+            }
+            _ => {}
+        }
+    }
+
+    let message = match oss_error {
+        Some(oss_err) => format!("{oss_err:?}"),
+        None => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
 }
 
-pub(super) use error::*;
+#[cfg(test)]
+mod error_tests {
+    use super::*;
+
+    fn parse_oss_error(ctx: ErrorContext, status: StatusCode, code: &str) -> Error {
+        let body = Buffer::from(format!(
+            "<Error><Code>{code}</Code><Message>test</Message></Error>"
+        ));
+        let resp = Response::builder()
+            .status(status)
+            .body(body)
+            .expect("response must build");
+        parse_error(ctx, resp)
+    }
+
+    #[test]
+    fn conflict_classification_uses_native_code_and_condition() {
+        let conditional = ErrorContext::new(ServiceOperation("PutObject")).with_if_not_exists(true);
+        assert_eq!(
+            parse_oss_error(conditional, StatusCode::CONFLICT, "FileAlreadyExists").kind(),
+            ErrorKind::ConditionNotMatch
+        );
+        assert_eq!(
+            parse_oss_error(
+                conditional,
+                StatusCode::PRECONDITION_FAILED,
+                "PreconditionFailed"
+            )
+            .kind(),
+            ErrorKind::ConditionNotMatch
+        );
+
+        let unconditional = ErrorContext::new(ServiceOperation("PutObject"));
+        assert_eq!(
+            parse_oss_error(unconditional, StatusCode::CONFLICT, "FileAlreadyExists").kind(),
+            ErrorKind::Conflict
+        );
+        assert_eq!(
+            parse_oss_error(unconditional, StatusCode::CONFLICT, "FileImmutable").kind(),
+            ErrorKind::Conflict
+        );
+        assert_eq!(
+            parse_oss_error(unconditional, StatusCode::CONFLICT, "UnknownConflict").kind(),
+            ErrorKind::Unexpected
+        );
+    }
+}

data/core/services/oss/src/deleter.rs

@@ -46,7 +46,10 @@ impl oio::BatchDelete for OssDeleter {
 
         match status {
             StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("DeleteObject")),
+                resp,
+            )),
         }
     }
 
@@ -63,7 +66,10 @@ impl oio::BatchDelete for OssDeleter {
         let status = resp.status();
 
         if status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("DeleteMultipleObjects")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();
@@ -88,10 +94,14 @@ impl oio::BatchDelete for OssDeleter {
 
         for i in result.deleted {
             let path = build_rel_path(&self.core.root, &i.key);
-            let mut op = OpDelete::default();
-            if let Some(version) = &i.version_id {
-                op = op.with_version(version);
-            }
+            let op = OpDelete::from_options(
+                &Capability::default(),
+                options::DeleteOptions {
+                    version: i.version_id,
+                    ..Default::default()
+                },
+            )
+            .expect("delete result does not contain a logical condition");
             let object = (path, op);
             keys.remove(&object);
             batched_result.succeeded.push(object);

data/core/services/oss/src/lister.rs

@@ -80,7 +80,10 @@ impl oio::PageList for OssLister {
             .await?;
 
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListObjectsV2")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();
@@ -94,7 +97,7 @@ impl oio::PageList for OssLister {
         for prefix in output.common_prefixes {
             let de = oio::Entry::new(
                 &build_rel_path(&self.core.root, &prefix.prefix),
-                Metadata::new(EntryMode::DIR),
+                MetadataBuilder::dir().build(),
             );
             ctx.entries.push_back(de);
         }
@@ -108,14 +111,17 @@ impl oio::PageList for OssLister {
                 continue;
             }
 
-            let mut meta = Metadata::new(EntryMode::from_path(&path));
-            meta.set_is_current(true);
-            meta.set_etag(&object.etag);
-            meta.set_content_md5(object.etag.trim_matches('"'));
-            meta.set_content_length(object.size);
-            meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
-
-            let de = oio::Entry::with(path, meta);
+            let mut meta = if path.ends_with('/') {
+                MetadataBuilder::dir()
+            } else {
+                MetadataBuilder::file(object.size)
+            };
+            meta.is_current(Some(true));
+            meta.etag(&object.etag);
+            meta.content_md5(object.etag.trim_matches('"'));
+            meta.last_modified(object.last_modified.parse::<Timestamp>()?);
+
+            let de = oio::Entry::with(path, meta.build());
             ctx.entries.push_back(de);
         }
 
@@ -176,7 +182,10 @@ impl oio::PageList for OssObjectVersionsLister {
             )
             .await?;
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListObjectVersions")),
+                resp,
+            ));
         }
 
         let body = resp.into_body();
@@ -203,7 +212,7 @@ impl oio::PageList for OssObjectVersionsLister {
         for prefix in output.common_prefixes {
             let de = oio::Entry::new(
                 &build_rel_path(&self.core.root, &prefix.prefix),
-                Metadata::new(EntryMode::DIR),
+                MetadataBuilder::dir().build(),
             );
             ctx.entries.push_back(de);
         }
@@ -222,17 +231,20 @@ impl oio::PageList for OssObjectVersionsLister {
                 path = "/".to_owned();
             }
 
-            let mut meta = Metadata::new(EntryMode::from_path(&path));
-            meta.set_version(&version_object.version_id);
-            meta.set_is_current(version_object.is_latest);
-            meta.set_content_length(version_object.size);
-            meta.set_last_modified(version_object.last_modified.parse::<Timestamp>()?);
+            let mut meta = if path.ends_with('/') {
+                MetadataBuilder::dir()
+            } else {
+                MetadataBuilder::file(version_object.size)
+            };
+            meta.version(&version_object.version_id);
+            meta.is_current(Some(version_object.is_latest));
+            meta.last_modified(version_object.last_modified.parse::<Timestamp>()?);
             if let Some(etag) = version_object.etag {
-                meta.set_etag(&etag);
-                meta.set_content_md5(etag.trim_matches('"'));
+                meta.etag(&etag);
+                meta.content_md5(etag.trim_matches('"'));
             }
 
-            let entry = oio::Entry::new(&path, meta);
+            let entry = oio::Entry::new(&path, meta.build());
             ctx.entries.push_back(entry);
         }
 
@@ -243,13 +255,17 @@ impl oio::PageList for OssObjectVersionsLister {
                     path = "/".to_owned();
                 }
 
-                let mut meta = Metadata::new(EntryMode::FILE);
-                meta.set_version(&delete_marker.version_id);
-                meta.set_is_deleted(true);
-                meta.set_is_current(delete_marker.is_latest);
-                meta.set_last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
-
-                let entry = oio::Entry::new(&path, meta);
+                let mut meta = if path.ends_with('/') {
+                    MetadataBuilder::dir()
+                } else {
+                    MetadataBuilder::file(0)
+                };
+                meta.version(&delete_marker.version_id);
+                meta.is_deleted(true);
+                meta.is_current(Some(delete_marker.is_latest));
+                meta.last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
+
+                let entry = oio::Entry::new(&path, meta.build());
                 ctx.entries.push_back(entry);
             }
         }

data/core/services/oss/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -66,7 +66,11 @@ impl oio::StreamRead for OssReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("GetObject"))
+                        .with_caller_condition(args.is_conditional()),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/oss/src/writer.rs

@@ -47,18 +47,18 @@ impl OssWriter {
     }
 
     fn parse_metadata(headers: &HeaderMap<HeaderValue>) -> Result<Metadata> {
-        let mut meta = Metadata::default();
+        let mut meta = MetadataBuilder::unknown();
         if let Some(etag) = parse_etag(headers)? {
-            meta.set_etag(etag);
+            meta.etag(etag);
         }
         if let Some(md5) = parse_content_md5(headers)? {
-            meta.set_content_md5(md5);
+            meta.content_md5(md5);
         }
         if let Some(version) = parse_header_to_str(headers, constants::X_OSS_VERSION_ID)? {
-            meta.set_version(version);
+            meta.version(version);
         }
 
-        Ok(meta)
+        Ok(meta.build())
     }
 }
 
@@ -76,22 +76,18 @@ impl oio::MultipartWrite for OssWriter {
 
         match status {
             StatusCode::CREATED | StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("PutObject"))
+                    .with_if_not_exists(self.op.if_not_exists()),
+                resp,
+            )),
         }
     }
 
     async fn initiate_part(&self) -> Result<String> {
         let resp = self
             .core
-            .oss_initiate_upload(
-                &self.ctx,
-                &self.path,
-                self.op.content_type(),
-                self.op.content_disposition(),
-                self.op.cache_control(),
-                self.op.content_encoding(),
-                false,
-            )
+            .oss_initiate_upload(&self.ctx, &self.path, &self.op, false)
             .await?;
 
         let status = resp.status();
@@ -106,7 +102,11 @@ impl oio::MultipartWrite for OssWriter {
 
                 Ok(result.upload_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("InitiateMultipartUpload"))
+                    .with_if_not_exists(self.op.if_not_exists()),
+                resp,
+            )),
         }
     }
 
@@ -153,7 +153,10 @@ impl oio::MultipartWrite for OssWriter {
                     size: None,
                 })
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadPart")),
+                resp,
+            )),
         }
     }
 
@@ -172,7 +175,9 @@ impl oio::MultipartWrite for OssWriter {
 
         let resp = self
             .core
-            .oss_complete_multipart_upload_request(&self.ctx, &self.path, upload_id, false, parts)
+            .oss_complete_multipart_upload_request(
+                &self.ctx, &self.path, upload_id, false, parts, &self.op,
+            )
             .await?;
 
         let meta = Self::parse_metadata(resp.headers())?;
@@ -180,7 +185,11 @@ impl oio::MultipartWrite for OssWriter {
 
         match status {
             StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CompleteMultipartUpload"))
+                    .with_if_not_exists(self.op.if_not_exists()),
+                resp,
+            )),
         }
     }
 
@@ -192,7 +201,10 @@ impl oio::MultipartWrite for OssWriter {
         match resp.status() {
             // OSS returns code 204 if abort succeeds.
             StatusCode::NO_CONTENT => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("AbortMultipartUpload")),
+                resp,
+            )),
         }
     }
 }
@@ -216,7 +228,10 @@ impl oio::AppendWrite for OssWriter {
                 Ok(content_length)
             }
             StatusCode::NOT_FOUND => Ok(0),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("HeadObject")),
+                resp,
+            )),
         }
     }
 
@@ -233,7 +248,10 @@ impl oio::AppendWrite for OssWriter {
 
         match status {
             StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("AppendObject")),
+                resp,
+            )),
         }
     }
 }

data/core/services/pcloud/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }