opendal 0.1.9 → 0.1.11

data/core/services/alluxio/src/core.rs

@@ -80,7 +80,10 @@ impl AlluxioCore {
         let status = resp.status();
         match status {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CreateDirectory")),
+                resp,
+            )),
         }
     }
 
@@ -119,7 +122,10 @@ impl AlluxioCore {
                     serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
                 Ok(steam_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CreateFile")),
+                resp,
+            )),
         }
     }
 
@@ -148,7 +154,10 @@ impl AlluxioCore {
                     serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
                 Ok(steam_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("OpenFile")),
+                resp,
+            )),
         }
     }
 
@@ -173,7 +182,7 @@ impl AlluxioCore {
         match status {
             StatusCode::OK => Ok(()),
             _ => {
-                let err = parse_error(resp);
+                let err = parse_error(ErrorContext::new(ServiceOperation("Delete")), resp);
                 if err.kind() == ErrorKind::NotFound {
                     return Ok(());
                 }
@@ -205,7 +214,10 @@ impl AlluxioCore {
 
         match status {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Rename")),
+                resp,
+            )),
         }
     }
 
@@ -235,7 +247,10 @@ impl AlluxioCore {
                     serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
                 Ok(file_info)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetStatus")),
+                resp,
+            )),
         }
     }
 
@@ -269,7 +284,10 @@ impl AlluxioCore {
                     serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
                 Ok(file_infos)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListStatus")),
+                resp,
+            )),
         }
     }
 
@@ -329,7 +347,10 @@ impl AlluxioCore {
                     serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
                 Ok(size)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("WriteStream")),
+                resp,
+            )),
         }
     }
 
@@ -351,13 +372,18 @@ impl AlluxioCore {
 
         match status {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CloseStream")),
+                resp,
+            )),
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use http::StatusCode;
+
     use super::*;
 
     #[tokio::test]
@@ -377,6 +403,38 @@ mod tests {
 
         assert_eq!(err.kind(), ErrorKind::Unsupported);
     }
+
+    /// Error response example is from https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+    #[test]
+    fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"statusCode":"ALREADY_EXISTS","message":"The resource you requested already exist"}"#,
+                ErrorKind::AlreadyExists,
+            ),
+            (
+                r#"{"statusCode":"NOT_FOUND","message":"The resource you requested does not exist"}"#,
+                ErrorKind::NotFound,
+            ),
+            (
+                r#"{"statusCode":"INTERNAL_SERVER_ERROR","message":"Internal server error"}"#,
+                ErrorKind::Unexpected,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder()
+                .status(StatusCode::INTERNAL_SERVER_ERROR)
+                .body(body)
+                .unwrap();
+
+            let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
+
+            assert_eq!(err.kind(), res.1);
+        }
+    }
 }
 
 #[derive(Debug, Serialize)]
@@ -413,103 +471,64 @@ impl TryFrom<FileInfo> for Metadata {
 
     fn try_from(file_info: FileInfo) -> Result<Metadata> {
         let mut metadata = if file_info.folder {
-            Metadata::new(EntryMode::DIR)
+            MetadataBuilder::dir()
         } else {
-            Metadata::new(EntryMode::FILE)
+            MetadataBuilder::file(file_info.length)
         };
-        metadata
-            .set_content_length(file_info.length)
-            .set_last_modified(Timestamp::from_millisecond(
-                file_info.last_modification_time_ms,
-            )?);
-        Ok(metadata)
+        metadata.last_modified(Timestamp::from_millisecond(
+            file_info.last_modification_time_ms,
+        )?);
+        Ok(metadata.build())
     }
 }
 
-mod error {
-    use bytes::Buf;
-    use http::Response;
-    use serde::Deserialize;
-
-    use opendal_core::raw::*;
-    use opendal_core::*;
-
-    /// the error response of alluxio
-    #[derive(Default, Debug, Deserialize)]
-    #[serde(rename_all = "camelCase")]
-    #[allow(dead_code)]
-    struct AlluxioError {
-        status_code: String,
-        message: String,
+/// the error response of alluxio
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+struct AlluxioError {
+    status_code: String,
+    message: String,
+}
+
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
+
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
     }
+}
 
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
 
-        let mut kind = match parts.status.as_u16() {
-            500 => ErrorKind::Unexpected,
-            _ => ErrorKind::Unexpected,
-        };
+    let mut kind = match parts.status.as_u16() {
+        500 => ErrorKind::Unexpected,
+        _ => ErrorKind::Unexpected,
+    };
 
-        let (message, alluxio_err) =
-            serde_json::from_reader::<_, AlluxioError>(bs.clone().reader())
-                .map(|alluxio_err| (format!("{alluxio_err:?}"), Some(alluxio_err)))
-                .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+    let (message, alluxio_err) = serde_json::from_reader::<_, AlluxioError>(bs.clone().reader())
+        .map(|alluxio_err| (format!("{alluxio_err:?}"), Some(alluxio_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
 
-        if let Some(alluxio_err) = alluxio_err {
-            kind = match alluxio_err.status_code.as_str() {
-                "ALREADY_EXISTS" => ErrorKind::AlreadyExists,
-                "NOT_FOUND" => ErrorKind::NotFound,
-                _ => ErrorKind::Unexpected,
-            }
+    if let Some(alluxio_err) = alluxio_err {
+        kind = match alluxio_err.status_code.as_str() {
+            "ALREADY_EXISTS" => ErrorKind::AlreadyExists,
+            "NOT_FOUND" => ErrorKind::NotFound,
+            _ => ErrorKind::Unexpected,
         }
+    }
 
-        let mut err = Error::new(kind, message);
-
-        err = with_error_response_context(err, parts);
+    let mut err = Error::new(kind, message);
 
-        err
-    }
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
 
-    #[cfg(test)]
-    mod tests {
-        use http::StatusCode;
-
-        use super::*;
-
-        /// Error response example is from https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-        #[test]
-        fn test_parse_error() {
-            let err_res = vec![
-                (
-                    r#"{"statusCode":"ALREADY_EXISTS","message":"The resource you requested already exist"}"#,
-                    ErrorKind::AlreadyExists,
-                ),
-                (
-                    r#"{"statusCode":"NOT_FOUND","message":"The resource you requested does not exist"}"#,
-                    ErrorKind::NotFound,
-                ),
-                (
-                    r#"{"statusCode":"INTERNAL_SERVER_ERROR","message":"Internal server error"}"#,
-                    ErrorKind::Unexpected,
-                ),
-            ];
-
-            for res in err_res {
-                let bs = bytes::Bytes::from(res.0);
-                let body = Buffer::from(bs);
-                let resp = Response::builder()
-                    .status(StatusCode::INTERNAL_SERVER_ERROR)
-                    .body(body)
-                    .unwrap();
-
-                let err = parse_error(resp);
-
-                assert_eq!(err.kind(), res.1);
-            }
-        }
-    }
+    err
 }
-
-pub(super) use error::*;

data/core/services/alluxio/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use opendal_core::raw::*;
 use opendal_core::*;
@@ -47,16 +47,20 @@ impl oio::StreamRead for AlluxioReader {
     async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
         let backend = &self.backend;
         let path = self.path.as_str();
+        let metadata = backend.core.get_status(&self.ctx, path).await?.try_into()?;
         let stream_id = backend.core.open_file(&self.ctx, path).await?;
 
         let resp = backend.core.read(&self.ctx, stream_id, range).await?;
         if !resp.status().is_success() {
             let (part, mut body) = resp.into_parts();
             let buf = body.to_buffer().await?;
-            return Err(parse_error(Response::from_parts(part, buf)));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ReadStream")),
+                Response::from_parts(part, buf),
+            ));
         }
 
-        let rp = RpRead::new(parse_into_metadata(path, resp.headers())?);
+        let rp = RpRead::new(metadata);
         let stream = resp.into_body();
 
         Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))

data/core/services/alluxio/src/writer.rs

@@ -60,11 +60,11 @@ impl oio::Write for AlluxioWriter {
 
     async fn close(&mut self) -> Result<Metadata> {
         let Some(stream_id) = self.stream_id else {
-            return Ok(Metadata::default());
+            return Ok(MetadataBuilder::unknown().build());
         };
         self.core.close(&self.ctx, stream_id).await?;
 
-        Ok(Metadata::default())
+        Ok(MetadataBuilder::unknown().build())
     }
 
     async fn abort(&mut self) -> Result<()> {

data/core/services/azblob/Cargo.toml

@@ -36,10 +36,10 @@ base64 = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false, features = [
   "reqsign",
 ] }
-opendal-service-azure-common = { path = "../azure-common", version = "0.58.1" }
+opendal-service-azure-common = { path = "../azure-common", version = "0.59.0" }
 quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
 reqsign-azure-storage = { version = "3.1.1", default-features = false }
 reqsign-core = { version = "3.2.0", default-features = false }

data/core/services/azblob/src/backend.rs

@@ -37,6 +37,7 @@ use super::config::AzblobConfig;
 use super::copier::AzblobCopiers;
 use super::copier::new_azblob_copier;
 use super::core::AzblobCore;
+use super::core::ErrorContext;
 use super::core::constants::AZBLOB_COPY_MAX_BLOCK_SIZE;
 use super::core::constants::AZBLOB_COPY_MIN_BLOCK_SIZE;
 use super::core::constants::X_MS_META_PREFIX;
@@ -407,15 +408,20 @@ impl Builder for AzblobBuilder {
             write_can_multi: true,
             write_with_cache_control: true,
             write_with_content_type: true,
+            write_with_if_match: true,
             write_with_if_not_exists: true,
             write_with_if_none_match: true,
             write_with_user_metadata: true,
 
             delete: true,
+            delete_with_if_match: true,
+            delete_with_if_none_match: true,
             delete_max_size: Some(AZBLOB_BATCH_LIMIT),
 
             copy: true,
             copy_with_if_not_exists: true,
+            copy_with_if_match: true,
+            copy_with_if_none_match: true,
             copy_can_multi: true,
             copy_multi_min_size: Some(AZBLOB_COPY_MIN_BLOCK_SIZE),
             copy_multi_max_size: Some(AZBLOB_COPY_MAX_BLOCK_SIZE),
@@ -462,6 +468,7 @@ impl Service for AzblobBackend {
     type Lister = oio::PageLister<AzblobLister>;
     type Deleter = oio::BatchDeleter<AzblobDeleter>;
     type Copier = AzblobCopiers;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -484,6 +491,8 @@ impl Service for AzblobBackend {
     }
 
     async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
+        let error_ctx = ErrorContext::new(ServiceOperation("GetBlobProperties"))
+            .with_caller_condition(args.is_conditional());
         let resp = self
             .core
             .azblob_get_blob_properties(ctx, path, &args)
@@ -494,19 +503,19 @@ impl Service for AzblobBackend {
         match status {
             StatusCode::OK => {
                 let headers = resp.headers();
-                let mut meta = parse_into_metadata(path, headers)?;
+                let mut meta = parse_into_metadata(path, headers)?.into_builder();
                 if let Some(version_id) = parse_header_to_str(headers, X_MS_VERSION_ID)? {
-                    meta.set_version(version_id);
+                    meta.version(version_id);
                 }
 
                 let user_meta = parse_prefixed_headers(headers, X_MS_META_PREFIX);
                 if !user_meta.is_empty() {
-                    meta = meta.with_user_metadata(user_meta);
+                    meta.user_metadata(user_meta);
                 }
 
-                Ok(RpStat::new(meta))
+                Ok(RpStat::new(meta.build()))
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(error_ctx, resp)),
         }
     }
     fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -579,10 +588,9 @@ impl Service for AzblobBackend {
         from: &str,
         to: &str,
         args: OpCopy,
-        opts: OpCopier,
     ) -> Result<Self::Copier> {
         let output: AzblobCopiers = {
-            let copier = new_azblob_copier(self.core.clone(), ctx, from, to, args, opts)?;
+            let copier = new_azblob_copier(self.core.clone(), ctx, from, to, args)?;
             Ok(copier)
         }?;
 
@@ -611,9 +619,9 @@ impl Service for AzblobBackend {
         let req = match args.operation() {
             PresignOperation::Stat(v) => self.core.azblob_head_blob_request(path, v),
             PresignOperation::Read(range, v) => self.core.azblob_get_blob_request(path, *range, v),
-            PresignOperation::Write(_) => {
+            PresignOperation::Write(v) => {
                 self.core
-                    .azblob_put_blob_request(path, None, &OpWrite::default(), Buffer::new())
+                    .azblob_put_blob_request(path, None, v, Buffer::new())
             }
             PresignOperation::Delete(_) => Err(Error::new(
                 ErrorKind::Unsupported,

data/core/services/azblob/src/copier.rs

@@ -21,6 +21,7 @@ use http::StatusCode;
 use uuid::Uuid;
 
 use super::core::AzblobCore;
+use super::core::ErrorContext;
 use super::core::constants::AZBLOB_COPY_MAX_BLOCK_SIZE;
 use super::core::constants::AZBLOB_COPY_MIN_BLOCK_SIZE;
 use super::core::constants::X_MS_VERSION_ID;
@@ -38,8 +39,10 @@ pub fn new_azblob_copier(
     from: &str,
     to: &str,
     args: OpCopy,
-    opts: OpCopier,
 ) -> Result<AzblobCopiers> {
+    let chunk = args.chunk();
+    let source_content_length_hint = args.source_content_length_hint();
+    let concurrent = args.concurrent();
     let copier = AzblobCopier {
         core,
         ctx: ctx.clone(),
@@ -48,9 +51,15 @@ pub fn new_azblob_copier(
         args,
     };
 
-    let Some(chunk) = opts.chunk() else {
+    let Some(chunk) = chunk else {
         return Ok(TwoWays::One(oio::OneShotCopier::new(async move {
-            copier.copy_once().await
+            let source_size = match source_content_length_hint {
+                Some(size) => size,
+                None => copier.source_metadata().await?.content_length(),
+            };
+            let mut metadata = copier.copy_once().await?.into_builder();
+            metadata.set_file(source_size);
+            Ok(metadata.build())
         })));
     };
 
@@ -59,10 +68,10 @@ pub fn new_azblob_copier(
     Ok(TwoWays::Two(oio::BlockCopier::new(
         ctx.executor().clone(),
         copier,
-        opts.source_content_length_hint(),
+        source_content_length_hint,
         block_size.saturating_sub(1),
         block_size,
-        opts.concurrent(),
+        concurrent,
     )))
 }
 
@@ -76,10 +85,11 @@ pub struct AzblobCopier {
 
 impl oio::BlockCopy for AzblobCopier {
     async fn source_metadata(&self) -> Result<Metadata> {
-        let mut args = OpStat::default();
-        if let Some(version) = self.args.source_version() {
-            args = args.with_version(version);
+        let args = options::StatOptions {
+            version: self.args.source_version().map(str::to_owned),
+            ..Default::default()
         }
+        .into();
 
         let resp = self
             .core
@@ -89,13 +99,16 @@ impl oio::BlockCopy for AzblobCopier {
         match resp.status() {
             StatusCode::OK => {
                 let headers = resp.headers();
-                let mut meta = parse_into_metadata(&self.from, headers)?;
+                let mut meta = parse_into_metadata(&self.from, headers)?.into_builder();
                 if let Some(version_id) = parse_header_to_str(headers, X_MS_VERSION_ID)? {
-                    meta.set_version(version_id);
+                    meta.version(version_id);
                 }
-                Ok(meta)
+                Ok(meta.build())
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetBlobProperties")),
+                resp,
+            )),
         }
     }
 
@@ -107,7 +120,12 @@ impl oio::BlockCopy for AzblobCopier {
 
         match resp.status() {
             StatusCode::ACCEPTED => AzblobWriter::parse_metadata(resp.headers()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CopyBlob"))
+                    .with_caller_condition(self.args.is_conditional())
+                    .with_if_not_exists(self.args.if_not_exists()),
+                resp,
+            )),
         }
     }
 
@@ -126,7 +144,10 @@ impl oio::BlockCopy for AzblobCopier {
 
         match resp.status() {
             StatusCode::CREATED | StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("PutBlockFromUrl")),
+                resp,
+            )),
         }
     }
 
@@ -139,7 +160,12 @@ impl oio::BlockCopy for AzblobCopier {
         let meta = AzblobWriter::parse_metadata(resp.headers())?;
         match resp.status() {
             StatusCode::CREATED | StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("PutBlockList"))
+                    .with_caller_condition(self.args.is_conditional())
+                    .with_if_not_exists(self.args.if_not_exists()),
+                resp,
+            )),
         }
     }