opendal 0.1.9 → 0.1.11

data/core/services/azdls/src/writer.rs

@@ -17,13 +17,13 @@
 
 use std::sync::Arc;
 
+use asyncband::once::OnceCell;
 use http::StatusCode;
-use mea::once::OnceCell;
 
-use super::core::AzdlsCore;
 use super::core::FILE;
 use super::core::X_MS_VERSION_ID;
 use super::core::parse_error;
+use super::core::{AzdlsCore, ErrorContext};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -77,30 +77,39 @@ impl AzdlsWriter {
             .await?;
         match resp.status() {
             StatusCode::CREATED | StatusCode::OK => Ok(()),
-            StatusCode::CONFLICT if self.op.if_not_exists() => {
-                Err(parse_error(resp).with_operation("Backend::azdls_create_request"))
+            _ => {
+                let err = parse_error(
+                    ErrorContext::new(ServiceOperation("CreateFile"))
+                        .with_caller_condition(self.op.is_conditional())
+                        .with_if_not_exists(self.op.if_not_exists()),
+                    resp,
+                )
+                .with_operation("Backend::azdls_create_request");
+                if err.kind() == ErrorKind::AlreadyExists {
+                    Ok(())
+                } else {
+                    Err(err)
+                }
             }
-            StatusCode::CONFLICT => Ok(()),
-            _ => Err(parse_error(resp).with_operation("Backend::azdls_create_request")),
         }
     }
 
     fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
-        let mut metadata = Metadata::default();
+        let mut metadata = MetadataBuilder::unknown();
 
         if let Some(last_modified) = parse_last_modified(headers)? {
-            metadata.set_last_modified(last_modified);
+            metadata.last_modified(last_modified);
         }
         let etag = parse_etag(headers)?;
         if let Some(etag) = etag {
-            metadata.set_etag(etag);
+            metadata.etag(etag);
         }
         let version_id = parse_header_to_str(headers, X_MS_VERSION_ID)?;
         if let Some(version_id) = version_id {
-            metadata.set_version(version_id);
+            metadata.version(version_id);
         }
 
-        Ok(metadata)
+        Ok(metadata.build())
     }
 }
 
@@ -114,7 +123,10 @@ impl oio::PositionWrite for AzdlsWriter {
 
         match resp.status() {
             StatusCode::OK | StatusCode::ACCEPTED => Ok(()),
-            _ => Err(parse_error(resp).with_operation("Backend::azdls_append_request")),
+            _ => Err(
+                parse_error(ErrorContext::new(ServiceOperation("AppendData")), resp)
+                    .with_operation("Backend::azdls_append_request"),
+            ),
         }
     }
 
@@ -125,12 +137,15 @@ impl oio::PositionWrite for AzdlsWriter {
             .azdls_flush(&self.ctx, &self.path, size, true)
             .await?;
 
-        let mut meta = AzdlsWriter::parse_metadata(resp.headers())?;
-        meta.set_content_length(size);
+        let mut meta = AzdlsWriter::parse_metadata(resp.headers())?.into_builder();
+        meta.set_file(size);
 
         match resp.status() {
-            StatusCode::OK | StatusCode::ACCEPTED => Ok(meta),
-            _ => Err(parse_error(resp).with_operation("Backend::azdls_flush_request")),
+            StatusCode::OK | StatusCode::ACCEPTED => Ok(meta.build()),
+            _ => Err(
+                parse_error(ErrorContext::new(ServiceOperation("FlushData")), resp)
+                    .with_operation("Backend::azdls_flush_request"),
+            ),
         }
     }
 
@@ -162,7 +177,7 @@ impl oio::AppendWrite for AzdlsWriter {
     async fn offset(&self) -> Result<u64> {
         let resp = self
             .core
-            .azdls_get_properties(&self.ctx, &self.path)
+            .azdls_get_properties(&self.ctx, &self.path, &OpStat::default())
             .await?;
 
         let status = resp.status();
@@ -171,7 +186,10 @@ impl oio::AppendWrite for AzdlsWriter {
         match status {
             StatusCode::OK => Ok(parse_content_length(headers)?.unwrap_or_default()),
             StatusCode::NOT_FOUND => Ok(0),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetPathProperties")),
+                resp,
+            )),
         }
     }
 
@@ -187,16 +205,19 @@ impl oio::AppendWrite for AzdlsWriter {
             .azdls_append(&self.ctx, &self.path, Some(size), offset, true, false, body)
             .await?;
 
-        let mut meta = AzdlsWriter::parse_metadata(resp.headers())?;
+        let mut meta = AzdlsWriter::parse_metadata(resp.headers())?.into_builder();
         let md5 = parse_content_md5(resp.headers())?;
         if let Some(md5) = md5 {
-            meta.set_content_md5(md5);
+            meta.content_md5(md5);
         }
-        meta.set_content_length(offset + size);
+        meta.set_file(offset + size);
 
         match resp.status() {
-            StatusCode::OK | StatusCode::ACCEPTED => Ok(meta),
-            _ => Err(parse_error(resp).with_operation("Backend::azdls_append_request")),
+            StatusCode::OK | StatusCode::ACCEPTED => Ok(meta.build()),
+            _ => Err(
+                parse_error(ErrorContext::new(ServiceOperation("AppendData")), resp)
+                    .with_operation("Backend::azdls_append_request"),
+            ),
         }
     }
 }

data/core/services/azfile/Cargo.toml

@@ -35,10 +35,10 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false, features = [
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false, features = [
   "reqsign",
 ] }
-opendal-service-azure-common = { path = "../azure-common", version = "0.58.1" }
+opendal-service-azure-common = { path = "../azure-common", version = "0.59.0" }
 quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
 reqsign-azure-storage = { version = "3.1.1", default-features = false }
 reqsign-core = { version = "3.2.0", default-features = false }

data/core/services/azfile/src/backend.rs

@@ -32,9 +32,9 @@ use reqsign_file_read_tokio::TokioFileRead;
 
 use super::AZFILE_SCHEME;
 use super::config::AzfileConfig;
-use super::core::AzfileCore;
 use super::core::X_MS_META_PREFIX;
 use super::core::parse_error;
+use super::core::{AzfileCore, ErrorContext};
 use super::deleter::AzfileDeleter;
 use super::lister::AzfileLister;
 use super::reader::*;
@@ -237,6 +237,9 @@ impl Builder for AzfileBuilder {
 
             write: true,
             write_with_user_metadata: true,
+            // Azure Files limits each Put Range update to 4 MiB.
+            // ref: <https://learn.microsoft.com/en-us/rest/api/storageservices/put-range>
+            write_total_max_size: Some(4 * 1024 * 1024),
 
             create_dir: true,
             delete: true,
@@ -275,6 +278,7 @@ impl Service for AzfileBackend {
     type Lister = oio::PageLister<AzfileLister>;
     type Deleter = oio::OneShotDeleter<AzfileDeleter>;
     type Copier = ();
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -311,7 +315,10 @@ impl Service for AzfileBackend {
                 {
                     Ok(RpCreateDir::default())
                 } else {
-                    Err(parse_error(resp))
+                    Err(parse_error(
+                        ErrorContext::new(ServiceOperation("CreateDirectory")),
+                        resp,
+                    ))
                 }
             }
         }
@@ -328,14 +335,21 @@ impl Service for AzfileBackend {
         match status {
             StatusCode::OK => {
                 let headers = resp.headers();
-                let mut meta = parse_into_metadata(path, headers)?;
+                let mut meta = parse_into_metadata(path, headers)?.into_builder();
                 let user_meta = parse_prefixed_headers(headers, X_MS_META_PREFIX);
                 if !user_meta.is_empty() {
-                    meta = meta.with_user_metadata(user_meta);
+                    meta.user_metadata(user_meta);
                 }
-                Ok(RpStat::new(meta))
+                Ok(RpStat::new(meta.build()))
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(if path.ends_with('/') {
+                    ServiceOperation("GetDirectoryProperties")
+                } else {
+                    ServiceOperation("GetFileProperties")
+                }),
+                resp,
+            )),
         }
     }
     fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -402,7 +416,6 @@ impl Service for AzfileBackend {
         _from: &str,
         _to: &str,
         _args: OpCopy,
-        _opts: OpCopier,
     ) -> Result<Self::Copier> {
         Err(Error::new(
             ErrorKind::Unsupported,
@@ -422,7 +435,10 @@ impl Service for AzfileBackend {
         let status = resp.status();
         match status {
             StatusCode::OK => Ok(RpRename::default()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Rename")),
+                resp,
+            )),
         }
     }
 

data/core/services/azfile/src/core.rs

@@ -433,7 +433,7 @@ impl AzfileCore {
             .push("include", "Timestamps,ETag");
 
         if !continuation.is_empty() {
-            url = url.push("marker", continuation);
+            url = url.push("marker", &percent_encode_path(continuation));
         }
 
         if let Some(limit) = limit {
@@ -487,103 +487,221 @@ impl AzfileCore {
                 continue;
             }
 
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("CreateDirectory")),
+                resp,
+            ));
         }
 
         Ok(())
     }
 }
 
-mod error {
-    use std::fmt::Debug;
-
-    use bytes::Buf;
-    use http::Response;
-    use http::StatusCode;
-    use opendal_core::*;
-    use opendal_service_azure_common::with_azure_error_response_context;
-    use quick_xml::de;
-    use serde::Deserialize;
-
-    /// AzfileError is the error returned by azure file service.
-    #[derive(Default, Deserialize)]
-    #[serde(default, rename_all = "PascalCase")]
-    struct AzfileError {
-        code: String,
-        message: String,
-        query_parameter_name: String,
-        query_parameter_value: String,
-        reason: String,
+use bytes::Buf;
+use opendal_core::raw::ServiceOperation;
+use opendal_service_azure_common::with_azure_error_response_context;
+use quick_xml::de;
+use serde::Deserialize;
+
+/// AzfileError is the error returned by azure file service.
+#[derive(Default, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct AzfileError {
+    code: String,
+    message: String,
+    query_parameter_name: String,
+    query_parameter_value: String,
+    reason: String,
+}
+
+impl Debug for AzfileError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("AzfileError");
+        de.field("code", &self.code);
+        // replace `\n` to ` ` for better reading.
+        de.field("message", &self.message.replace('\n', " "));
+
+        if !self.query_parameter_name.is_empty() {
+            de.field("query_parameter_name", &self.query_parameter_name);
+        }
+        if !self.query_parameter_value.is_empty() {
+            de.field("query_parameter_value", &self.query_parameter_value);
+        }
+        if !self.reason.is_empty() {
+            de.field("reason", &self.reason);
+        }
+
+        de.finish()
+    }
+}
+
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
+
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
     }
+}
 
-    impl Debug for AzfileError {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-            let mut de = f.debug_struct("AzfileError");
-            de.field("code", &self.code);
-            // replace `\n` to ` ` for better reading.
-            de.field("message", &self.message.replace('\n', " "));
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let mut azfile_error = de::from_reader::<_, AzfileError>(bs.clone().reader()).ok();
+    if azfile_error.as_ref().is_none_or(|err| err.code.is_empty())
+        && let Some(code) = parts
+            .headers
+            .get("x-ms-error-code")
+            .and_then(|value| value.to_str().ok())
+    {
+        azfile_error.get_or_insert_with(AzfileError::default).code = code.to_string();
+    }
 
-            if !self.query_parameter_name.is_empty() {
-                de.field("query_parameter_name", &self.query_parameter_name);
+    let (mut kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    if let Some(azfile_error) = &azfile_error {
+        match azfile_error.code.as_str() {
+            "ParentNotFound" => {
+                (kind, retryable) = (ErrorKind::NotFound, false);
             }
-            if !self.query_parameter_value.is_empty() {
-                de.field("query_parameter_value", &self.query_parameter_value);
+            "ResourceAlreadyExists" => {
+                (kind, retryable) = (ErrorKind::AlreadyExists, false);
             }
-            if !self.reason.is_empty() {
-                de.field("reason", &self.reason);
+            "DeletePending" | "ShareBeingDeleted"
+                if ctx.service_operation == ServiceOperation("CreateDirectory") =>
+            {
+                (kind, retryable) = (ErrorKind::Conflict, true);
             }
-
-            de.finish()
+            "CannotDeleteFileOrDirectory"
+            | "ContainerQuotaDowngradeNotAllowed"
+            | "DeletePending"
+            | "DeleteShareWhenSnapshotLeased"
+            | "DirectoryNotEmpty"
+            | "FileGenerationMismatch"
+            | "FileLockConflict"
+            | "FileOpenBySmbClient"
+            | "LeaseAcquireDuringShareDelete"
+            | "LeaseIdMismatchWithFileLeaseOperation"
+            | "LeaseIdMismatchWithFileOperation"
+            | "LeaseIdMismatchWithFileShareLeaseOperation"
+            | "LeaseIdMismatchWithFileShareOperation"
+            | "LeaseIdMissingWithFileOperation"
+            | "LeaseIdMissingWithFileShareOperation"
+            | "LeaseLostWithFileOperation"
+            | "LeaseLostWithFileShareOperation"
+            | "LeaseNotPresentWithFileLeaseOperation"
+            | "LeaseNotPresentWithFileOperation"
+            | "LeaseNotPresentWithFileShareLeaseOperation"
+            | "LeaseNotPresentWithFileShareOperation"
+            | "PreviousSnapshotNotFound"
+            | "PreviousSnapshotOperationNotSupported"
+            | "ReadOnlyAttribute"
+            | "RenameCannotOverwriteDestinationFile"
+            | "RenameCycle"
+            | "RenameDirectoryHasOpenFiles"
+            | "ShareAlreadyExists"
+            | "ShareBeingDeleted"
+            | "ShareHasSnapshots"
+            | "ShareSnapshotInProgress"
+            | "SharingViolation" => {
+                (kind, retryable) = (ErrorKind::Conflict, false);
+            }
+            _ if matches!(
+                parts.status,
+                StatusCode::CONFLICT | StatusCode::PRECONDITION_FAILED
+            ) =>
+            {
+                (kind, retryable) = (ErrorKind::Unexpected, false);
+            }
+            _ => {}
         }
     }
 
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
+    let message = azfile_error
+        .map(|err| format!("{err:?}"))
+        .unwrap_or_else(|| String::from_utf8_lossy(&bs).into_owned());
 
-        let (kind, retryable) = match parts.status {
-            StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
-            StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
-            StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => {
-                (ErrorKind::ConditionNotMatch, false)
-            }
-            StatusCode::INTERNAL_SERVER_ERROR
-            | StatusCode::BAD_GATEWAY
-            | StatusCode::SERVICE_UNAVAILABLE
-            | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
+    let mut err = Error::new(kind, &message);
 
-        let mut message = match de::from_reader::<_, AzfileError>(bs.clone().reader()) {
-            Ok(azfile_err) => format!("{azfile_err:?}"),
-            Err(_) => String::from_utf8_lossy(&bs).into_owned(),
-        };
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_azure_error_response_context(err, parts);
 
-        // If there is no body here, fill with error code.
-        if message.is_empty()
-            && let Some(v) = parts.headers.get("x-ms-error-code")
-            && let Ok(code) = v.to_str()
-        {
-            message = format!(
-                "{:?}",
-                AzfileError {
-                    code: code.to_string(),
-                    ..Default::default()
-                }
-            )
-        }
+    if retryable {
+        err = err.set_temporary();
+    }
 
-        let mut err = Error::new(kind, &message);
+    err
+}
 
-        err = with_azure_error_response_context(err, parts);
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parse_native_error(operation: &'static str, status: StatusCode, code: &str) -> Error {
+        let resp = Response::builder()
+            .status(status)
+            .header("x-ms-error-code", code)
+            .body(Buffer::new())
+            .expect("response must build");
+        parse_error(ErrorContext::new(ServiceOperation(operation)), resp)
+    }
 
-        if retryable {
-            err = err.set_temporary();
-        }
+    #[test]
+    fn test_parse_native_conflict_codes() {
+        let err = parse_native_error("CreateDirectory", StatusCode::CONFLICT, "DeletePending");
+        assert_eq!(err.kind(), ErrorKind::Conflict);
+        assert!(err.is_temporary());
+
+        let err = parse_native_error("PutRange", StatusCode::CONFLICT, "FileLockConflict");
+        assert_eq!(err.kind(), ErrorKind::Conflict);
+        assert!(!err.is_temporary());
 
-        err
+        let err = parse_native_error(
+            "GetFileProperties",
+            StatusCode::PRECONDITION_FAILED,
+            "LeaseIdMismatchWithFileOperation",
+        );
+        assert_eq!(err.kind(), ErrorKind::Conflict);
+        assert!(!err.is_temporary());
     }
-}
 
-pub(super) use error::*;
+    #[test]
+    fn test_parse_non_conflict_native_codes() {
+        assert_eq!(
+            parse_native_error(
+                "CreateDirectory",
+                StatusCode::CONFLICT,
+                "ResourceAlreadyExists"
+            )
+            .kind(),
+            ErrorKind::AlreadyExists
+        );
+        assert_eq!(
+            parse_native_error("CreateDirectory", StatusCode::NOT_FOUND, "ParentNotFound").kind(),
+            ErrorKind::NotFound
+        );
+        assert_eq!(
+            parse_native_error(
+                "GetFileProperties",
+                StatusCode::PRECONDITION_FAILED,
+                "UnknownPrecondition"
+            )
+            .kind(),
+            ErrorKind::Unexpected
+        );
+    }
+}

data/core/services/azfile/src/deleter.rs

@@ -46,7 +46,14 @@ impl oio::OneShotDelete for AzfileDeleter {
         let status = resp.status();
         match status {
             StatusCode::ACCEPTED | StatusCode::NOT_FOUND => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(if path.ends_with('/') {
+                    ServiceOperation("DeleteDirectory")
+                } else {
+                    ServiceOperation("DeleteFile")
+                }),
+                resp,
+            )),
         }
     }
 }

data/core/services/azfile/src/lister.rs

@@ -22,8 +22,8 @@ use http::StatusCode;
 use quick_xml::de;
 use serde::Deserialize;
 
-use super::core::AzfileCore;
 use super::core::parse_error;
+use super::core::{AzfileCore, ErrorContext};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -64,12 +64,15 @@ impl oio::PageList for AzfileLister {
                 ctx.done = true;
                 return Ok(());
             }
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListDirectoriesAndFiles")),
+                resp,
+            ));
         }
 
         // Return self at the first page.
         if ctx.token.is_empty() && !ctx.done {
-            let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+            let e = oio::Entry::new(&self.path, MetadataBuilder::dir().build());
             ctx.entries.push_back(e);
         }
 
@@ -85,22 +88,25 @@ impl oio::PageList for AzfileLister {
         }
 
         for file in results.entries.file {
-            let mut meta = Metadata::new(EntryMode::FILE)
-                .with_etag(file.properties.etag)
-                .with_last_modified(Timestamp::parse_rfc2822(&file.properties.last_modified)?);
-            if let Some(size) = file.properties.content_length {
-                meta.set_content_length(size);
-            }
+            let content_length = file.properties.content_length.ok_or_else(|| {
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "azfile list response does not contain file content length",
+                )
+            })?;
+            let mut meta = MetadataBuilder::file(content_length);
+            meta.etag(file.properties.etag)
+                .last_modified(Timestamp::parse_rfc2822(&file.properties.last_modified)?);
             let path = self.path.clone().trim_start_matches('/').to_string() + &file.name;
-            ctx.entries.push_back(oio::Entry::new(&path, meta));
+            ctx.entries.push_back(oio::Entry::new(&path, meta.build()));
         }
 
         for dir in results.entries.directory {
-            let meta = Metadata::new(EntryMode::DIR)
-                .with_etag(dir.properties.etag)
-                .with_last_modified(Timestamp::parse_rfc2822(&dir.properties.last_modified)?);
+            let mut meta = MetadataBuilder::dir();
+            meta.etag(dir.properties.etag)
+                .last_modified(Timestamp::parse_rfc2822(&dir.properties.last_modified)?);
             let path = self.path.clone().trim_start_matches('/').to_string() + &dir.name + "/";
-            ctx.entries.push_back(oio::Entry::new(&path, meta));
+            ctx.entries.push_back(oio::Entry::new(&path, meta.build()));
         }
 
         Ok(())

data/core/services/azfile/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -59,7 +59,10 @@ impl oio::StreamRead for AzfileReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("GetFile")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };