opendal 0.1.9 → 0.1.11

data/core/services/ipmfs/src/writer.rs

@@ -19,8 +19,8 @@ use std::sync::Arc;
 
 use http::StatusCode;
 
-use super::core::IpmfsCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, IpmfsCore};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -43,8 +43,11 @@ impl oio::OneShotWrite for IpmfsWriter {
         let status = resp.status();
 
         match status {
-            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::CREATED | StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Add")),
+                resp,
+            )),
         }
     }
 }

data/core/services/koofr/Cargo.toml

@@ -32,11 +32,11 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
+asyncband = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-mea = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/koofr/src/backend.rs

@@ -18,18 +18,18 @@
 use std::fmt::Debug;
 use std::sync::Arc;
 
+use asyncband::mutex::Mutex;
+use asyncband::once::OnceCell;
 use bytes::Buf;
 use http::StatusCode;
 use log::debug;
-use mea::mutex::Mutex;
-use mea::once::OnceCell;
 
 use super::KOOFR_SCHEME;
 use super::config::KoofrConfig;
 use super::core::File;
-use super::core::KoofrCore;
 use super::core::KoofrSigner;
 use super::core::parse_error;
+use super::core::{ErrorContext, KoofrCore};
 use super::deleter::KoofrDeleter;
 use super::lister::KoofrLister;
 use super::reader::*;
@@ -190,6 +190,7 @@ impl Service for KoofrBackend {
     type Lister = oio::PageLister<KoofrLister>;
     type Deleter = oio::OneShotDeleter<KoofrDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -231,15 +232,21 @@ impl Service for KoofrBackend {
                     EntryMode::FILE
                 };
 
-                let mut md = Metadata::new(mode);
+                let mut md = if mode == EntryMode::FILE {
+                    MetadataBuilder::file(file.size)
+                } else {
+                    MetadataBuilder::dir()
+                };
 
-                md.set_content_length(file.size)
-                    .set_content_type(&file.content_type)
-                    .set_last_modified(Timestamp::from_millisecond(file.modified)?);
+                md.content_type(&file.content_type)
+                    .last_modified(Timestamp::from_millisecond(file.modified)?);
 
-                Ok(RpStat::new(md))
+                Ok(RpStat::new(md.build()))
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesInfo")),
+                resp,
+            )),
         }
     }
     fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -292,33 +299,49 @@ impl Service for KoofrBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _args: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
+        let backend = self.clone();
         let core = self.core.clone();
         let ctx = ctx.clone();
         let from = from.to_string();
         let to = to.to_string();
+        let source_content_length_hint = args.source_content_length_hint();
 
         Ok(oio::OneShotCopier::new(async move {
+            let source_size = match source_content_length_hint {
+                Some(size) => size,
+                None => backend
+                    .stat(&ctx, &from, OpStat::default())
+                    .await?
+                    .into_metadata()
+                    .content_length(),
+            };
+
             core.ensure_dir_exists(&ctx, &to).await?;
             if from == to {
-                Ok(Metadata::default())
+                Ok(MetadataBuilder::file(source_size).build())
             } else {
                 let resp = core.remove(&ctx, &to).await?;
 
                 let status = resp.status();
 
                 if status != StatusCode::OK && status != StatusCode::NOT_FOUND {
-                    Err(parse_error(resp))
+                    Err(parse_error(
+                        ErrorContext::new(ServiceOperation("FilesRemove")),
+                        resp,
+                    ))
                 } else {
                     let resp = core.copy(&ctx, &from, &to).await?;
 
                     let status = resp.status();
 
                     match status {
-                        StatusCode::OK => Ok(Metadata::default()),
-                        _ => Err(parse_error(resp)),
+                        StatusCode::OK => Ok(MetadataBuilder::file(source_size).build()),
+                        _ => Err(parse_error(
+                            ErrorContext::new(ServiceOperation("FilesCopy")),
+                            resp,
+                        )),
                     }
                 }
             }
@@ -343,7 +366,10 @@ impl Service for KoofrBackend {
         let status = resp.status();
 
         if status != StatusCode::OK && status != StatusCode::NOT_FOUND {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesRemove")),
+                resp,
+            ));
         }
 
         let resp = self.core.move_object(ctx, from, to).await?;
@@ -352,7 +378,10 @@ impl Service for KoofrBackend {
 
         match status {
             StatusCode::OK => Ok(RpRename::default()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesMove")),
+                resp,
+            )),
         }
     }
 

data/core/services/koofr/src/core.rs

@@ -19,6 +19,8 @@ use std::collections::VecDeque;
 use std::fmt::Debug;
 use std::sync::Arc;
 
+use asyncband::mutex::Mutex;
+use asyncband::once::OnceCell;
 use bytes::Buf;
 use bytes::Bytes;
 use http::Request;
@@ -26,8 +28,6 @@ use http::Response;
 use http::StatusCode;
 use http::header;
 use http::request;
-use mea::mutex::Mutex;
-use mea::once::OnceCell;
 use serde::Deserialize;
 use serde_json::json;
 
@@ -88,7 +88,10 @@ impl KoofrCore {
                 let status = resp.status();
 
                 if status != StatusCode::OK {
-                    return Err(parse_error(resp));
+                    return Err(parse_error(
+                        ErrorContext::new(ServiceOperation("ListMounts")),
+                        resp,
+                    ));
                 }
 
                 let bs = resp.into_body();
@@ -139,7 +142,10 @@ impl KoofrCore {
         let status = resp.status();
 
         if status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("Authenticate")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();
@@ -219,11 +225,17 @@ impl KoofrCore {
                     // When the directory already exists, Koofr returns 400 Bad Request.
                     // We should treat it as success.
                     StatusCode::OK | StatusCode::CREATED | StatusCode::BAD_REQUEST => Ok(()),
-                    _ => Err(parse_error(resp)),
+                    _ => Err(parse_error(
+                        ErrorContext::new(ServiceOperation("CreateFolder")),
+                        resp,
+                    )),
                 }
             }
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesInfo")),
+                resp,
+            )),
         }
     }
 
@@ -493,62 +505,66 @@ pub struct File {
     pub content_type: String,
 }
 
-mod error {
-    use http::Response;
-
-    use opendal_core::raw::*;
-    use opendal_core::*;
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
 
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
+    }
+}
 
-        let (kind, retryable) = match parts.status.as_u16() {
-            403 => (ErrorKind::PermissionDenied, false),
-            404 => (ErrorKind::NotFound, false),
-            304 | 412 => (ErrorKind::ConditionNotMatch, false),
-            // Service like Koofr could return 499 error with a message like:
-            // Client Disconnect, we should retry it.
-            499 => (ErrorKind::Unexpected, true),
-            500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
 
-        let message = String::from_utf8_lossy(&bs).into_owned();
+    let (kind, retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // Service like Koofr could return 499 error with a message like:
+        // Client Disconnect, we should retry it.
+        499 => (ErrorKind::Unexpected, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
 
-        let mut err = Error::new(kind, message);
+    let message = String::from_utf8_lossy(&bs).into_owned();
 
-        err = with_error_response_context(err, parts);
+    let mut err = Error::new(kind, message);
 
-        if retryable {
-            err = err.set_temporary();
-        }
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
 
-        err
+    if retryable {
+        err = err.set_temporary();
     }
 
-    #[cfg(test)]
-    mod test {
-        use http::StatusCode;
+    err
+}
 
-        use super::*;
+#[cfg(test)]
+mod tests {
+    use http::StatusCode;
 
-        #[tokio::test]
-        async fn test_parse_error() {
-            let err_res = vec![(r#""#, ErrorKind::NotFound, StatusCode::NOT_FOUND)];
+    use super::*;
 
-            for res in err_res {
-                let bs = bytes::Bytes::from(res.0);
-                let body = Buffer::from(bs);
-                let resp = Response::builder().status(res.2).body(body).unwrap();
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![(r#""#, ErrorKind::NotFound, StatusCode::NOT_FOUND)];
 
-                let err = parse_error(resp);
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
 
-                assert_eq!(err.kind(), res.1);
-            }
+            let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
+
+            assert_eq!(err.kind(), res.1);
         }
     }
 }
-
-pub(super) use error::*;

data/core/services/koofr/src/deleter.rs

@@ -45,7 +45,10 @@ impl oio::OneShotDelete for KoofrDeleter {
             StatusCode::OK => Ok(()),
             // Allow 404 when deleting a non-existing object
             StatusCode::NOT_FOUND => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesRemove")),
+                resp,
+            )),
         }
     }
 }

data/core/services/koofr/src/lister.rs

@@ -19,11 +19,10 @@ use std::sync::Arc;
 
 use bytes::Buf;
 
-use super::core::KoofrCore;
 use super::core::ListResponse;
 use super::core::parse_error;
-use opendal_core::EntryMode;
-use opendal_core::Metadata;
+use super::core::{ErrorContext, KoofrCore};
+use opendal_core::MetadataBuilder;
 use opendal_core::OperationContext;
 use opendal_core::Result;
 use opendal_core::raw::oio::Entry;
@@ -58,7 +57,10 @@ impl oio::PageList for KoofrLister {
         match resp.status() {
             http::StatusCode::OK => {}
             _ => {
-                return Err(parse_error(resp));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("FilesList")),
+                    resp,
+                ));
             }
         }
 
@@ -72,13 +74,12 @@ impl oio::PageList for KoofrLister {
 
             let entry = if file.ty == "dir" {
                 let path = format!("{path}/");
-                Entry::new(&path, Metadata::new(EntryMode::DIR))
+                Entry::new(&path, MetadataBuilder::dir().build())
             } else {
-                let m = Metadata::new(EntryMode::FILE)
-                    .with_content_length(file.size)
-                    .with_content_type(file.content_type)
-                    .with_last_modified(Timestamp::from_millisecond(file.modified)?);
-                Entry::new(&path, m)
+                let mut m = MetadataBuilder::file(file.size);
+                m.content_type(file.content_type)
+                    .last_modified(Timestamp::from_millisecond(file.modified)?);
+                Entry::new(&path, m.build())
             };
 
             ctx.entries.push_back(entry);

data/core/services/koofr/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -54,7 +54,10 @@ impl oio::StreamRead for KoofrReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("FilesGet")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/koofr/src/writer.rs

@@ -21,8 +21,8 @@ use bytes::Buf;
 use http::StatusCode;
 
 use super::core::File;
-use super::core::KoofrCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, KoofrCore};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -46,12 +46,15 @@ impl KoofrWriter {
             EntryMode::FILE
         };
 
-        let mut meta = Metadata::new(mode);
-        meta.set_content_length(file.size);
-        meta.set_content_type(&file.content_type);
-        meta.set_last_modified(Timestamp::from_millisecond(file.modified)?);
+        let mut meta = if mode == EntryMode::FILE {
+            MetadataBuilder::file(file.size)
+        } else {
+            MetadataBuilder::dir()
+        };
+        meta.content_type(&file.content_type);
+        meta.last_modified(Timestamp::from_millisecond(file.modified)?);
 
-        Ok(meta)
+        Ok(meta.build())
     }
 }
 
@@ -71,7 +74,10 @@ impl oio::OneShotWrite for KoofrWriter {
                 let metadata = Self::parse_metadata(&file)?;
                 Ok(metadata)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("FilesPut")),
+                resp,
+            )),
         }
     }
 }

data/core/services/lakefs/Cargo.toml

@@ -35,7 +35,7 @@ all-features = true
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 

data/core/services/lakefs/src/backend.rs

@@ -25,9 +25,9 @@ use opendal_core::*;
 
 use super::LAKEFS_SCHEME;
 use super::config::LakefsConfig;
-use super::core::LakefsCore;
 use super::core::LakefsStatus;
 use super::core::parse_error;
+use super::core::{ErrorContext, LakefsCore};
 use super::deleter::LakefsDeleter;
 use super::lister::LakefsLister;
 use super::reader::*;
@@ -191,6 +191,7 @@ impl Service for LakefsBackend {
     type Lister = oio::PageLister<LakefsLister>;
     type Deleter = oio::OneShotDeleter<LakefsDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -215,7 +216,7 @@ impl Service for LakefsBackend {
     async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
         // Stat root always returns a DIR.
         if path == "/" {
-            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+            return Ok(RpStat::new(MetadataBuilder::dir().build()));
         }
 
         let resp = self.core.get_object_metadata(ctx, path).await?;
@@ -230,11 +231,14 @@ impl Service for LakefsBackend {
                     serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
 
                 // Use the helper function to parse LakefsStatus into Metadata
-                let meta = LakefsCore::parse_lakefs_status_into_metadata(&decoded_response);
+                let meta = LakefsCore::parse_lakefs_status_into_metadata(&decoded_response)?;
 
                 Ok(RpStat::new(meta))
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("StatObject")),
+                resp,
+            )),
         }
     }
     fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -296,21 +300,34 @@ impl Service for LakefsBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _args: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
+        let backend = self.clone();
         let core = self.core.clone();
         let ctx = ctx.clone();
         let from = from.to_string();
         let to = to.to_string();
+        let source_content_length_hint = args.source_content_length_hint();
 
         Ok(oio::OneShotCopier::new(async move {
+            let source_size = match source_content_length_hint {
+                Some(size) => size,
+                None => backend
+                    .stat(&ctx, &from, OpStat::default())
+                    .await?
+                    .into_metadata()
+                    .content_length(),
+            };
+
             let resp = core.copy_object(&ctx, &from, &to).await?;
             let status = resp.status();
 
             match status {
-                StatusCode::CREATED => Ok(Metadata::default()),
-                _ => Err(parse_error(resp)),
+                StatusCode::CREATED => Ok(MetadataBuilder::file(source_size).build()),
+                _ => Err(parse_error(
+                    ErrorContext::new(ServiceOperation("CopyObject")),
+                    resp,
+                )),
             }
         }))
     }