opendal 0.1.9 → 0.1.11

data/core/services/vercel-blob/src/backend.rs

@@ -25,9 +25,9 @@ use log::debug;
 use super::VERCEL_BLOB_SCHEME;
 use super::config::VercelBlobConfig;
 use super::core::Blob;
-use super::core::VercelBlobCore;
 use super::core::parse_blob;
 use super::core::parse_error;
+use super::core::{ErrorContext, VercelBlobCore};
 use super::deleter::VercelBlobDeleter;
 use super::lister::VercelBlobLister;
 use super::reader::*;
@@ -138,6 +138,7 @@ impl Service for VercelBlobBackend {
     type Lister = oio::PageLister<VercelBlobLister>;
     type Deleter = oio::OneShotDeleter<VercelBlobDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -173,7 +174,10 @@ impl Service for VercelBlobBackend {
 
                 parse_blob(&resp).map(RpStat::new)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("HeadBlob")),
+                resp,
+            )),
         }
     }
     fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
@@ -208,21 +212,34 @@ impl Service for VercelBlobBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _args: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
+        let backend = self.clone();
         let core = self.core.clone();
         let ctx = ctx.clone();
         let from = from.to_string();
         let to = to.to_string();
+        let source_content_length_hint = args.source_content_length_hint();
 
         Ok(oio::OneShotCopier::new(async move {
+            let source_size = match source_content_length_hint {
+                Some(size) => size,
+                None => backend
+                    .stat(&ctx, &from, OpStat::default())
+                    .await?
+                    .into_metadata()
+                    .content_length(),
+            };
+
             let resp = core.copy(&ctx, &from, &to).await?;
             let status = resp.status();
 
             match status {
-                StatusCode::OK => Ok(Metadata::default()),
-                _ => Err(parse_error(resp)),
+                StatusCode::OK => Ok(MetadataBuilder::file(source_size).build()),
+                _ => Err(parse_error(
+                    ErrorContext::new(ServiceOperation("CopyBlob")),
+                    resp,
+                )),
             }
         }))
     }

data/core/services/vercel-blob/src/core.rs

@@ -96,7 +96,7 @@ impl VercelBlobCore {
         let p = build_abs_path(&self.root, path);
         // Vercel blob use an unguessable random id url to download the file
         // So we use list to get the url of the file and then use it to download the file
-        let resp = self.list(ctx, &p, Some(1)).await?;
+        let resp = self.list(ctx, &p, Some(1), None).await?;
 
         // Use the mtach url to download the file
         let url = resolve_blob(resp.blobs, p);
@@ -161,7 +161,7 @@ impl VercelBlobCore {
     pub async fn head(&self, ctx: &OperationContext, path: &str) -> Result<Response<Buffer>> {
         let p = build_abs_path(&self.root, path);
 
-        let resp = self.list(ctx, &p, Some(1)).await?;
+        let resp = self.list(ctx, &p, Some(1), None).await?;
 
         let url = resolve_blob(resp.blobs, p);
 
@@ -194,7 +194,7 @@ impl VercelBlobCore {
     ) -> Result<Response<Buffer>> {
         let from = build_abs_path(&self.root, from);
 
-        let resp = self.list(ctx, &from, Some(1)).await?;
+        let resp = self.list(ctx, &from, Some(1), None).await?;
 
         let from_url = resolve_blob(resp.blobs, from);
 
@@ -229,6 +229,7 @@ impl VercelBlobCore {
         ctx: &OperationContext,
         prefix: &str,
         limit: Option<usize>,
+        cursor: Option<&str>,
     ) -> Result<ListResponse> {
         let prefix = if prefix == "/" { "" } else { prefix };
 
@@ -241,6 +242,10 @@ impl VercelBlobCore {
             url.push_str(&format!("&limit={limit}"))
         }
 
+        if let Some(cursor) = cursor {
+            url.push_str(&format!("&cursor={}", percent_encode_path(cursor)));
+        }
+
         let req = Request::get(&url);
 
         let req = self.sign(req);
@@ -257,7 +262,10 @@ impl VercelBlobCore {
         let status = resp.status();
 
         if status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListBlobs")),
+                resp,
+            ));
         }
 
         let body = resp.into_body();
@@ -271,7 +279,7 @@ impl VercelBlobCore {
     pub async fn vercel_delete_blob(&self, ctx: &OperationContext, path: &str) -> Result<()> {
         let p = build_abs_path(&self.root, path);
 
-        let resp = self.list(ctx, &p, Some(1)).await?;
+        let resp = self.list(ctx, &p, Some(1), None).await?;
 
         let url = resolve_blob(resp.blobs, p);
 
@@ -301,7 +309,10 @@ impl VercelBlobCore {
 
         match status {
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("DeleteBlob")),
+                resp,
+            )),
         }
     }
 
@@ -411,19 +422,17 @@ impl VercelBlobCore {
 }
 
 pub fn parse_blob(blob: &Blob) -> Result<Metadata> {
-    let mode = if blob.pathname.ends_with('/') {
-        EntryMode::DIR
+    let mut md = if blob.pathname.ends_with('/') {
+        MetadataBuilder::dir()
     } else {
-        EntryMode::FILE
+        MetadataBuilder::file(blob.size)
     };
-    let mut md = Metadata::new(mode);
     if let Some(content_type) = blob.content_type.clone() {
-        md.set_content_type(&content_type);
+        md.content_type(&content_type);
     }
-    md.set_content_length(blob.size);
-    md.set_last_modified(blob.uploaded_at.parse::<Timestamp>()?);
-    md.set_content_disposition(&blob.content_disposition);
-    Ok(md)
+    md.last_modified(blob.uploaded_at.parse::<Timestamp>()?);
+    md.content_disposition(&blob.content_disposition);
+    Ok(md.build())
 }
 
 fn resolve_blob(blobs: Vec<Blob>, path: String) -> String {
@@ -474,101 +483,103 @@ pub struct UploadPartResponse {
     pub etag: String,
 }
 
-mod error {
-    use bytes::Buf;
-    use http::Response;
-    use quick_xml::de;
-    use serde::Deserialize;
+use quick_xml::de;
 
-    use opendal_core::raw::*;
-    use opendal_core::*;
+/// VercelBlobError is the error returned by VercelBlob service.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default)]
+struct VercelBlobError {
+    error: VercelBlobErrorDetail,
+}
 
-    /// VercelBlobError is the error returned by VercelBlob service.
-    #[derive(Default, Debug, Deserialize)]
-    #[serde(default)]
-    struct VercelBlobError {
-        error: VercelBlobErrorDetail,
-    }
+#[derive(Default, Debug, Deserialize)]
+#[serde(default)]
+struct VercelBlobErrorDetail {
+    code: String,
+    message: Option<String>,
+}
 
-    #[derive(Default, Debug, Deserialize)]
-    #[serde(default)]
-    struct VercelBlobErrorDetail {
-        code: String,
-        message: Option<String>,
-    }
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+}
 
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
-        let (parts, body) = resp.into_parts();
-        let bs = body.to_bytes();
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self { service_operation }
+    }
+}
 
-        let (kind, retryable) = match parts.status.as_u16() {
-            403 => (ErrorKind::PermissionDenied, false),
-            404 => (ErrorKind::NotFound, false),
-            500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
-            _ => (ErrorKind::Unexpected, false),
-        };
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
 
-        let (message, _vercel_blob_err) =
-            de::from_reader::<_, VercelBlobError>(bs.clone().reader())
-                .map(|vercel_blob_err| (format!("{vercel_blob_err:?}"), Some(vercel_blob_err)))
-                .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+    let (kind, retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
 
-        let mut err = Error::new(kind, message);
+    let (message, _vercel_blob_err) = de::from_reader::<_, VercelBlobError>(bs.clone().reader())
+        .map(|vercel_blob_err| (format!("{vercel_blob_err:?}"), Some(vercel_blob_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
 
-        err = with_error_response_context(err, parts);
+    let mut err = Error::new(kind, message);
 
-        if retryable {
-            err = err.set_temporary();
-        }
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
 
-        err
+    if retryable {
+        err = err.set_temporary();
     }
 
-    #[cfg(test)]
-    mod test {
-        use http::StatusCode;
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use http::StatusCode;
 
-        use super::*;
+    use super::*;
 
-        #[tokio::test]
-        async fn test_parse_error() {
-            let err_res = vec![(
-                r#"{
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![(
+            r#"{
                     "error": {
                         "code": "forbidden",
                         "message": "Invalid token"
                     }
                 }"#,
-                ErrorKind::PermissionDenied,
-                StatusCode::FORBIDDEN,
-            )];
+            ErrorKind::PermissionDenied,
+            StatusCode::FORBIDDEN,
+        )];
 
-            for res in err_res {
-                let body = Buffer::from(res.0.as_bytes().to_vec());
-                let resp = Response::builder().status(res.2).body(body).unwrap();
+        for res in err_res {
+            let body = Buffer::from(res.0.as_bytes().to_vec());
+            let resp = Response::builder().status(res.2).body(body).unwrap();
 
-                let err = parse_error(resp);
+            let err = parse_error(ErrorContext::new(ServiceOperation("Test")), resp);
 
-                assert_eq!(err.kind(), res.1);
-            }
+            assert_eq!(err.kind(), res.1);
+        }
 
-            let bs = bytes::Bytes::from(
-                r#"{
+        let bs = bytes::Bytes::from(
+            r#"{
                 "error": {
                     "code": "forbidden",
                     "message": "Invalid token"
                 }
             }"#,
-            );
+        );
 
-            let out: VercelBlobError = serde_json::from_reader(bs.reader()).expect("must success");
-            println!("{out:?}");
+        let out: VercelBlobError = serde_json::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
 
-            assert_eq!(out.error.code, "forbidden");
-            assert_eq!(out.error.message, Some("Invalid token".to_string()));
-        }
+        assert_eq!(out.error.code, "forbidden");
+        assert_eq!(out.error.message, Some("Invalid token".to_string()));
     }
 }
-
-pub(super) use error::*;

data/core/services/vercel-blob/src/lister.rs

@@ -52,12 +52,21 @@ impl oio::PageList for VercelBlobLister {
     async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
         let p = build_abs_path(&self.core.root, &self.path);
 
-        let resp = self.core.list(&self.ctx, &p, self.limit).await?;
+        let cursor = if ctx.token.is_empty() {
+            None
+        } else {
+            Some(ctx.token.as_str())
+        };
+
+        let resp = self.core.list(&self.ctx, &p, self.limit, cursor).await?;
 
         ctx.done = !resp.has_more;
 
-        if let Some(cursor) = resp.cursor {
-            ctx.token = cursor;
+        match resp.cursor {
+            Some(cursor) => ctx.token = cursor,
+            // More pages were reported but nothing was handed back to resume from. Stopping
+            // truncates the listing; the alternative is re-requesting the same page forever.
+            None => ctx.done = true,
         }
 
         for blob in resp.blobs {

data/core/services/vercel-blob/src/reader.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use super::backend::*;
-use super::core::parse_error;
+use super::core::{ErrorContext, parse_error};
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -63,7 +63,10 @@ impl oio::StreamRead for VercelBlobReader {
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("DownloadBlob")),
+                    Response::from_parts(part, buf),
+                ));
             }
         };
 

data/core/services/vercel-blob/src/writer.rs

@@ -23,8 +23,8 @@ use http::StatusCode;
 use super::core::InitiateMultipartUploadResponse;
 use super::core::Part;
 use super::core::UploadPartResponse;
-use super::core::VercelBlobCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, VercelBlobCore};
 use opendal_core::raw::*;
 use opendal_core::*;
 
@@ -63,8 +63,11 @@ impl oio::MultipartWrite for VercelBlobWriter {
         let status = resp.status();
 
         match status {
-            StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("PutBlob")),
+                resp,
+            )),
         }
     }
 
@@ -85,7 +88,10 @@ impl oio::MultipartWrite for VercelBlobWriter {
 
                 Ok(resp.upload_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CreateMultipartUpload")),
+                resp,
+            )),
         }
     }
 
@@ -119,7 +125,10 @@ impl oio::MultipartWrite for VercelBlobWriter {
                     size: None,
                 })
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadPart")),
+                resp,
+            )),
         }
     }
 
@@ -144,8 +153,11 @@ impl oio::MultipartWrite for VercelBlobWriter {
         let status = resp.status();
 
         match status {
-            StatusCode::OK => Ok(Metadata::default()),
-            _ => Err(parse_error(resp)),
+            StatusCode::OK => Ok(MetadataBuilder::unknown().build()),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CompleteMultipartUpload")),
+                resp,
+            )),
         }
     }
 

data/core/services/webdav/Cargo.toml

@@ -33,11 +33,11 @@ all-features = true
 
 [dependencies]
 anyhow = { version = "1.0.100", features = ["std"] }
+asyncband = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-mea = { workspace = true }
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
 serde = { workspace = true, features = ["derive"] }
 

data/core/services/webdav/src/backend.rs

@@ -273,6 +273,7 @@ impl Service for WebdavBackend {
     type Lister = oio::PageLister<WebdavLister>;
     type Deleter = oio::OneShotDeleter<WebdavDeleter>;
     type Copier = oio::OneShotCopier;
+    type Composer = ();
 
     fn info(&self) -> ServiceInfo {
         self.core.info.clone()
@@ -351,27 +352,43 @@ impl Service for WebdavBackend {
         ctx: &OperationContext,
         from: &str,
         to: &str,
-        _args: OpCopy,
-        _opts: OpCopier,
+        args: OpCopy,
     ) -> Result<Self::Copier> {
+        let backend = self.clone();
         let core = self.core.clone();
         let ctx = ctx.clone();
         let from = from.to_string();
         let to = to.to_string();
+        let source_content_length_hint = args.source_content_length_hint();
 
         Ok(oio::OneShotCopier::new_with(move || {
+            let backend = backend.clone();
             let core = core.clone();
             let ctx = ctx.clone();
             let from = from.clone();
             let to = to.clone();
 
             async move {
+                let source_size = match source_content_length_hint {
+                    Some(size) => size,
+                    None => backend
+                        .stat(&ctx, &from, OpStat::default())
+                        .await?
+                        .into_metadata()
+                        .content_length(),
+                };
+
                 let resp = core.webdav_copy(&ctx, &from, &to).await?;
                 let status = resp.status();
 
                 match status {
-                    StatusCode::CREATED | StatusCode::NO_CONTENT => Ok(Metadata::default()),
-                    _ => Err(parse_error(resp)),
+                    StatusCode::CREATED | StatusCode::NO_CONTENT => {
+                        Ok(MetadataBuilder::file(source_size).build())
+                    }
+                    _ => Err(parse_error(
+                        ErrorContext::new(ServiceOperation("Copy")),
+                        resp,
+                    )),
                 }
             }
         }))
@@ -391,7 +408,10 @@ impl Service for WebdavBackend {
             StatusCode::CREATED | StatusCode::NO_CONTENT | StatusCode::OK => {
                 Ok(RpRename::default())
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("Move")),
+                resp,
+            )),
         }
     }