opendal 0.1.9 → 0.1.11

data/core/services/onedrive/src/core.rs

@@ -19,13 +19,13 @@ use std::fmt::Debug;
 use std::sync::Arc;
 use std::time::Duration;
 
+use asyncband::mutex::Mutex;
 use bytes::Buf;
 use bytes::Bytes;
 use http::Request;
 use http::Response;
 use http::StatusCode;
 use http::header;
-use mea::mutex::Mutex;
 
 use opendal_core::raw::*;
 use opendal_core::*;
@@ -132,10 +132,20 @@ impl OneDriveCore {
                         serde_json::from_reader(bytes.reader())
                             .map_err(new_json_deserialize_error)?
                     }
-                    _ => return Err(parse_error(response)),
+                    _ => {
+                        return Err(parse_error(
+                            ErrorContext::new(ServiceOperation("GetItem")),
+                            response,
+                        ));
+                    }
                 }
             }
-            _ => return Err(parse_error(response)),
+            _ => {
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("GetItem")),
+                    response,
+                ));
+            }
         };
 
         Ok(item)
@@ -164,24 +174,14 @@ const MONITOR_WAIT_SECOND: u64 = 1;
 // `services-onedrive` uses the file path based API for simplicity.
 // Read more at https://learn.microsoft.com/en-us/graph/onedrive-addressing-driveitems
 impl OneDriveCore {
-    /// Send a stat request about a particular path, including:
-    ///
-    /// - Get stat object only if ETag not matches
-    /// - whether to get the object version
-    ///
-    /// See also [`onedrive_get_stat_plain()`].
+    /// Send a stat request about a particular path.
     pub(crate) async fn onedrive_stat(
         &self,
         ctx: &OperationContext,
         path: &str,
         args: OpStat,
     ) -> Result<Metadata> {
-        let mut url: String = self.onedrive_item_url(path, true);
-        if args.version().is_some() {
-            url += "?$expand=versions(";
-            url += VERSION_SELECT_PARAM;
-            url += ")";
-        }
+        let url: String = self.onedrive_item_url(path, true);
 
         let mut request = Request::get(&url);
         if let Some(etag) = args.if_none_match() {
@@ -198,7 +198,11 @@ impl OneDriveCore {
 
         let response = ctx.http_transport().send(request).await?;
         if !response.status().is_success() {
-            return Err(parse_error(response));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetItem"))
+                    .with_caller_condition(args.is_conditional()),
+                response,
+            ));
         }
 
         let bytes = response.into_body();
@@ -210,31 +214,18 @@ impl OneDriveCore {
             ItemType::File { .. } => EntryMode::FILE,
         };
 
-        let mut meta = Metadata::new(entry_mode)
-            .with_etag(decoded_response.e_tag)
-            .with_content_length(decoded_response.size.max(0) as u64);
-
-        if let Some(version) = args.version() {
-            for item_version in decoded_response.versions.as_deref().unwrap_or_default() {
-                if item_version.id == version {
-                    meta.set_version(version);
-                    break; // early exit
-                }
-            }
-
-            if meta.version().is_none() {
-                return Err(Error::new(
-                    ErrorKind::NotFound,
-                    "cannot find this version of the item",
-                ));
-            }
-        }
+        let mut meta = if entry_mode == EntryMode::FILE {
+            MetadataBuilder::file(decoded_response.size.max(0) as u64)
+        } else {
+            MetadataBuilder::dir()
+        };
+        meta.etag(decoded_response.e_tag);
 
         let last_modified = decoded_response.last_modified_date_time;
         let date_utc_last_modified = last_modified.parse::<Timestamp>()?;
-        meta.set_last_modified(date_utc_last_modified);
+        meta.last_modified(date_utc_last_modified);
 
-        Ok(meta)
+        Ok(meta.build())
     }
 
     /// Return versions of an item
@@ -263,6 +254,12 @@ impl OneDriveCore {
         self.sign(ctx, &mut request).await?;
 
         let response = ctx.http_transport().send(request).await?;
+        if !response.status().is_success() {
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListVersions")),
+                response,
+            ));
+        }
         let decoded_response: GraphApiOneDriveVersionsResponse =
             serde_json::from_reader(response.into_body().reader())
                 .map_err(new_json_deserialize_error)?;
@@ -453,11 +450,10 @@ impl OneDriveCore {
         path: &str,
         args: &OpWrite,
     ) -> Result<Response<Buffer>> {
-        let parent_path = get_parent(path);
         let file_name = get_basename(path);
         let url = format!(
             "{}:/createUploadSession",
-            self.onedrive_item_url(parent_path, true),
+            self.onedrive_item_url(path, true),
         );
         let mut request = Request::post(url).header(header::CONTENT_TYPE, "application/json");
 
@@ -552,7 +548,10 @@ impl OneDriveCore {
         // we must validate if source exist
         let response = self.onedrive_get_stat_plain(ctx, source).await?;
         if !response.status().is_success() {
-            return Err(parse_error(response));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("GetItem")),
+                response,
+            ));
         }
 
         // We need to stat the destination parent folder to get a parent reference
@@ -580,11 +579,21 @@ impl OneDriveCore {
                 let response = self.onedrive_delete(ctx, destination).await?;
                 match response.status() {
                     StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => {} // expected, intentionally empty
-                    _ => return Err(parse_error(response)),
+                    _ => {
+                        return Err(parse_error(
+                            ErrorContext::new(ServiceOperation("DeleteItem")),
+                            response,
+                        ));
+                    }
                 }
             }
             StatusCode::NOT_FOUND => {} // expected, intentionally empty
-            _ => return Err(parse_error(response)),
+            _ => {
+                return Err(parse_error(
+                    ErrorContext::new(ServiceOperation("GetItem")),
+                    response,
+                ));
+            }
         }
 
         let url: String = format!("{}:/copy", self.onedrive_item_url(source, true));
@@ -610,7 +619,10 @@ impl OneDriveCore {
                     )
                 })
                 .map(String::from),
-            _ => Err(parse_error(response)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CopyItem")),
+                response,
+            )),
         }
     }
 
@@ -692,7 +704,10 @@ impl OneDriveCore {
         match response.status() {
             // can get etag, metadata, etc...
             StatusCode::OK => Ok(()),
-            _ => Err(parse_error(response)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("MoveItem")),
+                response,
+            )),
         }
     }
 }
@@ -753,7 +768,10 @@ impl OneDriveSigner {
                     - Duration::from_secs(120); // assumes 2 mins graceful transmission for implementation simplicity
                 Ok(())
             }
-            _ => Err(parse_error(response)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("RefreshAccessToken")),
+                response,
+            )),
         }
     }
 
@@ -896,13 +914,7 @@ mod tests {
     async fn list_root_returns_entries() {
         let core = test_core("/");
         let ctx = test_ctx();
-        let lister = OneDriveLister::new(
-            "/".to_string(),
-            core,
-            ctx,
-            Capability::default(),
-            &OpList::default(),
-        );
+        let lister = OneDriveLister::new("/".to_string(), core, ctx, &OpList::default());
         let mut lister = oio::PageLister::new(lister);
 
         let mut entries = Vec::new();
@@ -915,53 +927,142 @@ mod tests {
         assert_eq!(entries[1].path(), "test.txt");
         assert_eq!(entries[1].mode(), EntryMode::FILE);
     }
-}
 
-mod error {
-    use http::Response;
-    use http::StatusCode;
+    #[test]
+    fn invalid_request_is_temporary() {
+        let response = Response::builder()
+            .status(StatusCode::BAD_REQUEST)
+            .body(Buffer::from(r#"{"error":{"code":"invalidRequest"}}"#))
+            .unwrap();
+
+        let err = parse_error(
+            ErrorContext::new(ServiceOperation("CreateFolder")),
+            response,
+        );
 
-    use opendal_core::raw::*;
-    use opendal_core::*;
-
-    /// Parse error response into Error.
-    pub(crate) fn parse_error(response: Response<Buffer>) -> Error {
-        let (parts, body) = response.into_parts();
-        let bs = body.to_bytes();
-
-        let (kind, retryable) = match parts.status {
-            StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
-            // The OneDrive service replaces resources.
-            // However, the Onedrive doesn't have Strong Read-After-Write properties,
-            // the concurrent requests to create directories might result in errors.
-            //
-            // Running behavior tests can yield HTTP 409 Conflict because of the consistency guarantee.
-            //
-            // Read more about `REPLACE_EXISTING_ITEM_WHEN_CONFLICT` in `graph_model.rs`.
-            StatusCode::CONFLICT => (ErrorKind::AlreadyExists, true),
-            StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
-            StatusCode::INTERNAL_SERVER_ERROR
-            | StatusCode::BAD_GATEWAY
-            | StatusCode::SERVICE_UNAVAILABLE
-            | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
-            StatusCode::NOT_MODIFIED | StatusCode::PRECONDITION_FAILED => {
-                (ErrorKind::ConditionNotMatch, false)
-            }
-            _ => (ErrorKind::Unexpected, false),
+        assert_eq!(err.kind(), ErrorKind::Unexpected);
+        assert!(err.is_temporary());
+    }
+
+    #[test]
+    fn conflict_classification_uses_native_code() {
+        let parse_conflict = |code: &str, operation| {
+            let body = Buffer::from(format!(r#"{{"error":{{"code":"{code}"}}}}"#));
+            let response = Response::builder()
+                .status(StatusCode::CONFLICT)
+                .body(body)
+                .expect("response must build");
+            parse_error(ErrorContext::new(operation), response)
         };
 
-        let message = String::from_utf8_lossy(&bs);
+        let concurrent = parse_conflict(
+            "Directory_ConcurrencyViolation",
+            ServiceOperation("CreateFolder"),
+        );
+        assert_eq!(concurrent.kind(), ErrorKind::Conflict);
+        assert!(concurrent.is_temporary());
+
+        let name_conflict = parse_conflict("nameAlreadyExists", ServiceOperation("UploadFragment"));
+        assert_eq!(name_conflict.kind(), ErrorKind::Conflict);
+        assert!(!name_conflict.is_temporary());
+
+        assert_eq!(
+            parse_conflict("unknownConflict", ServiceOperation("MoveItem")).kind(),
+            ErrorKind::Unexpected
+        );
+    }
 
-        let mut err = Error::new(kind, message);
+    #[test]
+    fn precondition_failed_requires_caller_condition() {
+        let response = || {
+            Response::builder()
+                .status(StatusCode::PRECONDITION_FAILED)
+                .body(Buffer::new())
+                .expect("response must build")
+        };
+        let caller =
+            ErrorContext::new(ServiceOperation("UploadContent")).with_caller_condition(true);
+        assert_eq!(
+            parse_error(caller, response()).kind(),
+            ErrorKind::ConditionNotMatch
+        );
+        let no_condition = ErrorContext::new(ServiceOperation("UploadFragment"));
+        assert_eq!(
+            parse_error(no_condition, response()).kind(),
+            ErrorKind::Unexpected
+        );
+    }
+}
 
-        err = with_error_response_context(err, parts);
+use crate::graph_model::GraphErrorResponse;
 
-        if retryable {
-            err = err.set_temporary();
+/// Context needed to classify an error from this service.
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct ErrorContext {
+    service_operation: ServiceOperation,
+    caller_condition: bool,
+}
+
+impl ErrorContext {
+    pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
+        Self {
+            service_operation,
+            caller_condition: false,
         }
+    }
 
-        err
+    pub(crate) const fn with_caller_condition(mut self, caller_condition: bool) -> Self {
+        self.caller_condition = caller_condition;
+        self
     }
 }
 
-pub(super) use error::*;
+/// Parse an error response using its service request context.
+pub(crate) fn parse_error(ctx: ErrorContext, response: Response<Buffer>) -> Error {
+    let (parts, body) = response.into_parts();
+    let bs = body.to_bytes();
+
+    let graph_error = serde_json::from_slice::<GraphErrorResponse>(&bs).ok();
+    let graph_code = graph_error
+        .as_ref()
+        .map(|response| response.error.code.as_str());
+    let consistency_lag =
+        parts.status == StatusCode::BAD_REQUEST && graph_code == Some("invalidRequest");
+
+    let (kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::CONFLICT if graph_code == Some("Directory_ConcurrencyViolation") => {
+            (ErrorKind::Conflict, true)
+        }
+        StatusCode::CONFLICT if graph_code == Some("nameAlreadyExists") => {
+            (ErrorKind::Conflict, false)
+        }
+        StatusCode::CONFLICT => (ErrorKind::Unexpected, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        StatusCode::NOT_MODIFIED | StatusCode::PRECONDITION_FAILED if ctx.caller_condition => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    if consistency_lag {
+        retryable = true;
+    }
+
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, message);
+
+    err = err.with_context("service_operation", ctx.service_operation.0);
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/services/onedrive/src/deleter.rs

@@ -22,8 +22,8 @@ use http::StatusCode;
 use opendal_core::raw::*;
 use opendal_core::*;
 
-use super::core::OneDriveCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, OneDriveCore};
 
 /// Delete operation
 /// Documentation: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delete?view=odsp-graph-online
@@ -43,7 +43,10 @@ impl oio::OneShotDelete for OneDriveDeleter {
         let response = self.core.onedrive_delete(&self.ctx, &path).await?;
         match response.status() {
             StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => Ok(()),
-            _ => Err(parse_error(response)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("DeleteItem")),
+                response,
+            )),
         }
     }
 }

data/core/services/onedrive/src/graph_model.rs

@@ -18,6 +18,16 @@
 use serde::Deserialize;
 use serde::Serialize;
 
+#[derive(Debug, Deserialize)]
+pub struct GraphErrorResponse {
+    pub error: GraphError,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct GraphError {
+    pub code: String,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct GraphOAuthRefreshTokenResponseBody {
     pub access_token: String,
@@ -56,7 +66,6 @@ pub struct OneDriveItem {
     pub parent_reference: ParentReference,
     #[serde(flatten)]
     pub item_type: ItemType,
-    pub versions: Option<Vec<OneDriveItemVersion>>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -327,7 +336,6 @@ mod tests {
         assert_eq!(item.e_tag, "\"{3B131E1C-7D81-20AF-80D0-720000000000},2\"");
         assert_eq!(item.size, 3);
         assert_eq!(item.parent_reference.id, "A0AA0A000A000A0A!113");
-        assert!(item.versions.is_none());
         if let ItemType::File { file, .. } = &item.item_type {
             assert_eq!(file.mime_type, "text/plain");
         } else {
@@ -335,55 +343,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_parse_one_drive_file_with_version_json() {
-        let data = r#"{
-            "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('opendal%40outlook.com')/drive/root(id,name,lastModifiedDateTime,eTag,size,parentReference,folder,file)/$entity",
-            "@odata.etag": "\"{3B131E1C-7D81-20AF-80D0-720000000000},2\"",
-            "eTag": "\"{3B131E1C-7D81-20AF-80D0-720000000000},2\"",
-            "id": "A0AA0A000A000A0A!114",
-            "lastModifiedDateTime": "2025-02-16T19:49:05Z",
-            "name": "filename.txt",
-            "size": 3,
-            "parentReference": {
-                "driveType": "personal",
-                "driveId": "A0AA0A000A000A0A",
-                "id": "A0AA0A000A000A0A!113",
-                "name": "folder_a",
-                "path": "/drive/root:/folder_a",
-                "siteId": "5f1f11f3-a6b4-4414-aee0-215c774f80db"
-            },
-            "file": {
-                "mimeType": "text/plain",
-                "hashes": {
-                    "quickXorHash": "79jFLwAAAAAAAAAAAwAAAAAAAAA=",
-                    "sha1Hash": "57218C316B6921E2CD61027A2387EDC31A2D9471",
-                    "sha256Hash": "F1945CD6C19E56B3C1C78943EF5EC18116907A4CA1EFC40A57D48AB1DB7ADFC5"
-                }
-            },
-            "versions@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('opendal%40outlook.com')/drive/root/versions",
-            "versions": [
-                {
-                    "@microsoft.graph.downloadUrl": "https://my.microsoftpersonalcontent.com/personal/A0AA0A000A000A0A/_layouts/15/download.aspx?UniqueId=3b131e1c-7d81-20af-80d0-720000000000&Translate=false&tempauth=v1e.a&ApiVersion=2.0",
-                    "id": "1.0",
-                    "lastModifiedDateTime": "2025-02-16T19:49:05Z",
-                    "size": 3,
-                    "lastModifiedBy": {
-                        "user": {
-                            "email": "erickgdev@outlook.com",
-                            "displayName": "erickgdev@outlook.com"
-                        }
-                    }
-                }
-            ]
-        }"#;
-
-        let item: OneDriveItem = serde_json::from_str(data).unwrap();
-        let versions = item.versions.expect("Versions present");
-        assert_eq!("1.0", versions[0].id);
-        assert_eq!("2025-02-16T19:49:05Z", versions[0].last_modified_date_time);
-    }
-
     #[test]
     fn test_parse_one_drive_monitor_status_json() {
         let data = r#"{

data/core/services/onedrive/src/lister.rs

@@ -23,15 +23,14 @@ use opendal_core::raw::oio;
 use opendal_core::raw::*;
 use opendal_core::*;
 
-use super::core::OneDriveCore;
 use super::core::parse_error;
+use super::core::{ErrorContext, OneDriveCore};
 use super::graph_model::GraphApiOneDriveListResponse;
 use super::graph_model::ItemType;
 
 pub struct OneDriveLister {
     core: Arc<OneDriveCore>,
     ctx: OperationContext,
-    capability: Capability,
     path: String,
     op: OpList,
 }
@@ -43,17 +42,21 @@ impl OneDriveLister {
         path: String,
         core: Arc<OneDriveCore>,
         ctx: OperationContext,
-        capability: Capability,
         args: &OpList,
     ) -> Self {
         Self {
             core,
             ctx,
-            capability,
             path,
             op: args.clone(),
         }
     }
+
+    fn is_after_start(&self, path: &str) -> bool {
+        self.op
+            .start_after()
+            .is_none_or(|start_after| path > start_after)
+    }
 }
 
 impl oio::PageList for OneDriveLister {
@@ -74,14 +77,17 @@ impl oio::PageList for OneDriveLister {
                 ctx.done = true;
                 return Ok(());
             }
-            return Err(parse_error(response));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListChildren")),
+                response,
+            ));
         }
 
         let bytes = response.into_body();
         let decoded_response: GraphApiOneDriveListResponse =
             serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
 
-        let list_with_versions = self.capability.list_with_versions;
+        let list_with_versions = self.op.versions();
 
         // Include the current directory itself when handling the first page of the listing.
         if ctx.token.is_empty() && !ctx.done {
@@ -100,8 +106,10 @@ impl oio::PageList for OneDriveLister {
 
             // skip `list_with_versions` intentionally because a folder doesn't have versions
 
-            let entry = oio::Entry::new(&path, meta);
-            ctx.entries.push_back(entry);
+            if self.is_after_start(&path) {
+                let entry = oio::Entry::new(&path, meta);
+                ctx.entries.push_back(entry);
+            }
         }
 
         if let Some(next_link) = decoded_response.next_link {
@@ -129,25 +137,53 @@ impl oio::PageList for OneDriveLister {
                 normalized_path.push('/');
             }
 
-            let mut meta = Metadata::new(entry_mode)
-                .with_etag(drive_item.e_tag)
-                .with_content_length(drive_item.size.max(0) as u64);
+            if !self.is_after_start(&normalized_path) {
+                continue;
+            }
+
+            let mut meta = if entry_mode == EntryMode::FILE {
+                MetadataBuilder::file(drive_item.size.max(0) as u64)
+            } else {
+                MetadataBuilder::dir()
+            };
+            meta.etag(drive_item.e_tag);
             let last_modified = drive_item.last_modified_date_time.parse::<Timestamp>()?;
-            meta.set_last_modified(last_modified);
-
-            // When listing a directory with `$expand=versions`, OneDrive returns 400 "Operation not supported".
-            // Thus, `list_with_versions` induces N+1 requests. This N+1 is intentional.
-            // N+1 is horrendous but we can't do any better without OneDrive's API support.
-            // When OneDrive supports listing with versions API, remove this.
-            if list_with_versions {
-                let versions = self.core.onedrive_list_versions(&self.ctx, &path).await?;
-                if let Some(version) = versions.first() {
-                    meta.set_version(&version.id);
+            meta.last_modified(last_modified);
+            let meta = meta.build();
+
+            // OneDrive doesn't support expanding versions while listing a directory.
+            // Query each file separately when callers request versions.
+            if list_with_versions && entry_mode == EntryMode::FILE {
+                let versions = self
+                    .core
+                    .onedrive_list_versions(&self.ctx, &normalized_path)
+                    .await?;
+
+                if versions.is_empty() {
+                    ctx.entries
+                        .push_back(oio::Entry::new(&normalized_path, meta));
+                    continue;
                 }
-            }
 
-            let entry = oio::Entry::new(&normalized_path, meta);
-            ctx.entries.push_back(entry)
+                for (index, version) in versions.into_iter().enumerate() {
+                    let mut version_meta = MetadataBuilder::file(version.size.max(0) as u64);
+                    version_meta
+                        .last_modified(version.last_modified_date_time.parse::<Timestamp>()?);
+                    version_meta.version(&version.id);
+                    version_meta.is_current(Some(index == 0));
+
+                    // OneDrive exposes the ETag only for the current item.
+                    if index == 0 {
+                        version_meta.etag(meta.etag().unwrap_or_default());
+                    }
+
+                    ctx.entries
+                        .push_back(oio::Entry::new(&normalized_path, version_meta.build()));
+                }
+            } else {
+                ctx.entries
+                    .push_back(oio::Entry::new(&normalized_path, meta));
+            }
         }
 
         Ok(())