opendal 0.1.6.pre.rc.1

data/core/src/services/obs/error.rs

@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// ObsError is the error returned by obs service.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct ObsError {
+    code: String,
+    message: String,
+    resource: String,
+    request_id: String,
+    host_id: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        // OBS could return `520 Origin Error` errors which should be retried.
+        v if v.as_u16() == 520 => (ErrorKind::Unexpected, true),
+
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match de::from_reader::<_, ObsError>(bs.clone().reader()) {
+        Ok(obs_error) => format!("{obs_error:?}"),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+<Code>NoSuchKey</Code>
+<Message>The resource you requested does not exist</Message>
+<Resource>/example-bucket/object</Resource>
+<RequestId>001B21A61C6C0000013402C4616D5285</RequestId>
+<HostId>RkRCRDJENDc5MzdGQkQ4OUY3MTI4NTQ3NDk2Mjg0M0FBQUFBQUFBYmJiYmJiYmJD</HostId>
+</Error>
+"#,
+        );
+
+        let out: ObsError = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code, "NoSuchKey");
+        assert_eq!(out.message, "The resource you requested does not exist");
+        assert_eq!(out.resource, "/example-bucket/object");
+        assert_eq!(out.request_id, "001B21A61C6C0000013402C4616D5285");
+        assert_eq!(
+            out.host_id,
+            "RkRCRDJENDc5MzdGQkQ4OUY3MTI4NTQ3NDk2Mjg0M0FBQUFBQUFBYmJiYmJiYmJD"
+        );
+    }
+}

data/core/src/services/obs/lister.rs

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use quick_xml::de;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Metadata;
+use crate::Result;
+
+pub struct ObsLister {
+    core: Arc<ObsCore>,
+    path: String,
+    delimiter: &'static str,
+    limit: Option<usize>,
+}
+
+impl ObsLister {
+    pub fn new(core: Arc<ObsCore>, path: &str, recursive: bool, limit: Option<usize>) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+
+        Self {
+            core,
+            path: path.to_string(),
+            delimiter,
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for ObsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .obs_list_objects(&self.path, &ctx.token, self.delimiter, self.limit)
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let output: ListObjectsOutput =
+            de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+
+        // Try our best to check whether this list is done.
+        //
+        // - Check `next_marker`
+        ctx.done = match output.next_marker.as_ref() {
+            None => true,
+            Some(next_marker) => next_marker.is_empty(),
+        };
+        ctx.token = output.next_marker.clone().unwrap_or_default();
+
+        let common_prefixes = output.common_prefixes;
+
+        for prefix in common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+
+            ctx.entries.push_back(de);
+        }
+
+        for object in output.contents {
+            let mut path = build_rel_path(&self.core.root, &object.key);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+
+            let meta = Metadata::new(EntryMode::from_path(&path)).with_content_length(object.size);
+
+            let de = oio::Entry::with(path, meta);
+
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/obs/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for obs service.
+#[cfg(feature = "services-obs")]
+pub(super) const DEFAULT_SCHEME: &str = "obs";
+#[cfg(feature = "services-obs")]
+mod core;
+#[cfg(feature = "services-obs")]
+mod delete;
+#[cfg(feature = "services-obs")]
+mod error;
+#[cfg(feature = "services-obs")]
+mod lister;
+#[cfg(feature = "services-obs")]
+mod writer;
+
+#[cfg(feature = "services-obs")]
+mod backend;
+#[cfg(feature = "services-obs")]
+pub use backend::ObsBuilder as Obs;
+
+mod config;
+pub use config::ObsConfig;

data/core/src/services/obs/writer.rs

@@ -0,0 +1,235 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::HeaderMap;
+use http::HeaderValue;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::MultipartPart;
+use crate::raw::*;
+use crate::*;
+
+pub type ObsWriters = TwoWays<oio::MultipartWriter<ObsWriter>, oio::AppendWriter<ObsWriter>>;
+
+pub struct ObsWriter {
+    core: Arc<ObsCore>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl ObsWriter {
+    pub fn new(core: Arc<ObsCore>, path: &str, op: OpWrite) -> Self {
+        ObsWriter {
+            core,
+            path: path.to_string(),
+            op,
+        }
+    }
+
+    fn parse_metadata(headers: &HeaderMap<HeaderValue>) -> Result<Metadata> {
+        let mut meta = Metadata::default();
+        if let Some(etag) = parse_etag(headers)? {
+            meta.set_etag(etag);
+        }
+        if let Some(md5) = parse_content_md5(headers)? {
+            meta.set_content_md5(md5);
+        }
+        if let Some(version) = parse_header_to_str(headers, constants::X_OBS_VERSION_ID)? {
+            meta.set_version(version);
+        }
+
+        Ok(meta)
+    }
+}
+
+impl oio::MultipartWrite for ObsWriter {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .obs_put_object_request(&self.path, Some(size), &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let meta = Self::parse_metadata(resp.headers())?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self
+            .core
+            .obs_initiate_multipart_upload(&self.path, self.op.content_type())
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: InitiateMultipartUploadResult =
+                    quick_xml::de::from_reader(bytes::Buf::reader(bs))
+                        .map_err(new_xml_deserialize_error)?;
+
+                Ok(result.upload_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<MultipartPart> {
+        // Obs service requires part number must between [1..=10000]
+        let part_number = part_number + 1;
+
+        let resp = self
+            .core
+            .obs_upload_part_request(&self.path, upload_id, part_number, Some(size), body)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let etag = parse_etag(resp.headers())?
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "ETag not present in returning response",
+                        )
+                    })?
+                    .to_string();
+
+                Ok(MultipartPart {
+                    part_number,
+                    etag,
+                    checksum: None,
+                })
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_part(&self, upload_id: &str, parts: &[MultipartPart]) -> Result<Metadata> {
+        let parts = parts
+            .iter()
+            .map(|p| CompleteMultipartUploadRequestPart {
+                part_number: p.part_number,
+                etag: p.etag.clone(),
+            })
+            .collect();
+
+        let mut resp = self
+            .core
+            .obs_complete_multipart_upload(&self.path, upload_id, parts)
+            .await?;
+
+        let mut meta = Self::parse_metadata(resp.headers())?;
+
+        let result: CompleteMultipartUploadResult =
+            quick_xml::de::from_reader(resp.body_mut().reader())
+                .map_err(new_xml_deserialize_error)?;
+        meta.set_etag(&result.etag);
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self
+            .core
+            .obs_abort_multipart_upload(&self.path, upload_id)
+            .await?;
+        match resp.status() {
+            // Obs returns code 204 No Content if abort succeeds.
+            // Reference: https://support.huaweicloud.com/intl/en-us/api-obs/obs_04_0103.html
+            StatusCode::NO_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+impl oio::AppendWrite for ObsWriter {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self
+            .core
+            .obs_head_object(&self.path, &OpStat::default())
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let content_length = parse_content_length(resp.headers())?.ok_or_else(|| {
+                    Error::new(
+                        ErrorKind::Unexpected,
+                        "Content-Length not present in returning response",
+                    )
+                })?;
+                Ok(content_length)
+            }
+            StatusCode::NOT_FOUND => Ok(0),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .obs_append_object_request(&self.path, offset, size, &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let mut meta = Metadata::default();
+        if let Some(md5) = parse_content_md5(resp.headers())? {
+            meta.set_content_md5(md5);
+        }
+        if let Some(version) = parse_header_to_str(resp.headers(), constants::X_OBS_VERSION_ID)? {
+            meta.set_version(version);
+        }
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/onedrive/backend.rs

@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+
+use super::core::OneDriveCore;
+use super::delete::OneDriveDeleter;
+use super::error::parse_error;
+use super::lister::OneDriveLister;
+use super::writer::OneDriveWriter;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Clone)]
+pub struct OnedriveBackend {
+    pub core: Arc<OneDriveCore>,
+}
+
+impl Debug for OnedriveBackend {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("OnedriveBackend")
+            .field("core", &self.core)
+            .finish()
+    }
+}
+
+impl Access for OnedriveBackend {
+    type Reader = HttpBody;
+    type Writer = oio::OneShotWriter<OneDriveWriter>;
+    type Lister = oio::PageLister<OneDriveLister>;
+    type Deleter = oio::OneShotDeleter<OneDriveDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+        if path == "/" {
+            // skip, the root path exists in the personal OneDrive.
+            return Ok(RpCreateDir::default());
+        }
+
+        let response = self.core.onedrive_create_dir(path).await?;
+        match response.status() {
+            StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
+            _ => Err(parse_error(response)),
+        }
+    }
+
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        let meta = self.core.onedrive_stat(path, args).await?;
+
+        Ok(RpStat::new(meta))
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let response = self.core.onedrive_get_content(path, &args).await?;
+        match response.status() {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), response.into_body()))
+            }
+            _ => {
+                let (part, mut body) = response.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        Ok((
+            RpWrite::default(),
+            oio::OneShotWriter::new(OneDriveWriter::new(
+                self.core.clone(),
+                args,
+                path.to_string(),
+            )),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(OneDriveDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        let monitor_url = self.core.initialize_copy(from, to).await?;
+        self.core.wait_until_complete(monitor_url).await?;
+        Ok(RpCopy::default())
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        if from == to {
+            return Ok(RpRename::default());
+        }
+
+        self.core.onedrive_move(from, to).await?;
+
+        Ok(RpRename::default())
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = OneDriveLister::new(path.to_string(), self.core.clone(), &args);
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+}