opendal 0.1.6.pre.rc.1

data/core/src/services/oss/delete.rs

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::*;
+
+pub struct OssDeleter {
+    core: Arc<OssCore>,
+}
+
+impl OssDeleter {
+    pub fn new(core: Arc<OssCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::BatchDelete for OssDeleter {
+    async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
+        let resp = self.core.oss_delete_object(&path, &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT | StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+        // Sadly, OSS will not return failed keys, so we will build
+        // a set to calculate the failed keys.
+        let mut keys: HashSet<(String, OpDelete)> = batch
+            .iter()
+            .map(|path| (path.0.to_owned(), path.1.clone()))
+            .collect();
+
+        let resp = self.core.oss_delete_objects(batch).await?;
+
+        let status = resp.status();
+
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let result: DeleteObjectsResult =
+            quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+
+        if result.deleted.is_empty() {
+            return Err(Error::new(
+                ErrorKind::Unexpected,
+                "oss delete this key failed for reason we don't know",
+            ));
+        }
+
+        let mut batched_result = BatchDeleteResult {
+            succeeded: Vec::with_capacity(result.deleted.len()),
+            failed: Vec::with_capacity(keys.len() - result.deleted.len()),
+        };
+
+        for i in result.deleted {
+            let path = build_rel_path(&self.core.root, &i.key);
+            let mut op = OpDelete::default();
+            if let Some(version) = &i.version_id {
+                op = op.with_version(version);
+            }
+            let object = (path, op);
+            keys.remove(&object);
+            batched_result.succeeded.push(object);
+        }
+        // TODO: we should handle those errors with code.
+        for (path, op) in keys {
+            batched_result.failed.push((
+                path,
+                op,
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "oss delete this key failed for reason we don't know",
+                ),
+            ));
+        }
+
+        Ok(batched_result)
+    }
+}

data/core/src/services/oss/docs.md

@@ -0,0 +1,74 @@
+
+
+# Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] append
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [x] presign
+- [ ] blocking
+
+# Configuration
+
+- `root`: Set the work dir for backend.
+- `bucket`: Set the container name for backend.
+- `endpoint`: Set the endpoint for backend.
+- `addressing_style`: Set the addressing style for endpoint.
+- `presign_endpoint`: Set the endpoint for presign.
+- `presign_addressing_style`: Set the addressing style for presign endpoint.
+- `access_key_id`: Set the access_key_id for backend.
+- `access_key_secret`: Set the access_key_secret for backend.
+- `role_arn`: Set the role of backend.
+- `oidc_token`: Set the oidc_token for backend.
+- `allow_anonymous`: Set the backend access OSS in anonymous way.
+
+Refer to [`OssBuilder`]'s public API docs for more information.
+
+# Example
+
+## Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Oss;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create OSS backend builder.
+    let mut builder = Oss::default()
+        // Set the root for oss, all operations will happen under this root.
+        //
+        // NOTE: the root must be absolute path.
+        .root("/path/to/dir")
+        // Set the bucket name, this is required.
+        .bucket("test")
+        // Set the endpoint.
+        //
+        // For example:
+        // - "https://oss-ap-northeast-1.aliyuncs.com"
+        // - "https://oss-hangzhou.aliyuncs.com"
+        .endpoint("https://oss-cn-beijing.aliyuncs.com")
+        // Set the access_key_id and access_key_secret.
+        //
+        // OpenDAL will try load credential from the env.
+        // If credential not set and no valid credential in env, OpenDAL will
+        // send request without signing like anonymous user.
+        .access_key_id("access_key_id")
+        .access_key_secret("access_key_secret");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/oss/error.rs

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// OssError is the error returned by oss service.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct OssError {
+    code: String,
+    message: String,
+    request_id: String,
+    host_id: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED | StatusCode::CONFLICT => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match de::from_reader::<_, OssError>(bs.clone().reader()) {
+        Ok(oss_err) => format!("{oss_err:?}"),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Error response example is from https://www.alibabacloud.com/help/en/object-storage-service/latest/error-responses
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" ?>
+<Error xmlns="http://doc.oss-cn-hangzhou.aliyuncs.com">
+    <Code>
+        AccessDenied
+    </Code>
+    <Message>
+        Query-string authentication requires the Signature, Expires and OSSAccessKeyId parameters
+    </Message>
+    <RequestId>
+        1D842BC54255****
+    </RequestId>
+    <HostId>
+        oss-cn-hangzhou.aliyuncs.com
+    </HostId>
+</Error>
+"#,
+        );
+
+        let out: OssError = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code.trim(), "AccessDenied");
+        assert_eq!(
+            out.message.trim(),
+            "Query-string authentication requires the Signature, Expires and OSSAccessKeyId parameters"
+        );
+        assert_eq!(out.request_id.trim(), "1D842BC54255****");
+        assert_eq!(out.host_id.trim(), "oss-cn-hangzhou.aliyuncs.com");
+    }
+}

data/core/src/services/oss/lister.rs

@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use quick_xml::de;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::PageContext;
+use crate::raw::*;
+use crate::*;
+
+pub type OssListers = TwoWays<oio::PageLister<OssLister>, oio::PageLister<OssObjectVersionsLister>>;
+
+pub struct OssLister {
+    core: Arc<OssCore>,
+
+    path: String,
+    delimiter: &'static str,
+    limit: Option<usize>,
+    /// Filter results to objects whose names are lexicographically
+    /// **equal to or after** startOffset
+    start_after: Option<String>,
+}
+
+impl OssLister {
+    pub fn new(
+        core: Arc<OssCore>,
+        path: &str,
+        recursive: bool,
+        limit: Option<usize>,
+        start_after: Option<&str>,
+    ) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+        Self {
+            core,
+            path: path.to_string(),
+            delimiter,
+            limit,
+            start_after: start_after.map(String::from),
+        }
+    }
+}
+
+impl oio::PageList for OssLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .oss_list_object(
+                &self.path,
+                &ctx.token,
+                self.delimiter,
+                self.limit,
+                if ctx.token.is_empty() {
+                    self.start_after.clone()
+                } else {
+                    None
+                },
+            )
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let output: ListObjectsOutput = de::from_reader(bs.reader())
+            .map_err(|e| Error::new(ErrorKind::Unexpected, "deserialize xml").set_source(e))?;
+
+        ctx.done = !output.is_truncated;
+        ctx.token = output.next_continuation_token.unwrap_or_default();
+
+        for prefix in output.common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+            ctx.entries.push_back(de);
+        }
+
+        for object in output.contents {
+            let mut path = build_rel_path(&self.core.root, &object.key);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+            if self.start_after.as_ref() == Some(&path) {
+                continue;
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+            meta.set_is_current(true);
+            meta.set_etag(&object.etag);
+            meta.set_content_md5(object.etag.trim_matches('"'));
+            meta.set_content_length(object.size);
+            meta.set_last_modified(parse_datetime_from_rfc3339(object.last_modified.as_str())?);
+
+            let de = oio::Entry::with(path, meta);
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}
+
+/// refer: https://help.aliyun.com/zh/oss/developer-reference/listobjectversions?spm=a2c4g.11186623.help-menu-31815.d_3_1_1_5_5_2.53f67237GJlMPw&scm=20140722.H_112467._.OR_help-T_cn~zh-V_1
+pub struct OssObjectVersionsLister {
+    core: Arc<OssCore>,
+
+    prefix: String,
+    args: OpList,
+
+    delimiter: &'static str,
+    abs_start_after: Option<String>,
+}
+
+impl OssObjectVersionsLister {
+    pub fn new(core: Arc<OssCore>, path: &str, args: OpList) -> Self {
+        let delimiter = if args.recursive() { "" } else { "/" };
+        let abs_start_after = args
+            .start_after()
+            .map(|start_after| build_abs_path(&core.root, start_after));
+
+        Self {
+            core,
+            prefix: path.to_string(),
+            args,
+            delimiter,
+            abs_start_after,
+        }
+    }
+}
+
+impl oio::PageList for OssObjectVersionsLister {
+    async fn next_page(&self, ctx: &mut PageContext) -> Result<()> {
+        let markers = ctx.token.rsplit_once(" ");
+        let (key_marker, version_id_marker) = if let Some(data) = markers {
+            data
+        } else if let Some(start_after) = &self.abs_start_after {
+            (start_after.as_str(), "")
+        } else {
+            ("", "")
+        };
+
+        let resp = self
+            .core
+            .oss_list_object_versions(
+                &self.prefix,
+                self.delimiter,
+                self.args.limit(),
+                key_marker,
+                version_id_marker,
+            )
+            .await?;
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let body = resp.into_body();
+        let output: ListObjectVersionsOutput = de::from_reader(body.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow Cos list to retry on XML deserialization errors.
+            //
+            // This is because the Cos list API may return incomplete XML data under high load.
+            // We are confident that our XML decoding logic is correct. When this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
+
+        ctx.done = if let Some(is_truncated) = output.is_truncated {
+            !is_truncated
+        } else {
+            false
+        };
+        ctx.token = format!(
+            "{} {}",
+            output.next_key_marker.unwrap_or_default(),
+            output.next_version_id_marker.unwrap_or_default()
+        );
+
+        for prefix in output.common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+            ctx.entries.push_back(de);
+        }
+
+        for version_object in output.version {
+            // `list` must be additive, so we need to include the latest version object
+            // even if `versions` is not enabled.
+            //
+            // Here we skip all non-latest version objects if `versions` is not enabled.
+            if !(self.args.versions() || version_object.is_latest) {
+                continue;
+            }
+
+            let mut path = build_rel_path(&self.core.root, &version_object.key);
+            if path.is_empty() {
+                path = "/".to_owned();
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+            meta.set_version(&version_object.version_id);
+            meta.set_is_current(version_object.is_latest);
+            meta.set_content_length(version_object.size);
+            meta.set_last_modified(parse_datetime_from_rfc3339(
+                version_object.last_modified.as_str(),
+            )?);
+            if let Some(etag) = version_object.etag {
+                meta.set_etag(&etag);
+                meta.set_content_md5(etag.trim_matches('"'));
+            }
+
+            let entry = oio::Entry::new(&path, meta);
+            ctx.entries.push_back(entry);
+        }
+
+        if self.args.deleted() {
+            for delete_marker in output.delete_marker {
+                let mut path = build_rel_path(&self.core.root, &delete_marker.key);
+                if path.is_empty() {
+                    path = "/".to_owned();
+                }
+
+                let mut meta = Metadata::new(EntryMode::FILE);
+                meta.set_version(&delete_marker.version_id);
+                meta.set_is_deleted(true);
+                meta.set_is_current(delete_marker.is_latest);
+                meta.set_last_modified(parse_datetime_from_rfc3339(
+                    delete_marker.last_modified.as_str(),
+                )?);
+
+                let entry = oio::Entry::new(&path, meta);
+                ctx.entries.push_back(entry);
+            }
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/oss/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for oss service.
+#[cfg(feature = "services-oss")]
+pub(super) const DEFAULT_SCHEME: &str = "oss";
+#[cfg(feature = "services-oss")]
+mod core;
+#[cfg(feature = "services-oss")]
+mod delete;
+#[cfg(feature = "services-oss")]
+mod error;
+#[cfg(feature = "services-oss")]
+mod lister;
+#[cfg(feature = "services-oss")]
+mod writer;
+
+#[cfg(feature = "services-oss")]
+mod backend;
+#[cfg(feature = "services-oss")]
+pub use backend::OssBuilder as Oss;
+
+mod config;
+pub use config::OssConfig;