opendal 0.1.6.pre.rc.1

data/core/src/services/s3/delete.rs

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use super::error::parse_s3_error_code;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::*;
+
+pub struct S3Deleter {
+    core: Arc<S3Core>,
+}
+
+impl S3Deleter {
+    pub fn new(core: Arc<S3Core>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::BatchDelete for S3Deleter {
+    async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
+        // This would delete the bucket, do not perform
+        if self.core.root == "/" && path == "/" {
+            return Ok(());
+        }
+
+        let resp = self.core.s3_delete_object(&path, &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT => Ok(()),
+            // Allow 404 when deleting a non-existing object
+            // This is not a standard behavior, only some s3 alike service like GCS XML API do this.
+            // ref: <https://cloud.google.com/storage/docs/xml-api/delete-object>
+            StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+        let resp = self.core.s3_delete_objects(batch).await?;
+
+        let status = resp.status();
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let result: DeleteObjectsResult =
+            quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+
+        let mut batched_result = BatchDeleteResult {
+            succeeded: Vec::with_capacity(result.deleted.len()),
+            failed: Vec::with_capacity(result.error.len()),
+        };
+        for i in result.deleted {
+            let path = build_rel_path(&self.core.root, &i.key);
+            let mut op = OpDelete::new();
+            if let Some(version_id) = i.version_id {
+                op = op.with_version(version_id.as_str());
+            }
+            batched_result.succeeded.push((path, op));
+        }
+        for i in result.error {
+            let path = build_rel_path(&self.core.root, &i.key);
+            let mut op = OpDelete::new();
+            if let Some(version_id) = &i.version_id {
+                op = op.with_version(version_id.as_str());
+            }
+            batched_result
+                .failed
+                .push((path, op, parse_delete_objects_result_error(i)));
+        }
+
+        Ok(batched_result)
+    }
+}
+
+fn parse_delete_objects_result_error(err: DeleteObjectsResultError) -> Error {
+    let (kind, retryable) =
+        parse_s3_error_code(err.code.as_str()).unwrap_or((ErrorKind::Unexpected, false));
+    let mut err: Error = Error::new(kind, format!("{err:?}"));
+    if retryable {
+        err = err.set_temporary();
+    }
+    err
+}

data/core/src/services/s3/docs.md

@@ -0,0 +1,244 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] append
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [x] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- `bucket`: Set the container name for backend.
+- `endpoint`: Set the endpoint for backend.
+- `region`: Set the region for backend.
+- `access_key_id`: Set the access_key_id for backend.
+- `secret_access_key`: Set the secret_access_key for backend.
+- `session_token`: Set the session_token for backend.
+- `default_storage_class`: Set the default storage_class for backend.
+- `server_side_encryption`: Set the server_side_encryption for backend.
+- `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend.
+- `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for backend.
+- `server_side_encryption_customer_key`: Set the server_side_encryption_customer_key for backend.
+- `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend.
+- `disable_config_load`: Disable aws config load from env.
+- `enable_virtual_host_style`: Enable virtual host style.
+- `disable_write_with_if_match`: Disable write with if match.
+- `enable_request_payer`: Enable the request payer for backend.
+
+Refer to [`S3Builder`]'s public API docs for more information.
+
+## Temporary security credentials
+
+OpenDAL now provides support for S3 temporary security credentials in IAM.
+
+The way to take advantage of this feature is to build your S3 backend with `Builder::session_token`.
+
+But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those credentials in time.
+
+## Server Side Encryption
+
+OpenDAL provides full support of S3 Server Side Encryption(SSE) features.
+
+The easiest way to configure them is to use helper functions like
+
+- SSE-KMS: `server_side_encryption_with_aws_managed_kms_key`
+- SSE-KMS: `server_side_encryption_with_customer_managed_kms_key`
+- SSE-S3: `server_side_encryption_with_s3_key`
+- SSE-C: `server_side_encryption_with_customer_key`
+
+If those functions don't fulfill need, low-level options are also provided:
+
+- Use service managed kms key
+    - `server_side_encryption="aws:kms"`
+- Use customer provided kms key
+    - `server_side_encryption="aws:kms"`
+    - `server_side_encryption_aws_kms_key_id="your-kms-key"`
+- Use S3 managed key
+    - `server_side_encryption="AES256"`
+- Use customer key
+    - `server_side_encryption_customer_algorithm="AES256"`
+    - `server_side_encryption_customer_key="base64-of-your-aes256-key"`
+    - `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"`
+
+After SSE have been configured, all requests send by this backed will attach those headers.
+
+Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html)
+
+## Example
+
+## Via Builder
+
+### Basic Setup
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::S3;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create s3 backend builder.
+    let mut builder = S3::default()
+      // Set the root for s3, all operations will happen under this root.
+      //
+      // NOTE: the root must be absolute path.
+      .root("/path/to/dir")
+      // Set the bucket name. This is required.
+      .bucket("test")
+      // Set the region. This is required for some services, if you don't care about it, for example Minio service, just set it to "auto", it will be ignored.
+      .region("us-east-1")
+      // Set the endpoint.
+      //
+      // For examples:
+      // - "https://s3.amazonaws.com"
+      // - "http://127.0.0.1:9000"
+      // - "https://oss-ap-northeast-1.aliyuncs.com"
+      // - "https://cos.ap-seoul.myqcloud.com"
+      //
+      // Default to "https://s3.amazonaws.com"
+      .endpoint("https://s3.amazonaws.com")
+      // Set the access_key_id and secret_access_key.
+      //
+      // OpenDAL will try load credential from the env.
+      // If credential not set and no valid credential in env, OpenDAL will
+      // send request without signing like anonymous user.
+      .access_key_id("access_key_id")
+      .secret_access_key("secret_access_key");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+
+```
+
+### S3 with SSE-C
+
+```rust,no_run
+use anyhow::Result;
+use log::info;
+use opendal::services::S3;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = S3::default()
+      .root("/path/to/dir")
+      .bucket("test")
+      .region("us-east-1")
+      .endpoint("https://s3.amazonaws.com")
+      .access_key_id("access_key_id")
+      .secret_access_key("secret_access_key")
+      // Enable SSE-C
+      .server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes());
+
+    let op = Operator::new(builder)?.finish();
+    info!("operator: {:?}", op);
+
+    // Writing your testing code here.
+
+    Ok(())
+}
+```
+
+### S3 with SSE-KMS and aws managed kms key
+
+```rust,no_run
+use anyhow::Result;
+use log::info;
+use opendal::services::S3;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = S3::default()
+      // Setup builders
+      .root("/path/to/dir")
+      .bucket("test")
+      .region("us-east-1")
+      .endpoint("https://s3.amazonaws.com")
+      .access_key_id("access_key_id")
+      .secret_access_key("secret_access_key")
+      // Enable SSE-KMS with aws managed kms key
+      .server_side_encryption_with_aws_managed_kms_key();
+
+    let op = Operator::new(builder)?.finish();
+    info!("operator: {:?}", op);
+
+    // Writing your testing code here.
+
+    Ok(())
+}
+```
+
+### S3 with SSE-KMS and customer managed kms key
+
+```rust,no_run
+use anyhow::Result;
+use log::info;
+use opendal::services::S3;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = S3::default()
+      // Setup builders
+      .root("/path/to/dir")
+      .bucket("test")
+      .region("us-east-1")
+      .endpoint("https://s3.amazonaws.com")
+      .access_key_id("access_key_id")
+      .secret_access_key("secret_access_key")
+      // Enable SSE-KMS with customer managed kms key
+      .server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id");
+
+    let op = Operator::new(builder)?.finish();
+    info!("operator: {:?}", op);
+
+    // Writing your testing code here.
+
+    Ok(())
+}
+```
+
+### S3 with SSE-S3
+
+```rust,no_run
+use anyhow::Result;
+use log::info;
+use opendal::services::S3;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = S3::default()
+      // Setup builders
+      .root("/path/to/dir")
+      .bucket("test")
+      .region("us-east-1")
+      .endpoint("https://s3.amazonaws.com")
+      .access_key_id("access_key_id")
+      .secret_access_key("secret_access_key")
+      // Enable SSE-S3
+      .server_side_encryption_with_s3_key();
+
+    let op = Operator::new(builder)?.finish();
+    info!("operator: {:?}", op);
+
+    // Writing your testing code here.
+
+    Ok(())
+}
+```
+

data/core/src/services/s3/error.rs

@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::response::Parts;
+use http::Response;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// S3Error is the error returned by s3 service.
+#[derive(Default, Debug, Deserialize, PartialEq, Eq)]
+#[serde(default, rename_all = "PascalCase")]
+pub(crate) struct S3Error {
+    pub code: String,
+    pub message: String,
+    pub resource: String,
+    pub request_id: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // Service like R2 could return 499 error with a message like:
+        // Client Disconnect, we should retry it.
+        499 => (ErrorKind::Unexpected, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        429 => (ErrorKind::RateLimited, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let body_content = bs.chunk();
+    let (message, s3_err) = de::from_reader::<_, S3Error>(body_content.reader())
+        .map(|s3_err| (format!("{s3_err:?}"), Some(s3_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(s3_err) = s3_err {
+        (kind, retryable) = parse_s3_error_code(s3_err.code.as_str()).unwrap_or((kind, retryable));
+    }
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+/// Util function to build [`Error`] from a [`S3Error`] object.
+pub(crate) fn from_s3_error(s3_error: S3Error, parts: Parts) -> Error {
+    let (kind, retryable) =
+        parse_s3_error_code(s3_error.code.as_str()).unwrap_or((ErrorKind::Unexpected, false));
+    let mut err = Error::new(kind, format!("{s3_error:?}"));
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+/// Returns the `Error kind` of this code and whether the error is retryable.
+/// All possible error code: <https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList>
+pub fn parse_s3_error_code(code: &str) -> Option<(ErrorKind, bool)> {
+    match code {
+        // > The specified bucket does not exist.
+        //
+        // Although the status code is 404, NoSuchBucket is
+        // a config invalid error, and it's not retryable from OpenDAL.
+        "NoSuchBucket" => Some((ErrorKind::ConfigInvalid, false)),
+        // > Your socket connection to the server was not read from
+        // > or written to within the timeout period."
+        //
+        // It's Ok for us to retry it again.
+        "RequestTimeout" => Some((ErrorKind::Unexpected, true)),
+        // > An internal error occurred. Try again.
+        "InternalError" => Some((ErrorKind::Unexpected, true)),
+        // > A conflicting conditional operation is currently in progress
+        // > against this resource. Try again.
+        "OperationAborted" => Some((ErrorKind::Unexpected, true)),
+        // > Please reduce your request rate.
+        //
+        // It's Ok to retry since later on the request rate may get reduced.
+        "SlowDown" => Some((ErrorKind::RateLimited, true)),
+        // > Service is unable to handle request.
+        //
+        // ServiceUnavailable is considered a retryable error because it typically
+        // indicates a temporary issue with the service or server, such as high load,
+        // maintenance, or an internal problem.
+        "ServiceUnavailable" => Some((ErrorKind::Unexpected, true)),
+        // > Too Many Requests - rate limit exceeded.
+        //
+        // It's Ok to retry since later on the request rate may get reduced.
+        "TooManyRequests" => Some((ErrorKind::RateLimited, true)),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Error response example is from https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<Error>
+  <Code>NoSuchKey</Code>
+  <Message>The resource you requested does not exist</Message>
+  <Resource>/mybucket/myfoto.jpg</Resource>
+  <RequestId>4442587FB7D0A2F9</RequestId>
+</Error>
+"#,
+        );
+
+        let out: S3Error = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code, "NoSuchKey");
+        assert_eq!(out.message, "The resource you requested does not exist");
+        assert_eq!(out.resource, "/mybucket/myfoto.jpg");
+        assert_eq!(out.request_id, "4442587FB7D0A2F9");
+    }
+
+    #[test]
+    fn test_parse_error_from_unrelated_input() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" encoding="UTF-8"?>
+<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Location>http://Example-Bucket.s3.ap-southeast-1.amazonaws.com/Example-Object</Location>
+  <Bucket>Example-Bucket</Bucket>
+  <Key>Example-Object</Key>
+  <ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
+</CompleteMultipartUploadResult>
+"#,
+        );
+
+        let out: S3Error = de::from_reader(bs.reader()).expect("must success");
+        assert_eq!(out, S3Error::default());
+    }
+}