opendal 0.1.6.pre.rc.1

data/core/src/services/s3/lister.rs

@@ -0,0 +1,405 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use quick_xml::de;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::PageContext;
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Error;
+use crate::Metadata;
+use crate::Result;
+
+pub type S3Listers = ThreeWays<
+    oio::PageLister<S3ListerV1>,
+    oio::PageLister<S3ListerV2>,
+    oio::PageLister<S3ObjectVersionsLister>,
+>;
+
+/// S3ListerV1 implements ListObjectV1 for s3 backend.
+pub struct S3ListerV1 {
+    core: Arc<S3Core>,
+
+    path: String,
+    args: OpList,
+
+    delimiter: &'static str,
+    /// marker can also be used as `start-after` for list objects v1.
+    /// We will use it as `start-after` for the first page and then ignore
+    /// it in the following pages.
+    first_marker: String,
+}
+
+impl S3ListerV1 {
+    pub fn new(core: Arc<S3Core>, path: &str, args: OpList) -> Self {
+        let delimiter = if args.recursive() { "" } else { "/" };
+        let first_marker = args
+            .start_after()
+            .map(|start_after| build_abs_path(&core.root, start_after))
+            .unwrap_or_default();
+
+        Self {
+            core,
+
+            path: path.to_string(),
+            args,
+            delimiter,
+            first_marker,
+        }
+    }
+}
+
+impl oio::PageList for S3ListerV1 {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .s3_list_objects_v1(
+                &self.path,
+                // `marker` is used as `start-after` for the first page.
+                if !ctx.token.is_empty() {
+                    &ctx.token
+                } else {
+                    &self.first_marker
+                },
+                self.delimiter,
+                self.args.limit(),
+            )
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+        let bs = resp.into_body();
+
+        let output: ListObjectsOutputV1 = de::from_reader(bs.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow S3 list to retry on XML deserialization errors.
+            //
+            // This is because the S3 list API may return incomplete XML data under high load.
+            // We are confident that our XML decoding logic is correct. When this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
+
+        // Try our best to check whether this list is done.
+        //
+        // - Check `is_truncated`
+        // - Check the length of `common_prefixes` and `contents` (very rare case)
+        ctx.done = if let Some(is_truncated) = output.is_truncated {
+            !is_truncated
+        } else {
+            output.common_prefixes.is_empty() && output.contents.is_empty()
+        };
+        // Try out best to find the next marker.
+        //
+        // - Check `next-marker`
+        // - Check the last object key
+        // - Check the last common prefix
+        ctx.token = if let Some(next_marker) = &output.next_marker {
+            next_marker.clone()
+        } else if let Some(content) = output.contents.last() {
+            content.key.clone()
+        } else if let Some(prefix) = output.common_prefixes.last() {
+            prefix.prefix.clone()
+        } else {
+            "".to_string()
+        };
+
+        for prefix in output.common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+
+            ctx.entries.push_back(de);
+        }
+
+        for object in output.contents {
+            let mut path = build_rel_path(&self.core.root, &object.key);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+            meta.set_is_current(true);
+            if let Some(etag) = &object.etag {
+                meta.set_etag(etag);
+                meta.set_content_md5(etag.trim_matches('"'));
+            }
+            meta.set_content_length(object.size);
+
+            // object.last_modified provides more precise time that contains
+            // nanosecond, let's trim them.
+            meta.set_last_modified(parse_datetime_from_rfc3339(object.last_modified.as_str())?);
+
+            let de = oio::Entry::with(path, meta);
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}
+
+/// S3ListerV2 implements ListObjectV2 for s3 backend.
+pub struct S3ListerV2 {
+    core: Arc<S3Core>,
+
+    path: String,
+    args: OpList,
+
+    delimiter: &'static str,
+    abs_start_after: Option<String>,
+}
+
+impl S3ListerV2 {
+    pub fn new(core: Arc<S3Core>, path: &str, args: OpList) -> Self {
+        let delimiter = if args.recursive() { "" } else { "/" };
+        let abs_start_after = args
+            .start_after()
+            .map(|start_after| build_abs_path(&core.root, start_after));
+
+        Self {
+            core,
+
+            path: path.to_string(),
+            args,
+            delimiter,
+            abs_start_after,
+        }
+    }
+}
+
+impl oio::PageList for S3ListerV2 {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .s3_list_objects_v2(
+                &self.path,
+                &ctx.token,
+                self.delimiter,
+                self.args.limit(),
+                // start after should only be set for the first page.
+                if ctx.token.is_empty() {
+                    self.abs_start_after.clone()
+                } else {
+                    None
+                },
+            )
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+        let bs = resp.into_body();
+
+        let output: ListObjectsOutputV2 = de::from_reader(bs.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow S3 list to retry on XML deserialization errors.
+            //
+            // This is because the S3 list API may return incomplete XML data under high load.
+            // We are confident that our XML decoding logic is correct. When this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
+
+        // Try our best to check whether this list is done.
+        //
+        // - Check `is_truncated`
+        // - Check `next_continuation_token`
+        // - Check the length of `common_prefixes` and `contents` (very rare case)
+        ctx.done = if let Some(is_truncated) = output.is_truncated {
+            !is_truncated
+        } else if let Some(next_continuation_token) = output.next_continuation_token.as_ref() {
+            next_continuation_token.is_empty()
+        } else {
+            output.common_prefixes.is_empty() && output.contents.is_empty()
+        };
+        ctx.token = output.next_continuation_token.clone().unwrap_or_default();
+
+        for prefix in output.common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+
+            ctx.entries.push_back(de);
+        }
+
+        for object in output.contents {
+            let mut path = build_rel_path(&self.core.root, &object.key);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+            meta.set_is_current(true);
+            if let Some(etag) = &object.etag {
+                meta.set_etag(etag);
+                meta.set_content_md5(etag.trim_matches('"'));
+            }
+            meta.set_content_length(object.size);
+
+            // object.last_modified provides more precise time that contains
+            // nanosecond, let's trim them.
+            meta.set_last_modified(parse_datetime_from_rfc3339(object.last_modified.as_str())?);
+
+            let de = oio::Entry::with(path, meta);
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}
+
+/// refer: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html
+pub struct S3ObjectVersionsLister {
+    core: Arc<S3Core>,
+
+    prefix: String,
+    args: OpList,
+
+    delimiter: &'static str,
+    abs_start_after: Option<String>,
+}
+
+impl S3ObjectVersionsLister {
+    pub fn new(core: Arc<S3Core>, path: &str, args: OpList) -> Self {
+        let delimiter = if args.recursive() { "" } else { "/" };
+        let abs_start_after = args
+            .start_after()
+            .map(|start_after| build_abs_path(&core.root, start_after));
+
+        Self {
+            core,
+            prefix: path.to_string(),
+            args,
+            delimiter,
+            abs_start_after,
+        }
+    }
+}
+
+impl oio::PageList for S3ObjectVersionsLister {
+    async fn next_page(&self, ctx: &mut PageContext) -> Result<()> {
+        let markers = ctx.token.rsplit_once(" ");
+        let (key_marker, version_id_marker) = if let Some(data) = markers {
+            data
+        } else if let Some(start_after) = &self.abs_start_after {
+            (start_after.as_str(), "")
+        } else {
+            ("", "")
+        };
+
+        let resp = self
+            .core
+            .s3_list_object_versions(
+                &self.prefix,
+                self.delimiter,
+                self.args.limit(),
+                key_marker,
+                version_id_marker,
+            )
+            .await?;
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let body = resp.into_body();
+        let output: ListObjectVersionsOutput = de::from_reader(body.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow S3 list to retry on XML deserialization errors.
+            //
+            // This is because the S3 list API may return incomplete XML data under high load.
+            // We are confident that our XML decoding logic is correct. When this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
+
+        ctx.done = if let Some(is_truncated) = output.is_truncated {
+            !is_truncated
+        } else {
+            false
+        };
+        ctx.token = format!(
+            "{} {}",
+            output.next_key_marker.unwrap_or_default(),
+            output.next_version_id_marker.unwrap_or_default()
+        );
+
+        for prefix in output.common_prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+            ctx.entries.push_back(de);
+        }
+
+        for version_object in output.version {
+            // `list` must be additive, so we need to include the latest version object
+            // even if `versions` is not enabled.
+            //
+            // Here we skip all non-latest version objects if `versions` is not enabled.
+            if !(self.args.versions() || version_object.is_latest) {
+                continue;
+            }
+
+            let mut path = build_rel_path(&self.core.root, &version_object.key);
+            if path.is_empty() {
+                path = "/".to_owned();
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+            meta.set_version(&version_object.version_id);
+            meta.set_is_current(version_object.is_latest);
+            meta.set_content_length(version_object.size);
+            meta.set_last_modified(parse_datetime_from_rfc3339(
+                version_object.last_modified.as_str(),
+            )?);
+            if let Some(etag) = version_object.etag {
+                meta.set_etag(&etag);
+                meta.set_content_md5(etag.trim_matches('"'));
+            }
+
+            let entry = oio::Entry::new(&path, meta);
+            ctx.entries.push_back(entry);
+        }
+
+        if self.args.deleted() {
+            for delete_marker in output.delete_marker {
+                let mut path = build_rel_path(&self.core.root, &delete_marker.key);
+                if path.is_empty() {
+                    path = "/".to_owned();
+                }
+
+                let mut meta = Metadata::new(EntryMode::FILE);
+                meta.set_version(&delete_marker.version_id);
+                meta.set_is_deleted(true);
+                meta.set_is_current(delete_marker.is_latest);
+                meta.set_last_modified(parse_datetime_from_rfc3339(
+                    delete_marker.last_modified.as_str(),
+                )?);
+
+                let entry = oio::Entry::new(&path, meta);
+                ctx.entries.push_back(entry);
+            }
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/s3/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for s3 service.
+#[cfg(feature = "services-s3")]
+pub const S3_SCHEME: &str = "s3";
+#[cfg(feature = "services-s3")]
+mod core;
+#[cfg(feature = "services-s3")]
+mod delete;
+#[cfg(feature = "services-s3")]
+mod error;
+#[cfg(feature = "services-s3")]
+mod lister;
+#[cfg(feature = "services-s3")]
+mod writer;
+
+#[cfg(feature = "services-s3")]
+mod backend;
+#[cfg(feature = "services-s3")]
+pub use backend::S3Builder as S3;
+
+mod config;
+pub use config::S3Config;

data/core/src/services/s3/writer.rs

@@ -0,0 +1,262 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use constants::X_AMZ_OBJECT_SIZE;
+use constants::X_AMZ_VERSION_ID;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::from_s3_error;
+use super::error::parse_error;
+use super::error::S3Error;
+use crate::raw::*;
+use crate::*;
+
+pub type S3Writers = TwoWays<oio::MultipartWriter<S3Writer>, oio::AppendWriter<S3Writer>>;
+
+pub struct S3Writer {
+    core: Arc<S3Core>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl S3Writer {
+    pub fn new(core: Arc<S3Core>, path: &str, op: OpWrite) -> Self {
+        S3Writer {
+            core,
+            path: path.to_string(),
+            op,
+        }
+    }
+
+    fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> {
+        let mut meta = Metadata::new(EntryMode::from_path(path));
+        if let Some(etag) = parse_etag(headers)? {
+            meta.set_etag(etag);
+        }
+        if let Some(version) = parse_header_to_str(headers, X_AMZ_VERSION_ID)? {
+            meta.set_version(version);
+        }
+        if let Some(size) = parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)? {
+            if let Ok(value) = size.parse() {
+                meta.set_content_length(value);
+            }
+        }
+        Ok(meta)
+    }
+}
+
+impl oio::MultipartWrite for S3Writer {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .s3_put_object_request(&self.path, Some(size), &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        let meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self
+            .core
+            .s3_initiate_multipart_upload(&self.path, &self.op)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: InitiateMultipartUploadResult =
+                    quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+
+                Ok(result.upload_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<oio::MultipartPart> {
+        // AWS S3 requires part number must between [1..=10000]
+        let part_number = part_number + 1;
+
+        let checksum = self.core.calculate_checksum(&body);
+
+        let mut req = self.core.s3_upload_part_request(
+            &self.path,
+            upload_id,
+            part_number,
+            size,
+            body,
+            checksum.clone(),
+        )?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let etag = parse_etag(resp.headers())?
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "ETag not present in returning response",
+                        )
+                    })?
+                    .to_string();
+
+                Ok(oio::MultipartPart {
+                    part_number,
+                    etag,
+                    checksum,
+                })
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        parts: &[oio::MultipartPart],
+    ) -> Result<Metadata> {
+        let parts = parts
+            .iter()
+            .map(|p| match &self.core.checksum_algorithm {
+                None => CompleteMultipartUploadRequestPart {
+                    part_number: p.part_number,
+                    etag: p.etag.clone(),
+                    ..Default::default()
+                },
+                Some(checksum_algorithm) => match checksum_algorithm {
+                    ChecksumAlgorithm::Crc32c => CompleteMultipartUploadRequestPart {
+                        part_number: p.part_number,
+                        etag: p.etag.clone(),
+                        checksum_crc32c: p.checksum.clone(),
+                    },
+                },
+            })
+            .collect();
+
+        let resp = self
+            .core
+            .s3_complete_multipart_upload(&self.path, upload_id, parts)
+            .await?;
+
+        let status = resp.status();
+
+        let mut meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
+
+        match status {
+            StatusCode::OK => {
+                // still check if there is any error because S3 might return error for status code 200
+                // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_Example_4
+                let (parts, body) = resp.into_parts();
+
+                let ret: CompleteMultipartUploadResult =
+                    quick_xml::de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
+                if !ret.code.is_empty() {
+                    return Err(from_s3_error(
+                        S3Error {
+                            code: ret.code,
+                            message: ret.message,
+                            resource: "".to_string(),
+                            request_id: ret.request_id,
+                        },
+                        parts,
+                    ));
+                }
+                meta.set_etag(&ret.etag);
+
+                Ok(meta)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self
+            .core
+            .s3_abort_multipart_upload(&self.path, upload_id)
+            .await?;
+        match resp.status() {
+            // s3 returns code 204 if abort succeeds.
+            StatusCode::NO_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+impl oio::AppendWrite for S3Writer {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self
+            .core
+            .s3_head_object(&self.path, OpStat::default())
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(parse_content_length(resp.headers())?.unwrap_or_default()),
+            StatusCode::NOT_FOUND => Ok(0),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .s3_append_object_request(&self.path, offset, size, &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        let meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}