opendal 0.1.6.pre.rc.1

data/core/src/services/swift/backend.rs

@@ -0,0 +1,275 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+use log::debug;
+
+use super::core::*;
+use super::delete::SwfitDeleter;
+use super::error::parse_error;
+use super::lister::SwiftLister;
+use super::writer::SwiftWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::SwiftConfig;
+use crate::*;
+impl Configurator for SwiftConfig {
+    type Builder = SwiftBuilder;
+    fn into_builder(self) -> Self::Builder {
+        SwiftBuilder { config: self }
+    }
+}
+
+/// [OpenStack Swift](https://docs.openstack.org/api-ref/object-store/#)'s REST API support.
+/// For more information about swift-compatible services, refer to [Compatible Services](#compatible-services).
+#[doc = include_str!("docs.md")]
+#[doc = include_str!("compatible_services.md")]
+#[derive(Default, Clone)]
+pub struct SwiftBuilder {
+    config: SwiftConfig,
+}
+
+impl Debug for SwiftBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("SwiftBuilder");
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl SwiftBuilder {
+    /// Set the remote address of this backend
+    ///
+    /// Endpoints should be full uri, e.g.
+    ///
+    /// - `http://127.0.0.1:8080/v1/AUTH_test`
+    /// - `http://192.168.66.88:8080/swift/v1`
+    /// - `https://openstack-controller.example.com:8080/v1/ccount`
+    ///
+    /// If user inputs endpoint without scheme, we will
+    /// prepend `https://` to it.
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        self.config.endpoint = if endpoint.is_empty() {
+            None
+        } else {
+            Some(endpoint.trim_end_matches('/').to_string())
+        };
+        self
+    }
+
+    /// Set container of this backend.
+    ///
+    /// All operations will happen under this container. It is required. e.g. `snapshots`
+    pub fn container(mut self, container: &str) -> Self {
+        self.config.container = if container.is_empty() {
+            None
+        } else {
+            Some(container.trim_end_matches('/').to_string())
+        };
+        self
+    }
+
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the token of this backend.
+    ///
+    /// Default to empty string.
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for SwiftBuilder {
+    type Config = SwiftConfig;
+
+    /// Build a SwiftBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let endpoint = match self.config.endpoint {
+            Some(endpoint) => {
+                if endpoint.starts_with("http") {
+                    endpoint
+                } else {
+                    format!("https://{endpoint}")
+                }
+            }
+            None => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "missing endpoint for Swift",
+                ));
+            }
+        };
+        debug!("backend use endpoint: {}", &endpoint);
+
+        let container = match self.config.container {
+            Some(container) => container,
+            None => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "missing container for Swift",
+                ));
+            }
+        };
+
+        let token = self.config.token.unwrap_or_default();
+
+        Ok(SwiftBackend {
+            core: Arc::new(SwiftCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_root(&root)
+                        .set_native_capability(Capability {
+                            stat: true,
+                            read: true,
+
+                            write: true,
+                            write_can_empty: true,
+                            write_with_user_metadata: true,
+
+                            delete: true,
+
+                            list: true,
+                            list_with_recursive: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+                    am.into()
+                },
+                root,
+                endpoint,
+                container,
+                token,
+            }),
+        })
+    }
+}
+
+/// Backend for Swift service
+#[derive(Debug, Clone)]
+pub struct SwiftBackend {
+    core: Arc<SwiftCore>,
+}
+
+impl Access for SwiftBackend {
+    type Reader = HttpBody;
+    type Writer = oio::OneShotWriter<SwiftWriter>;
+    type Lister = oio::PageLister<SwiftLister>;
+    type Deleter = oio::OneShotDeleter<SwfitDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let resp = self.core.swift_get_metadata(path).await?;
+
+        match resp.status() {
+            StatusCode::OK | StatusCode::NO_CONTENT => {
+                let headers = resp.headers();
+                let mut meta = parse_into_metadata(path, headers)?;
+                let user_meta = parse_prefixed_headers(headers, "x-object-meta-");
+                if !user_meta.is_empty() {
+                    meta = meta.with_user_metadata(user_meta);
+                }
+
+                Ok(RpStat::new(meta))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.swift_read(path, args.range(), &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let writer = SwiftWriter::new(self.core.clone(), args.clone(), path.to_string());
+
+        let w = oio::OneShotWriter::new(writer);
+
+        Ok((RpWrite::default(), w))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(SwfitDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = SwiftLister::new(
+            self.core.clone(),
+            path.to_string(),
+            args.recursive(),
+            args.limit(),
+        );
+
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        // cannot copy objects larger than 5 GB.
+        // Reference: https://docs.openstack.org/api-ref/object-store/#copy-object
+        let resp = self.core.swift_copy(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(RpCopy::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/swift/compatible_services.md

@@ -0,0 +1,53 @@
+## Compatible Services
+
+### OpenStack Swift
+
+[OpenStack Swift](https://docs.openstack.org/swift/latest/) is the default implementations of swift services.
+
+To connect to OpenStack Swift, we need to set:
+
+- `endpoint`: The endpoint of OpenStack Swift, for example: `http://127.0.0.1:8080/v1/AUTH_test`.
+- `container`: The name of OpenStack Swift container.
+- `token`: OpenStack Swift container personal access token.
+
+```rust,ignore
+builder.endpoint("http://127.0.0.1:8080/v1/AUTH_test");
+builder.container("container");
+builder.token("token");
+```
+
+`endpoint` is the full URL that serves as the access point to all containers under an OpenStack Swift account. It represents the entry point for accessing the resources of the account. Alongside `endpoint`, `token` is used as a credential to verify the user's identity and authorize access to the relevant resources. Both `endpoint` and `token` can be obtained through OpenStack Swift authentication service.
+
+`endpoint` consists of server address and port, API version, authenticated account ID. For instance, it might appear as follows:
+
+- `http://127.0.0.1:8080/v1/AUTH_test`.
+- `http://192.168.66.88:8080/swift/v1`.
+- `https://openstack-controller.example.com:8080/v1/account`.
+
+Please note that the exact format of `endpoint` may vary depending on the deployment configuration and version of swift services. Users can refer to the specific services documentation for the correct `endpoint` format and authentication method.
+
+For more information, refer:
+
+- [OpenStack Swift API](https://docs.openstack.org/api-ref/object-store/).
+- [OpenStack Swift Authentication](https://docs.openstack.org/swift/latest/api/object_api_v1_overview.html).
+
+### Ceph Rados Gateway
+
+[Ceph Rados Gateway](https://docs.ceph.com/en/quincy/radosgw/) supports a RESTful API that is compatible with the basic data access model of OpenStack Swift API.
+
+To connect to Ceph Rados Gateway, we need to set:
+
+- `endpoint`: The endpoint of swift services, for example: `http://127.0.0.1:8080/swift/v1`.
+- `container`: The name of swift container.
+- `token`: swift container personal access token.
+
+```rust,ignore
+builder.endpoint("http://127.0.0.1:8080/swift/v1");
+builder.container("container");
+builder.token("token");
+```
+
+For more information, refer:
+
+- [Ceph Rados Gateway Swift API](https://docs.ceph.com/en/latest/radosgw/swift/#api).
+- [Ceph Rados Gateway Swift Authentication](https://docs.ceph.com/en/latest/radosgw/swift/auth/).

data/core/src/services/swift/config.rs

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for OpenStack Swift support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct SwiftConfig {
+    /// The endpoint for Swift.
+    pub endpoint: Option<String>,
+    /// The container for Swift.
+    pub container: Option<String>,
+    /// The root for Swift.
+    pub root: Option<String>,
+    /// The token for Swift.
+    pub token: Option<String>,
+}
+
+impl Debug for SwiftConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("SwiftConfig");
+
+        ds.field("root", &self.root);
+        ds.field("endpoint", &self.endpoint);
+        ds.field("container", &self.container);
+
+        if self.token.is_some() {
+            ds.field("token", &"<redacted>");
+        }
+
+        ds.finish()
+    }
+}

data/core/src/services/swift/core.rs

@@ -0,0 +1,310 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use http::header;
+use http::Request;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct SwiftCore {
+    pub info: Arc<AccessorInfo>,
+    pub root: String,
+    pub endpoint: String,
+    pub container: String,
+    pub token: String,
+}
+
+impl Debug for SwiftCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SwiftCore")
+            .field("root", &self.root)
+            .field("endpoint", &self.endpoint)
+            .field("container", &self.container)
+            .finish_non_exhaustive()
+    }
+}
+
+impl SwiftCore {
+    pub async fn swift_delete(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::delete(&url);
+
+        req = req.header("X-Auth-Token", &self.token);
+
+        let body = Buffer::new();
+
+        let req = req
+            .extension(Operation::Delete)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn swift_list(
+        &self,
+        path: &str,
+        delimiter: &str,
+        limit: Option<usize>,
+        marker: &str,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        // The delimiter is used to disable recursive listing.
+        // Swift returns a 200 status code when there is no such pseudo directory in prefix.
+        let mut url = QueryPairsWriter::new(&format!("{}/{}/", &self.endpoint, &self.container,))
+            .push("prefix", &percent_encode_path(&p))
+            .push("delimiter", delimiter)
+            .push("format", "json");
+
+        if let Some(limit) = limit {
+            url = url.push("limit", &limit.to_string());
+        }
+        if !marker.is_empty() {
+            url = url.push("marker", marker);
+        }
+
+        let mut req = Request::get(url.finish());
+
+        req = req.header("X-Auth-Token", &self.token);
+
+        let req = req
+            .extension(Operation::List)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn swift_create_object(
+        &self,
+        path: &str,
+        length: u64,
+        args: &OpWrite,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+        let url = format!(
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::put(&url);
+
+        // Set user metadata headers.
+        if let Some(user_metadata) = args.user_metadata() {
+            for (k, v) in user_metadata {
+                req = req.header(format!("X-Object-Meta-{k}"), v);
+            }
+        }
+
+        req = req.header("X-Auth-Token", &self.token);
+        req = req.header(header::CONTENT_LENGTH, length);
+
+        let req = req
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn swift_read(
+        &self,
+        path: &str,
+        range: BytesRange,
+        _arg: &OpRead,
+    ) -> Result<Response<HttpBody>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::get(&url);
+
+        req = req.header("X-Auth-Token", &self.token);
+
+        if !range.is_full() {
+            req = req.header(header::RANGE, range.to_header());
+        }
+
+        let req = req
+            .extension(Operation::Read)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub async fn swift_copy(&self, src_p: &str, dst_p: &str) -> Result<Response<Buffer>> {
+        // NOTE: current implementation is limited to same container and root
+
+        let src_p = format!(
+            "/{}/{}",
+            self.container,
+            build_abs_path(&self.root, src_p).trim_end_matches('/')
+        );
+
+        let dst_p = build_abs_path(&self.root, dst_p)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
+            percent_encode_path(&dst_p)
+        );
+
+        // Request method doesn't support for COPY, we use PUT instead.
+        // Reference: https://docs.openstack.org/api-ref/object-store/#copy-object
+        let mut req = Request::put(&url);
+
+        req = req.header("X-Auth-Token", &self.token);
+        req = req.header("X-Copy-From", percent_encode_path(&src_p));
+
+        // if use PUT method, we need to set the content-length to 0.
+        req = req.header("Content-Length", "0");
+
+        let body = Buffer::new();
+
+        let req = req
+            .extension(Operation::Copy)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn swift_get_metadata(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "{}/{}/{}",
+            &self.endpoint,
+            &self.container,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::head(&url);
+
+        req = req.header("X-Auth-Token", &self.token);
+
+        let req = req
+            .extension(Operation::Stat)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+}
+
+#[derive(Debug, Eq, PartialEq, Deserialize)]
+#[serde(untagged)]
+pub enum ListOpResponse {
+    Subdir {
+        subdir: String,
+    },
+    FileInfo {
+        bytes: u64,
+        hash: String,
+        name: String,
+        last_modified: String,
+        content_type: Option<String>,
+    },
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_list_response_test() -> Result<()> {
+        let resp = bytes::Bytes::from(
+            r#"
+            [
+                {
+                    "subdir": "animals/"
+                },
+                {
+                    "subdir": "fruit/"
+                },
+                {
+                    "bytes": 147,
+                    "hash": "5e6b5b70b0426b1cc1968003e1afa5ad",
+                    "name": "test.txt",
+                    "content_type": "text/plain",
+                    "last_modified": "2023-11-01T03:00:23.147480"
+                }
+            ]
+            "#,
+        );
+
+        let mut out = serde_json::from_slice::<Vec<ListOpResponse>>(&resp)
+            .map_err(new_json_deserialize_error)?;
+
+        assert_eq!(out.len(), 3);
+        assert_eq!(
+            out.pop().unwrap(),
+            ListOpResponse::FileInfo {
+                bytes: 147,
+                hash: "5e6b5b70b0426b1cc1968003e1afa5ad".to_string(),
+                name: "test.txt".to_string(),
+                last_modified: "2023-11-01T03:00:23.147480".to_string(),
+                content_type: Some("text/plain".to_string()),
+            }
+        );
+
+        assert_eq!(
+            out.pop().unwrap(),
+            ListOpResponse::Subdir {
+                subdir: "fruit/".to_string()
+            }
+        );
+
+        assert_eq!(
+            out.pop().unwrap(),
+            ListOpResponse::Subdir {
+                subdir: "animals/".to_string()
+            }
+        );
+
+        Ok(())
+    }
+}