opendal 0.1.6.pre.rc.1

data/core/src/services/swift/delete.rs

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SwfitDeleter {
+    core: Arc<SwiftCore>,
+}
+
+impl SwfitDeleter {
+    pub fn new(core: Arc<SwiftCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for SwfitDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.swift_delete(&path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT | StatusCode::OK => Ok(()),
+            StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/swift/docs.md

@@ -0,0 +1,52 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] ~~rename~~
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configurations
+
+- `endpoint`: Set the endpoint for backend.
+- `container`: Swift container.
+- `token`: Swift personal access token.
+
+Refer to [`SwiftBuilder`]'s public API docs for more information.
+
+## Examples
+
+### Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Swift;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create Swift backend builder
+    let mut builder = Swift::default() 
+        // Set the root for swift, all operations will happen under this root
+        .root("/path/to/dir")
+        // set the endpoint of Swift backend
+        .endpoint("https://openstack-controller.example.com:8080/v1/account")
+        // set the container name of Swift workspace
+        .container("container")
+        // set the auth token for builder
+        .token("token");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/swift/error.rs

@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use bytes::Bytes;
+use http::Response;
+use http::StatusCode;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+#[allow(dead_code)]
+#[derive(Debug, Deserialize)]
+struct ErrorResponse {
+    h1: String,
+    p: String,
+}
+
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = parse_error_response(&bs);
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+fn parse_error_response(resp: &Bytes) -> String {
+    match de::from_reader::<_, ErrorResponse>(resp.clone().reader()) {
+        Ok(swift_err) => swift_err.p,
+        Err(_) => String::from_utf8_lossy(resp).into_owned(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parse_error_response_test() -> Result<()> {
+        let resp = Bytes::from(
+            r#"
+<html>
+<h1>Not Found</h1>
+<p>The resource could not be found.</p>
+
+</html>
+            "#,
+        );
+
+        let msg = parse_error_response(&resp);
+        assert_eq!(msg, "The resource could not be found.".to_string(),);
+        Ok(())
+    }
+}

data/core/src/services/swift/lister.rs

@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SwiftLister {
+    core: Arc<SwiftCore>,
+    path: String,
+    delimiter: &'static str,
+    limit: Option<usize>,
+}
+
+impl SwiftLister {
+    pub fn new(core: Arc<SwiftCore>, path: String, recursive: bool, limit: Option<usize>) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+        Self {
+            core,
+            path,
+            delimiter,
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for SwiftLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let response = self
+            .core
+            .swift_list(&self.path, self.delimiter, self.limit, &ctx.token)
+            .await?;
+
+        let status_code = response.status();
+
+        if !status_code.is_success() {
+            let error = parse_error(response);
+            return Err(error);
+        }
+
+        let bytes = response.into_body();
+        let decoded_response: Vec<ListOpResponse> =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        // Update token and done based on resp.
+        if let Some(entry) = decoded_response.last() {
+            let path = match entry {
+                ListOpResponse::Subdir { subdir } => subdir,
+                ListOpResponse::FileInfo { name, .. } => name,
+            };
+            ctx.token.clone_from(path);
+        } else {
+            ctx.done = true;
+        }
+
+        for status in decoded_response {
+            let entry: oio::Entry = match status {
+                ListOpResponse::Subdir { subdir } => {
+                    let mut path = build_rel_path(self.core.root.as_str(), subdir.as_str());
+                    if path.is_empty() {
+                        path = "/".to_string();
+                    }
+                    let meta = Metadata::new(EntryMode::DIR);
+                    oio::Entry::with(path, meta)
+                }
+                ListOpResponse::FileInfo {
+                    bytes,
+                    hash,
+                    name,
+                    content_type,
+                    mut last_modified,
+                } => {
+                    let mut path = build_rel_path(self.core.root.as_str(), name.as_str());
+                    if path.is_empty() {
+                        path = "/".to_string();
+                    }
+                    let mut meta = Metadata::new(EntryMode::from_path(path.as_str()));
+                    meta.set_content_length(bytes);
+                    meta.set_content_md5(hash.as_str());
+
+                    // OpenStack Swift returns time without 'Z' at the end,
+                    // which causes an error in parse_datetime_from_rfc3339.
+                    // we'll change "2023-10-28T19:18:11.682610" to "2023-10-28T19:18:11.682610Z".
+                    if !last_modified.ends_with('Z') {
+                        last_modified.push('Z');
+                    }
+                    meta.set_last_modified(parse_datetime_from_rfc3339(last_modified.as_str())?);
+
+                    if let Some(content_type) = content_type {
+                        meta.set_content_type(content_type.as_str());
+                    }
+
+                    oio::Entry::with(path, meta)
+                }
+            };
+            ctx.entries.push_back(entry);
+        }
+        Ok(())
+    }
+}

data/core/src/services/swift/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for swift service.
+#[cfg(feature = "services-swift")]
+pub(super) const DEFAULT_SCHEME: &str = "swift";
+#[cfg(feature = "services-swift")]
+mod core;
+#[cfg(feature = "services-swift")]
+mod delete;
+#[cfg(feature = "services-swift")]
+mod error;
+#[cfg(feature = "services-swift")]
+mod lister;
+#[cfg(feature = "services-swift")]
+mod writer;
+
+#[cfg(feature = "services-swift")]
+mod backend;
+#[cfg(feature = "services-swift")]
+pub use backend::SwiftBuilder as Swift;
+
+mod config;
+pub use config::SwiftConfig;

data/core/src/services/swift/writer.rs

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::SwiftCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SwiftWriter {
+    core: Arc<SwiftCore>,
+    op: OpWrite,
+    path: String,
+}
+
+impl SwiftWriter {
+    pub fn new(core: Arc<SwiftCore>, op: OpWrite, path: String) -> Self {
+        SwiftWriter { core, op, path }
+    }
+}
+
+impl oio::OneShotWrite for SwiftWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .swift_create_object(&self.path, bs.len() as u64, &self.op, bs)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/tikv/backend.rs

@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use tikv_client::Config;
+use tikv_client::RawClient;
+use tokio::sync::OnceCell;
+
+use crate::raw::adapters::kv;
+use crate::raw::Access;
+use crate::services::TikvConfig;
+use crate::Builder;
+use crate::Capability;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Scheme;
+use crate::*;
+
+impl Configurator for TikvConfig {
+    type Builder = TikvBuilder;
+    fn into_builder(self) -> Self::Builder {
+        TikvBuilder { config: self }
+    }
+}
+
+/// TiKV backend builder
+#[doc = include_str!("docs.md")]
+#[derive(Clone, Default)]
+pub struct TikvBuilder {
+    config: TikvConfig,
+}
+
+impl Debug for TikvBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("TikvBuilder");
+
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl TikvBuilder {
+    /// Set the network address of the TiKV service.
+    pub fn endpoints(mut self, endpoints: Vec<String>) -> Self {
+        if !endpoints.is_empty() {
+            self.config.endpoints = Some(endpoints)
+        }
+        self
+    }
+
+    /// Set the insecure connection to TiKV.
+    pub fn insecure(mut self) -> Self {
+        self.config.insecure = true;
+        self
+    }
+
+    /// Set the certificate authority file path.
+    pub fn ca_path(mut self, ca_path: &str) -> Self {
+        if !ca_path.is_empty() {
+            self.config.ca_path = Some(ca_path.to_string())
+        }
+        self
+    }
+
+    /// Set the certificate file path.
+    pub fn cert_path(mut self, cert_path: &str) -> Self {
+        if !cert_path.is_empty() {
+            self.config.cert_path = Some(cert_path.to_string())
+        }
+        self
+    }
+
+    /// Set the key file path.
+    pub fn key_path(mut self, key_path: &str) -> Self {
+        if !key_path.is_empty() {
+            self.config.key_path = Some(key_path.to_string())
+        }
+        self
+    }
+}
+
+impl Builder for TikvBuilder {
+    type Config = TikvConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let endpoints = self.config.endpoints.ok_or_else(|| {
+            Error::new(
+                ErrorKind::ConfigInvalid,
+                "endpoints is required but not set",
+            )
+            .with_context("service", Scheme::Tikv)
+        })?;
+
+        if self.config.insecure
+            && (self.config.ca_path.is_some()
+                || self.config.key_path.is_some()
+                || self.config.cert_path.is_some())
+        {
+            return Err(
+                Error::new(ErrorKind::ConfigInvalid, "invalid tls configuration")
+                    .with_context("service", Scheme::Tikv)
+                    .with_context("endpoints", format!("{endpoints:?}")),
+            )?;
+        }
+
+        Ok(TikvBackend::new(Adapter {
+            client: OnceCell::new(),
+            endpoints,
+            insecure: self.config.insecure,
+            ca_path: self.config.ca_path.clone(),
+            cert_path: self.config.cert_path.clone(),
+            key_path: self.config.key_path.clone(),
+        }))
+    }
+}
+
+/// Backend for TiKV service
+pub type TikvBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    client: OnceCell<RawClient>,
+    endpoints: Vec<String>,
+    insecure: bool,
+    ca_path: Option<String>,
+    cert_path: Option<String>,
+    key_path: Option<String>,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+
+        ds.field("endpoints", &self.endpoints);
+        ds.finish()
+    }
+}
+
+impl Adapter {
+    async fn get_connection(&self) -> Result<RawClient> {
+        if let Some(client) = self.client.get() {
+            return Ok(client.clone());
+        }
+        let client = if self.insecure {
+            RawClient::new(self.endpoints.clone())
+                .await
+                .map_err(parse_tikv_config_error)?
+        } else if self.ca_path.is_some() && self.key_path.is_some() && self.cert_path.is_some() {
+            let (ca_path, key_path, cert_path) = (
+                self.ca_path.clone().unwrap(),
+                self.key_path.clone().unwrap(),
+                self.cert_path.clone().unwrap(),
+            );
+            let config = Config::default().with_security(ca_path, cert_path, key_path);
+            RawClient::new_with_config(self.endpoints.clone(), config)
+                .await
+                .map_err(parse_tikv_config_error)?
+        } else {
+            return Err(
+                Error::new(ErrorKind::ConfigInvalid, "invalid configuration")
+                    .with_context("service", Scheme::Tikv)
+                    .with_context("endpoints", format!("{:?}", self.endpoints)),
+            );
+        };
+        self.client.set(client.clone()).ok();
+        Ok(client)
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Tikv,
+            "TiKV",
+            Capability {
+                read: true,
+                write: true,
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let result = self
+            .get_connection()
+            .await?
+            .get(path.to_owned())
+            .await
+            .map_err(parse_tikv_error)?;
+        Ok(result.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        self.get_connection()
+            .await?
+            .put(path.to_owned(), value.to_vec())
+            .await
+            .map_err(parse_tikv_error)
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        self.get_connection()
+            .await?
+            .delete(path.to_owned())
+            .await
+            .map_err(parse_tikv_error)
+    }
+}
+
+fn parse_tikv_error(e: tikv_client::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from tikv").set_source(e)
+}
+
+fn parse_tikv_config_error(e: tikv_client::Error) -> Error {
+    Error::new(ErrorKind::ConfigInvalid, "invalid configuration")
+        .with_context("service", Scheme::Tikv)
+        .set_source(e)
+}

data/core/src/services/tikv/config.rs

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Tikv services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct TikvConfig {
+    /// network address of the TiKV service.
+    pub endpoints: Option<Vec<String>>,
+    /// whether using insecure connection to TiKV
+    pub insecure: bool,
+    /// certificate authority file path
+    pub ca_path: Option<String>,
+    /// cert path
+    pub cert_path: Option<String>,
+    /// key path
+    pub key_path: Option<String>,
+}
+
+impl Debug for TikvConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("TikvConfig");
+
+        d.field("endpoints", &self.endpoints)
+            .field("insecure", &self.insecure)
+            .field("ca_path", &self.ca_path)
+            .field("cert_path", &self.cert_path)
+            .field("key_path", &self.key_path)
+            .finish()
+    }
+}