opendal 0.1.6.pre.rc.1

data/core/src/services/dropbox/lister.rs

@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DropboxLister {
+    core: Arc<DropboxCore>,
+    path: String,
+    recursive: bool,
+    limit: Option<usize>,
+}
+
+impl DropboxLister {
+    pub fn new(
+        core: Arc<DropboxCore>,
+        path: String,
+        recursive: bool,
+        limit: Option<usize>,
+    ) -> Self {
+        Self {
+            core,
+            path,
+            recursive,
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for DropboxLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        // The token is set when obtaining entries and returning `has_more` flag.
+        // When the token exists, we should retrieve more entries using the Dropbox continue API.
+        // Refer: https://www.dropbox.com/developers/documentation/http/documentation#files-list_folder-continue
+        let response = if !ctx.token.is_empty() {
+            self.core.dropbox_list_continue(&ctx.token).await?
+        } else {
+            self.core
+                .dropbox_list(&self.path, self.recursive, self.limit)
+                .await?
+        };
+
+        let status_code = response.status();
+
+        if !status_code.is_success() {
+            let error = parse_error(response);
+
+            let result = match error.kind() {
+                ErrorKind::NotFound => Ok(()),
+                _ => Err(error),
+            };
+
+            ctx.done = true;
+            return result;
+        }
+
+        let bytes = response.into_body();
+        let decoded_response: DropboxListResponse =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        for entry in decoded_response.entries {
+            let entry_mode = match entry.tag.as_str() {
+                "file" => EntryMode::FILE,
+                "folder" => EntryMode::DIR,
+                _ => EntryMode::Unknown,
+            };
+
+            let mut name = entry.name;
+            let mut meta = Metadata::new(entry_mode);
+
+            // Dropbox will return folder names that do not end with '/'.
+            if entry_mode == EntryMode::DIR && !name.ends_with('/') {
+                name.push('/');
+            }
+
+            // The behavior here aligns with Dropbox's stat function.
+            if entry_mode == EntryMode::FILE {
+                let date_utc_last_modified = parse_datetime_from_rfc3339(&entry.client_modified)?;
+                meta.set_last_modified(date_utc_last_modified);
+
+                if let Some(size) = entry.size {
+                    meta.set_content_length(size);
+                }
+            }
+
+            ctx.entries.push_back(oio::Entry::with(name, meta));
+        }
+
+        if decoded_response.has_more {
+            ctx.token = decoded_response.cursor;
+            ctx.done = false;
+        } else {
+            ctx.done = true;
+        }
+        Ok(())
+    }
+}

data/core/src/services/dropbox/mod.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for dropbox service.
+#[cfg(feature = "services-dropbox")]
+pub(super) const DEFAULT_SCHEME: &str = "dropbox";
+#[cfg(feature = "services-dropbox")]
+mod backend;
+#[cfg(feature = "services-dropbox")]
+mod core;
+#[cfg(feature = "services-dropbox")]
+mod delete;
+#[cfg(feature = "services-dropbox")]
+mod error;
+#[cfg(feature = "services-dropbox")]
+mod lister;
+#[cfg(feature = "services-dropbox")]
+mod writer;
+
+#[cfg(feature = "services-dropbox")]
+mod builder;
+#[cfg(feature = "services-dropbox")]
+pub use builder::DropboxBuilder as Dropbox;
+
+mod config;
+pub use config::DropboxConfig;

data/core/src/services/dropbox/writer.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::DropboxCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DropboxWriter {
+    core: Arc<DropboxCore>,
+    op: OpWrite,
+    path: String,
+}
+
+impl DropboxWriter {
+    pub fn new(core: Arc<DropboxCore>, op: OpWrite, path: String) -> Self {
+        DropboxWriter { core, op, path }
+    }
+}
+
+impl oio::OneShotWrite for DropboxWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .dropbox_update(&self.path, Some(bs.len()), &self.op, bs)
+            .await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/etcd/backend.rs

@@ -0,0 +1,345 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use etcd_client::Certificate;
+use etcd_client::ConnectOptions;
+use etcd_client::Identity;
+use etcd_client::TlsOptions;
+use tokio::sync::OnceCell;
+
+use super::core::constants::DEFAULT_ETCD_ENDPOINTS;
+use super::core::EtcdCore;
+use super::deleter::EtcdDeleter;
+use super::lister::EtcdLister;
+use super::writer::EtcdWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::EtcdConfig;
+use crate::*;
+
+impl Configurator for EtcdConfig {
+    type Builder = EtcdBuilder;
+    fn into_builder(self) -> Self::Builder {
+        EtcdBuilder { config: self }
+    }
+}
+
+/// [Etcd](https://etcd.io/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Clone, Default)]
+pub struct EtcdBuilder {
+    config: EtcdConfig,
+}
+
+impl Debug for EtcdBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Builder");
+
+        ds.field("config", &self.config);
+        ds.finish()
+    }
+}
+
+impl EtcdBuilder {
+    /// set the network address of etcd service.
+    ///
+    /// default: "http://127.0.0.1:2379"
+    pub fn endpoints(mut self, endpoints: &str) -> Self {
+        if !endpoints.is_empty() {
+            self.config.endpoints = Some(endpoints.to_owned());
+        }
+        self
+    }
+
+    /// set the username for etcd
+    ///
+    /// default: no username
+    pub fn username(mut self, username: &str) -> Self {
+        if !username.is_empty() {
+            self.config.username = Some(username.to_owned());
+        }
+        self
+    }
+
+    /// set the password for etcd
+    ///
+    /// default: no password
+    pub fn password(mut self, password: &str) -> Self {
+        if !password.is_empty() {
+            self.config.password = Some(password.to_owned());
+        }
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the certificate authority file path.
+    ///
+    /// default is None
+    pub fn ca_path(mut self, ca_path: &str) -> Self {
+        if !ca_path.is_empty() {
+            self.config.ca_path = Some(ca_path.to_string())
+        }
+        self
+    }
+
+    /// Set the certificate file path.
+    ///
+    /// default is None
+    pub fn cert_path(mut self, cert_path: &str) -> Self {
+        if !cert_path.is_empty() {
+            self.config.cert_path = Some(cert_path.to_string())
+        }
+        self
+    }
+
+    /// Set the key file path.
+    ///
+    /// default is None
+    pub fn key_path(mut self, key_path: &str) -> Self {
+        if !key_path.is_empty() {
+            self.config.key_path = Some(key_path.to_string())
+        }
+        self
+    }
+}
+
+impl Builder for EtcdBuilder {
+    type Config = EtcdConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let endpoints = self
+            .config
+            .endpoints
+            .clone()
+            .unwrap_or_else(|| DEFAULT_ETCD_ENDPOINTS.to_string());
+
+        let endpoints: Vec<String> = endpoints.split(',').map(|s| s.to_string()).collect();
+
+        let mut options = ConnectOptions::new();
+
+        if self.config.ca_path.is_some()
+            && self.config.cert_path.is_some()
+            && self.config.key_path.is_some()
+        {
+            let ca = self.load_pem(self.config.ca_path.clone().unwrap().as_str())?;
+            let key = self.load_pem(self.config.key_path.clone().unwrap().as_str())?;
+            let cert = self.load_pem(self.config.cert_path.clone().unwrap().as_str())?;
+
+            let tls_options = TlsOptions::default()
+                .ca_certificate(Certificate::from_pem(ca))
+                .identity(Identity::from_pem(cert, key));
+            options = options.with_tls(tls_options);
+        }
+
+        if let Some(username) = self.config.username.clone() {
+            options = options.with_user(
+                username,
+                self.config.password.clone().unwrap_or("".to_string()),
+            );
+        }
+
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+
+        let client = OnceCell::new();
+
+        let core = EtcdCore {
+            endpoints,
+            client,
+            options,
+        };
+
+        Ok(EtcdAccessor::new(core, &root))
+    }
+}
+
+impl EtcdBuilder {
+    fn load_pem(&self, path: &str) -> Result<String> {
+        std::fs::read_to_string(path)
+            .map_err(|err| Error::new(ErrorKind::Unexpected, "invalid file path").set_source(err))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EtcdAccessor {
+    core: Arc<EtcdCore>,
+    info: Arc<AccessorInfo>,
+}
+
+impl EtcdAccessor {
+    fn new(core: EtcdCore, root: &str) -> Self {
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME);
+        info.set_name("etcd");
+        info.set_root(root);
+        info.set_native_capability(Capability {
+            read: true,
+
+            write: true,
+            write_can_empty: true,
+
+            delete: true,
+            stat: true,
+            list: true,
+
+            shared: true,
+
+            ..Default::default()
+        });
+
+        Self {
+            core: Arc::new(core),
+            info: Arc::new(info),
+        }
+    }
+}
+
+impl Access for EtcdAccessor {
+    type Reader = Buffer;
+    type Writer = EtcdWriter;
+    type Lister = oio::HierarchyLister<EtcdLister>;
+    type Deleter = oio::OneShotDeleter<EtcdDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+        let abs_path = build_abs_path(&self.info.root(), path);
+
+        // In etcd, we simulate directory creation by storing an empty value
+        // with the directory path (ensuring it ends with '/')
+        let dir_path = if abs_path.ends_with('/') {
+            abs_path
+        } else {
+            format!("{abs_path}/")
+        };
+
+        // Store an empty buffer to represent the directory
+        self.core.set(&dir_path, Buffer::new()).await?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let abs_path = build_abs_path(&self.info.root(), path);
+
+        // First check if it's a direct key
+        match self.core.get(&abs_path).await? {
+            Some(buffer) => {
+                let mut metadata = Metadata::new(EntryMode::from_path(&abs_path));
+                metadata.set_content_length(buffer.len() as u64);
+                Ok(RpStat::new(metadata))
+            }
+            None => {
+                // Check if it's a directory by looking for keys with this prefix
+                let prefix = if abs_path.ends_with('/') {
+                    abs_path
+                } else {
+                    format!("{abs_path}/")
+                };
+
+                // Use etcd prefix query to check if any keys exist with this prefix
+                let has_children = self.core.has_prefix(&prefix).await?;
+                if has_children {
+                    // Has children, it's a directory
+                    let metadata = Metadata::new(EntryMode::DIR);
+                    Ok(RpStat::new(metadata))
+                } else {
+                    Err(Error::new(ErrorKind::NotFound, "path not found"))
+                }
+            }
+        }
+    }
+
+    async fn read(&self, path: &str, op: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let abs_path = build_abs_path(&self.info.root(), path);
+
+        match self.core.get(&abs_path).await? {
+            Some(buffer) => {
+                let range = op.range();
+
+                // If range is full, return the buffer directly
+                if range.is_full() {
+                    return Ok((RpRead::new(), buffer));
+                }
+
+                // Handle range requests
+                let offset = range.offset() as usize;
+                if offset >= buffer.len() {
+                    return Err(Error::new(
+                        ErrorKind::RangeNotSatisfied,
+                        "range start offset exceeds content length",
+                    ));
+                }
+
+                let size = range.size().map(|s| s as usize);
+                let end = size.map_or(buffer.len(), |s| (offset + s).min(buffer.len()));
+                let sliced_buffer = buffer.slice(offset..end);
+
+                Ok((RpRead::new(), sliced_buffer))
+            }
+            None => Err(Error::new(ErrorKind::NotFound, "path not found")),
+        }
+    }
+
+    async fn write(&self, path: &str, _op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let abs_path = build_abs_path(&self.info.root(), path);
+        let writer = EtcdWriter::new(self.core.clone(), abs_path);
+        Ok((RpWrite::new(), writer))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        let deleter = oio::OneShotDeleter::new(EtcdDeleter::new(
+            self.core.clone(),
+            self.info.root().to_string(),
+        ));
+        Ok((RpDelete::default(), deleter))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let lister = EtcdLister::new(
+            self.core.clone(),
+            self.info.root().to_string(),
+            path.to_string(),
+        )
+        .await?;
+        let lister = oio::HierarchyLister::new(lister, path, args.recursive());
+        Ok((RpList::default(), lister))
+    }
+}

data/core/src/services/etcd/config.rs

@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Etcd services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct EtcdConfig {
+    /// network address of the Etcd services.
+    /// If use https, must set TLS options: `ca_path`, `cert_path`, `key_path`.
+    /// e.g. "127.0.0.1:23790,127.0.0.1:23791,127.0.0.1:23792" or "http://127.0.0.1:23790,http://127.0.0.1:23791,http://127.0.0.1:23792" or "https://127.0.0.1:23790,https://127.0.0.1:23791,https://127.0.0.1:23792"
+    ///
+    /// default is "http://127.0.0.1:2379"
+    pub endpoints: Option<String>,
+    /// the username to connect etcd service.
+    ///
+    /// default is None
+    pub username: Option<String>,
+    /// the password for authentication
+    ///
+    /// default is None
+    pub password: Option<String>,
+    /// the working directory of the etcd service. Can be "/path/to/dir"
+    ///
+    /// default is "/"
+    pub root: Option<String>,
+    /// certificate authority file path
+    ///
+    /// default is None
+    pub ca_path: Option<String>,
+    /// cert path
+    ///
+    /// default is None
+    pub cert_path: Option<String>,
+    /// key path
+    ///
+    /// default is None
+    pub key_path: Option<String>,
+}
+
+impl Debug for EtcdConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("EtcdConfig");
+
+        ds.field("root", &self.root);
+        if let Some(endpoints) = self.endpoints.clone() {
+            ds.field("endpoints", &endpoints);
+        }
+        if let Some(username) = self.username.clone() {
+            ds.field("username", &username);
+        }
+        if self.password.is_some() {
+            ds.field("password", &"<redacted>");
+        }
+        if let Some(ca_path) = self.ca_path.clone() {
+            ds.field("ca_path", &ca_path);
+        }
+        if let Some(cert_path) = self.cert_path.clone() {
+            ds.field("cert_path", &cert_path);
+        }
+        if let Some(key_path) = self.key_path.clone() {
+            ds.field("key_path", &key_path);
+        }
+        ds.finish()
+    }
+}