opendal 0.1.6.pre.rc.1

data/core/src/services/etcd/core.rs

@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use bb8::PooledConnection;
+use bb8::RunError;
+use etcd_client::Client;
+use etcd_client::ConnectOptions;
+use tokio::sync::OnceCell;
+
+use crate::services::etcd::error::format_etcd_error;
+use crate::{Buffer, Error, ErrorKind, Result};
+
+pub mod constants {
+    pub const DEFAULT_ETCD_ENDPOINTS: &str = "http://127.0.0.1:2379";
+}
+
+#[derive(Clone)]
+pub struct Manager {
+    endpoints: Vec<String>,
+    options: ConnectOptions,
+}
+
+impl bb8::ManageConnection for Manager {
+    type Connection = Client;
+    type Error = Error;
+
+    async fn connect(&self) -> Result<Self::Connection, Self::Error> {
+        let conn = Client::connect(self.endpoints.clone(), Some(self.options.clone()))
+            .await
+            .map_err(format_etcd_error)?;
+
+        Ok(conn)
+    }
+
+    async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> {
+        let _ = conn.status().await.map_err(format_etcd_error)?;
+        Ok(())
+    }
+
+    /// Always allow reuse conn.
+    fn has_broken(&self, _: &mut Self::Connection) -> bool {
+        false
+    }
+}
+
+#[derive(Clone)]
+pub struct EtcdCore {
+    pub endpoints: Vec<String>,
+    pub options: ConnectOptions,
+    pub client: OnceCell<bb8::Pool<Manager>>,
+}
+
+impl Debug for EtcdCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("EtcdCore")
+            .field("endpoints", &self.endpoints.join(","))
+            .field("options", &self.options.clone())
+            .finish()
+    }
+}
+
+impl EtcdCore {
+    pub async fn conn(&self) -> Result<PooledConnection<'static, Manager>> {
+        let client = self
+            .client
+            .get_or_try_init(|| async {
+                bb8::Pool::builder()
+                    .max_size(64)
+                    .build(Manager {
+                        endpoints: self.endpoints.clone(),
+                        options: self.options.clone(),
+                    })
+                    .await
+            })
+            .await?;
+
+        client.get_owned().await.map_err(|err| match err {
+            RunError::User(err) => err,
+            RunError::TimedOut => {
+                Error::new(ErrorKind::Unexpected, "connection request: timeout").set_temporary()
+            }
+        })
+    }
+
+    pub async fn has_prefix(&self, prefix: &str) -> Result<bool> {
+        let mut client = self.conn().await?;
+        let get_options = Some(
+            etcd_client::GetOptions::new()
+                .with_prefix()
+                .with_keys_only()
+                .with_limit(1),
+        );
+        let resp = client
+            .get(prefix, get_options)
+            .await
+            .map_err(format_etcd_error)?;
+        Ok(!resp.kvs().is_empty())
+    }
+}
+
+impl EtcdCore {
+    pub async fn get(&self, key: &str) -> Result<Option<Buffer>> {
+        let mut client = self.conn().await?;
+        let resp = client.get(key, None).await.map_err(format_etcd_error)?;
+        if let Some(kv) = resp.kvs().first() {
+            Ok(Some(Buffer::from(kv.value().to_vec())))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub async fn set(&self, key: &str, value: Buffer) -> Result<()> {
+        let mut client = self.conn().await?;
+        let _ = client
+            .put(key, value.to_vec(), None)
+            .await
+            .map_err(format_etcd_error)?;
+        Ok(())
+    }
+
+    pub async fn delete(&self, key: &str) -> Result<()> {
+        let mut client = self.conn().await?;
+        let _ = client.delete(key, None).await.map_err(format_etcd_error)?;
+        Ok(())
+    }
+}

data/core/src/services/etcd/deleter.rs

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::EtcdCore;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct EtcdDeleter {
+    core: Arc<EtcdCore>,
+    root: String,
+}
+
+impl EtcdDeleter {
+    pub fn new(core: Arc<EtcdCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl oio::OneShotDelete for EtcdDeleter {
+    async fn delete_once(&self, path: String, _args: OpDelete) -> Result<()> {
+        let abs_path = build_abs_path(&self.root, &path);
+        self.core.delete(&abs_path).await
+    }
+}

data/core/src/services/etcd/docs.md

@@ -0,0 +1,45 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `endpoints`: Set the network address of etcd servers
+- `username`: Set the username of Etcd
+- `password`: Set the password for authentication
+- `ca_path`: Set the ca path to the etcd connection
+- `cert_path`: Set the cert path to the etcd connection
+- `key_path`: Set the key path to the etcd connection
+
+You can refer to [`EtcdBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Etcd;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Etcd::default();
+
+    // this will build a Operator accessing etcd which runs on http://127.0.0.1:2379
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/etcd/error.rs

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use etcd_client::Error as EtcdError;
+
+use crate::{Error, ErrorKind};
+
+pub fn format_etcd_error(e: EtcdError) -> Error {
+    Error::new(ErrorKind::Unexpected, e.to_string().as_str())
+        .set_source(e)
+        .set_temporary()
+}

data/core/src/services/etcd/lister.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::vec::IntoIter;
+
+use super::core::EtcdCore;
+use crate::raw::oio::Entry;
+use crate::raw::{build_abs_path, build_rel_path, oio};
+use crate::*;
+
+pub struct EtcdLister {
+    root: String,
+    path: String,
+    iter: IntoIter<String>,
+}
+
+impl EtcdLister {
+    pub async fn new(core: Arc<EtcdCore>, root: String, path: String) -> Result<Self> {
+        let abs_path = build_abs_path(&root, &path);
+
+        // Get all keys with the specified prefix
+        let mut client = core.conn().await?;
+        let get_options = Some(
+            etcd_client::GetOptions::new()
+                .with_prefix()
+                .with_keys_only(),
+        );
+        let resp = client
+            .get(abs_path.as_str(), get_options)
+            .await
+            .map_err(super::error::format_etcd_error)?;
+
+        // Collect all keys that match the prefix
+        let mut keys = Vec::new();
+        for kv in resp.kvs() {
+            let key = kv.key_str().map(String::from).map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "store key is not valid utf-8 string")
+                    .set_source(err)
+            })?;
+            keys.push(key);
+        }
+
+        Ok(Self {
+            root,
+            path: abs_path,
+            iter: keys.into_iter(),
+        })
+    }
+}
+
+impl oio::List for EtcdLister {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        for key in self.iter.by_ref() {
+            if key.starts_with(&self.path) {
+                let path = build_rel_path(&self.root, &key);
+
+                let entry = Entry::new(&path, Metadata::new(EntryMode::from_path(&key)));
+                return Ok(Some(entry));
+            }
+        }
+
+        Ok(None)
+    }
+}

data/core/src/services/etcd/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-etcd")]
+pub(super) const DEFAULT_SCHEME: &str = "etcd";
+#[cfg(feature = "services-etcd")]
+mod backend;
+#[cfg(feature = "services-etcd")]
+mod core;
+#[cfg(feature = "services-etcd")]
+mod deleter;
+#[cfg(feature = "services-etcd")]
+mod error;
+#[cfg(feature = "services-etcd")]
+mod lister;
+#[cfg(feature = "services-etcd")]
+mod writer;
+#[cfg(feature = "services-etcd")]
+pub use backend::EtcdBuilder as Etcd;
+
+mod config;
+pub use config::EtcdConfig;

data/core/src/services/etcd/writer.rs

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::EtcdCore;
+use crate::raw::oio;
+use crate::*;
+
+pub struct EtcdWriter {
+    core: Arc<EtcdCore>,
+    path: String,
+    buffer: oio::QueueBuf,
+}
+
+impl EtcdWriter {
+    pub fn new(core: Arc<EtcdCore>, path: String) -> Self {
+        Self {
+            core,
+            path,
+            buffer: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for EtcdWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buffer.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buffer.clone().collect();
+
+        self.core.set(&self.path, buf.clone()).await?;
+
+        let mut metadata = Metadata::new(EntryMode::from_path(&self.path));
+        metadata.set_content_length(buf.len() as u64);
+
+        Ok(metadata)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buffer.clear();
+        Ok(())
+    }
+}

data/core/src/services/foundationdb/backend.rs

@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use foundationdb::api::NetworkAutoStop;
+use foundationdb::Database;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::FoundationdbConfig;
+use crate::Builder;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Scheme;
+use crate::*;
+
+impl Configurator for FoundationdbConfig {
+    type Builder = FoundationdbBuilder;
+    fn into_builder(self) -> Self::Builder {
+        FoundationdbBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct FoundationdbBuilder {
+    config: FoundationdbConfig,
+}
+
+impl FoundationdbBuilder {
+    /// Set the root for Foundationdb.
+    pub fn root(mut self, path: &str) -> Self {
+        self.config.root = Some(path.into());
+        self
+    }
+
+    /// Set the config path for Foundationdb. If not set, will fallback to use default
+    pub fn config_path(mut self, path: &str) -> Self {
+        self.config.config_path = Some(path.into());
+        self
+    }
+}
+
+impl Builder for FoundationdbBuilder {
+    type Config = FoundationdbConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let _network = Arc::new(unsafe { foundationdb::boot() });
+        let db;
+        if let Some(cfg_path) = &self.config.config_path {
+            db = Database::from_path(cfg_path).map_err(|e| {
+                Error::new(ErrorKind::ConfigInvalid, "open foundation db")
+                    .with_context("service", Scheme::Foundationdb)
+                    .set_source(e)
+            })?;
+        } else {
+            db = Database::default().map_err(|e| {
+                Error::new(ErrorKind::ConfigInvalid, "open foundation db")
+                    .with_context("service", Scheme::Foundationdb)
+                    .set_source(e)
+            })?
+        }
+
+        let db = Arc::new(db);
+
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+
+        Ok(FoundationdbBackend::new(Adapter { db, _network }).with_normalized_root(root))
+    }
+}
+
+/// Backend for Foundationdb services.
+pub type FoundationdbBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    db: Arc<Database>,
+    _network: Arc<NetworkAutoStop>,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+        ds.finish()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Foundationdb,
+            "foundationdb",
+            Capability {
+                read: true,
+                write: true,
+                delete: true,
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let transaction = self.db.create_trx().expect("Unable to create transaction");
+
+        match transaction.get(path.as_bytes(), false).await {
+            Ok(slice) => match slice {
+                Some(data) => Ok(Some(Buffer::from(data.to_vec()))),
+                None => Err(Error::new(
+                    ErrorKind::NotFound,
+                    "foundationdb: key not found",
+                )),
+            },
+            Err(_) => Err(Error::new(
+                ErrorKind::NotFound,
+                "foundationdb: key not found",
+            )),
+        }
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let transaction = self.db.create_trx().expect("Unable to create transaction");
+
+        transaction.set(path.as_bytes(), &value.to_vec());
+
+        match transaction.commit().await {
+            Ok(_) => Ok(()),
+            Err(e) => Err(parse_transaction_commit_error(e)),
+        }
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let transaction = self.db.create_trx().expect("Unable to create transaction");
+        transaction.clear(path.as_bytes());
+
+        match transaction.commit().await {
+            Ok(_) => Ok(()),
+            Err(e) => Err(parse_transaction_commit_error(e)),
+        }
+    }
+}
+
+fn parse_transaction_commit_error(e: foundationdb::TransactionCommitError) -> Error {
+    Error::new(ErrorKind::Unexpected, e.to_string().as_str())
+        .with_context("service", Scheme::Foundationdb)
+}

data/core/src/services/foundationdb/config.rs

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// [foundationdb](https://www.foundationdb.org/) service support.
+///Config for FoundationDB.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct FoundationdbConfig {
+    ///root of the backend.
+    pub root: Option<String>,
+    ///config_path for the backend.
+    pub config_path: Option<String>,
+}
+
+impl Debug for FoundationdbConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("FoundationConfig");
+
+        ds.field("root", &self.root);
+        ds.field("config_path", &self.config_path);
+
+        ds.finish()
+    }
+}

data/core/src/services/foundationdb/docs.md

@@ -0,0 +1,42 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+**Note**: As for [Known Limitations - FoundationDB](https://apple.github.io/foundationdb/known-limitations), keys cannot exceed 10,000 bytes in size, and values cannot exceed 100,000 bytes in size. Errors will be raised by OpenDAL if these limits are exceeded.
+
+## Configuration
+
+- `root`: Set the work directory for this backend.
+- `config_path`: Set the configuration path for foundationdb. If not provided, the default configuration path will be used.
+
+You can refer to [`FoundationdbBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Foundationdb;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Foundationdb::default()
+        .config_path("/etc/foundationdb/foundationdb.conf");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```