opendal 0.1.6.pre.rc.1

data/core/src/services/redis/core.rs

@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::time::Duration;
+
+use bb8::RunError;
+use bytes::Bytes;
+use redis::aio::ConnectionLike;
+use redis::aio::ConnectionManager;
+use redis::cluster::ClusterClient;
+use redis::cluster_async::ClusterConnection;
+use redis::AsyncCommands;
+use redis::Client;
+use redis::Cmd;
+use redis::Pipeline;
+use redis::RedisError;
+use redis::RedisFuture;
+use redis::Value;
+use tokio::sync::OnceCell;
+
+use crate::*;
+
+#[derive(Clone)]
+pub enum RedisConnection {
+    Normal(ConnectionManager),
+    Cluster(ClusterConnection),
+}
+
+impl ConnectionLike for RedisConnection {
+    fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> {
+        match self {
+            RedisConnection::Normal(conn) => conn.req_packed_command(cmd),
+            RedisConnection::Cluster(conn) => conn.req_packed_command(cmd),
+        }
+    }
+
+    fn req_packed_commands<'a>(
+        &'a mut self,
+        cmd: &'a Pipeline,
+        offset: usize,
+        count: usize,
+    ) -> RedisFuture<'a, Vec<Value>> {
+        match self {
+            RedisConnection::Normal(conn) => conn.req_packed_commands(cmd, offset, count),
+            RedisConnection::Cluster(conn) => conn.req_packed_commands(cmd, offset, count),
+        }
+    }
+
+    fn get_db(&self) -> i64 {
+        match self {
+            RedisConnection::Normal(conn) => conn.get_db(),
+            RedisConnection::Cluster(conn) => conn.get_db(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RedisConnectionManager {
+    pub client: Option<Client>,
+    pub cluster_client: Option<ClusterClient>,
+}
+
+impl bb8::ManageConnection for RedisConnectionManager {
+    type Connection = RedisConnection;
+    type Error = Error;
+
+    async fn connect(&self) -> Result<RedisConnection, Self::Error> {
+        if let Some(client) = self.client.clone() {
+            ConnectionManager::new(client.clone())
+                .await
+                .map_err(format_redis_error)
+                .map(RedisConnection::Normal)
+        } else {
+            self.cluster_client
+                .clone()
+                .unwrap()
+                .get_async_connection()
+                .await
+                .map_err(format_redis_error)
+                .map(RedisConnection::Cluster)
+        }
+    }
+
+    async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> {
+        let pong: String = conn.ping().await.map_err(format_redis_error)?;
+
+        if pong == "PONG" {
+            Ok(())
+        } else {
+            Err(Error::new(ErrorKind::Unexpected, "PING ERROR"))
+        }
+    }
+
+    fn has_broken(&self, _: &mut Self::Connection) -> bool {
+        false
+    }
+}
+
+/// RedisCore holds the Redis connection and configuration
+#[derive(Clone)]
+pub struct RedisCore {
+    pub addr: String,
+    pub client: Option<Client>,
+    pub cluster_client: Option<ClusterClient>,
+    pub conn: OnceCell<bb8::Pool<RedisConnectionManager>>,
+    pub default_ttl: Option<Duration>,
+}
+
+impl Debug for RedisCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("RedisCore");
+        ds.field("addr", &self.addr);
+        ds.finish()
+    }
+}
+
+impl RedisCore {
+    pub async fn conn(&self) -> Result<bb8::PooledConnection<'_, RedisConnectionManager>> {
+        let pool = self
+            .conn
+            .get_or_try_init(|| async {
+                bb8::Pool::builder()
+                    .build(self.get_redis_connection_manager())
+                    .await
+                    .map_err(|err| {
+                        Error::new(ErrorKind::ConfigInvalid, "connect to redis failed")
+                            .set_source(err)
+                    })
+            })
+            .await?;
+        pool.get().await.map_err(|err| match err {
+            RunError::TimedOut => {
+                Error::new(ErrorKind::Unexpected, "get connection from pool failed").set_temporary()
+            }
+            RunError::User(err) => err,
+        })
+    }
+
+    pub fn get_redis_connection_manager(&self) -> RedisConnectionManager {
+        if let Some(_client) = self.client.clone() {
+            RedisConnectionManager {
+                client: self.client.clone(),
+                cluster_client: None,
+            }
+        } else {
+            RedisConnectionManager {
+                client: None,
+                cluster_client: self.cluster_client.clone(),
+            }
+        }
+    }
+
+    pub async fn get(&self, key: &str) -> Result<Option<Buffer>> {
+        let mut conn = self.conn().await?;
+        let result: Option<Bytes> = conn.get(key).await.map_err(format_redis_error)?;
+        Ok(result.map(Buffer::from))
+    }
+
+    pub async fn get_range(&self, key: &str, start: isize, end: isize) -> Result<Option<Buffer>> {
+        let mut conn = self.conn().await?;
+        let result: Option<Bytes> = conn
+            .getrange(key, start, end)
+            .await
+            .map_err(format_redis_error)?;
+        Ok(result.map(Buffer::from))
+    }
+
+    pub async fn set(&self, key: &str, value: Buffer) -> Result<()> {
+        let mut conn = self.conn().await?;
+        let value = value.to_vec();
+        if let Some(dur) = self.default_ttl {
+            let _: () = conn
+                .set_ex(key, value, dur.as_secs())
+                .await
+                .map_err(format_redis_error)?;
+        } else {
+            let _: () = conn.set(key, value).await.map_err(format_redis_error)?;
+        }
+        Ok(())
+    }
+
+    pub async fn delete(&self, key: &str) -> Result<()> {
+        let mut conn = self.conn().await?;
+        let _: () = conn.del(key).await.map_err(format_redis_error)?;
+        Ok(())
+    }
+}
+
+pub fn format_redis_error(e: RedisError) -> Error {
+    Error::new(ErrorKind::Unexpected, e.category())
+        .set_source(e)
+        .set_temporary()
+}

data/core/src/services/redis/delete.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::core::RedisCore;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct RedisDeleter {
+    core: std::sync::Arc<RedisCore>,
+    root: String,
+}
+
+impl RedisDeleter {
+    pub fn new(core: std::sync::Arc<RedisCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl oio::OneShotDelete for RedisDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = build_abs_path(&self.root, &path);
+        self.core.delete(&p).await?;
+        Ok(())
+    }
+}

data/core/src/services/redis/docs.md

@@ -0,0 +1,43 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] delete
+- [ ] ~~create_dir~~
+- [ ] ~~copy~~
+- [ ] ~~rename~~
+- [ ] ~~list~~
+- [ ] ~~presign~~
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `endpoint`: Set the network address of redis server
+- `cluster_endpoints`: Set the network address of redis cluster server. This parameter is mutually exclusive with the `endpoint` parameter.
+- `username`: Set the username of Redis
+- `password`: Set the password for authentication
+- `db`: Set the DB of redis
+
+You can refer to [`RedisBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Redis;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Redis::default();
+
+    // this will build a Operator accessing Redis which runs on tcp://localhost:6379
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/redis/mod.rs

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for redis service.
+#[cfg(feature = "services-redis")]
+pub(super) const DEFAULT_SCHEME: &str = "redis";
+#[cfg(feature = "services-redis")]
+mod backend;
+#[cfg(feature = "services-redis")]
+mod core;
+#[cfg(feature = "services-redis")]
+mod delete;
+#[cfg(feature = "services-redis")]
+mod writer;
+
+#[cfg(feature = "services-redis")]
+pub use backend::RedisBuilder as Redis;
+
+mod config;
+pub use config::RedisConfig;

data/core/src/services/redis/writer.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::core::RedisCore;
+use crate::raw::oio;
+use crate::*;
+
+pub struct RedisWriter {
+    core: std::sync::Arc<RedisCore>,
+    path: String,
+    buffer: oio::QueueBuf,
+}
+
+impl RedisWriter {
+    pub fn new(core: std::sync::Arc<RedisCore>, path: String) -> Self {
+        Self {
+            core,
+            path,
+            buffer: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for RedisWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buffer.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buffer.clone().collect();
+        let length = buf.len() as u64;
+        self.core.set(&self.path, buf).await?;
+
+        let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
+        Ok(meta)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buffer.clear();
+        Ok(())
+    }
+}

data/core/src/services/rocksdb/backend.rs

@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use rocksdb::DB;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::RocksdbConfig;
+use crate::Result;
+use crate::*;
+
+impl Configurator for RocksdbConfig {
+    type Builder = RocksdbBuilder;
+    fn into_builder(self) -> Self::Builder {
+        RocksdbBuilder { config: self }
+    }
+}
+
+/// RocksDB service support.
+#[doc = include_str!("docs.md")]
+#[derive(Clone, Default)]
+pub struct RocksdbBuilder {
+    config: RocksdbConfig,
+}
+
+impl RocksdbBuilder {
+    /// Set the path to the rocksdb data directory. Will create if not exists.
+    pub fn datadir(mut self, path: &str) -> Self {
+        self.config.datadir = Some(path.into());
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+}
+
+impl Builder for RocksdbBuilder {
+    type Config = RocksdbConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let path = self.config.datadir.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set")
+                .with_context("service", Scheme::Rocksdb)
+        })?;
+        let db = DB::open_default(&path).map_err(|e| {
+            Error::new(ErrorKind::ConfigInvalid, "open default transaction db")
+                .with_context("service", Scheme::Rocksdb)
+                .with_context("datadir", path)
+                .set_source(e)
+        })?;
+
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+
+        Ok(RocksdbBackend::new(Adapter { db: Arc::new(db) }).with_normalized_root(root))
+    }
+}
+
+/// Backend for rocksdb services.
+pub type RocksdbBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    db: Arc<DB>,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+        ds.field("path", &self.db.path());
+        ds.finish()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = kv::Scanner;
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Rocksdb,
+            &self.db.path().to_string_lossy(),
+            Capability {
+                read: true,
+                write: true,
+                list: true,
+                shared: false,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let result = self.db.get(path).map_err(parse_rocksdb_error)?;
+        Ok(result.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        self.db
+            .put(path, value.to_vec())
+            .map_err(parse_rocksdb_error)
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        self.db.delete(path).map_err(parse_rocksdb_error)
+    }
+
+    async fn scan(&self, path: &str) -> Result<Self::Scanner> {
+        let it = self.db.prefix_iterator(path).map(|r| r.map(|(k, _)| k));
+        let mut res = Vec::default();
+
+        for key in it {
+            let key = key.map_err(parse_rocksdb_error)?;
+            let key = String::from_utf8_lossy(&key);
+            if !key.starts_with(path) {
+                break;
+            }
+            res.push(key.to_string());
+        }
+
+        Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
+    }
+}
+
+fn parse_rocksdb_error(e: rocksdb::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "got rocksdb error").set_source(e)
+}

data/core/src/services/rocksdb/config.rs

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Rocksdb Service.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct RocksdbConfig {
+    /// The path to the rocksdb data directory.
+    pub datadir: Option<String>,
+    /// the working directory of the service. Can be "/path/to/dir"
+    ///
+    /// default is "/"
+    pub root: Option<String>,
+}

data/core/src/services/rocksdb/docs.md

@@ -0,0 +1,54 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [x] blocking
+
+## Note
+
+OpenDAL will build rocksdb from source by default.
+
+To link with existing rocksdb lib, please set one of the following:
+
+- `ROCKSDB_LIB_DIR` to the dir that contains `librocksdb.so`
+- `ROCKSDB_STATIC` to the dir that contains `librocksdb.a`
+
+If the version of RocksDB is below 6.0, you may encounter compatibility
+issues. It is advisable to follow the steps provided in the [`INSTALL`](https://github.com/facebook/rocksdb/blob/main/INSTALL.md)
+file to build rocksdb, rather than relying on system libraries that
+may be outdated and incompatible.
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `datadir`: Set the path to the rocksdb data directory
+
+You can refer to [`RocksdbBuilder`]'s docs for more information.
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Rocksdb;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Rocksdb::default()
+        .datadir("/tmp/opendal/rocksdb");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/rocksdb/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-rocksdb")]
+mod backend;
+#[cfg(feature = "services-rocksdb")]
+pub use backend::RocksdbBuilder as Rocksdb;
+
+mod config;
+pub use config::RocksdbConfig;