opendal 0.1.6.pre.rc.1

data/core/src/services/moka/docs.md

@@ -0,0 +1,42 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+- `name`: Set the name for this cache instance.
+- `max_capacity`: Set the max capacity of the cache.
+- `time_to_live`: Set the time to live of the cache.
+- `time_to_idle`: Set the time to idle of the cache.
+
+You can refer to [`MokaBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Moka;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Moka::default()
+        .name("opendal");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/moka/lister.rs

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::vec::IntoIter;
+
+use super::core::MokaCore;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MokaLister {
+    root: String,
+    keys: IntoIter<String>,
+}
+
+impl MokaLister {
+    pub fn new(core: Arc<MokaCore>, root: String, _path: String) -> Self {
+        // Get all keys from the cache
+        let keys: Vec<String> = core.cache.iter().map(|kv| kv.0.to_string()).collect();
+
+        Self {
+            root,
+            keys: keys.into_iter(),
+        }
+    }
+}
+
+impl oio::List for MokaLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        match self.keys.next() {
+            Some(key) => {
+                // Convert absolute path to relative path
+                let mut path = build_rel_path(&self.root, &key);
+                if path.is_empty() {
+                    path = "/".to_string();
+                }
+
+                // Determine if it's a file or directory based on trailing slash
+                let mode = if key.ends_with('/') {
+                    EntryMode::DIR
+                } else {
+                    EntryMode::FILE
+                };
+
+                Ok(Some(oio::Entry::new(&path, Metadata::new(mode))))
+            }
+            None => Ok(None),
+        }
+    }
+}

data/core/src/services/moka/mod.rs

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for moka service.
+#[cfg(feature = "services-moka")]
+pub(super) const DEFAULT_SCHEME: &str = "moka";
+#[cfg(feature = "services-moka")]
+mod backend;
+#[cfg(feature = "services-moka")]
+mod core;
+#[cfg(feature = "services-moka")]
+mod delete;
+#[cfg(feature = "services-moka")]
+mod lister;
+#[cfg(feature = "services-moka")]
+mod writer;
+
+#[cfg(feature = "services-moka")]
+pub use backend::MokaBuilder as Moka;
+#[cfg(feature = "services-moka")]
+pub use backend::MokaCacheBuilder;
+
+#[cfg(feature = "services-moka")]
+pub use core::MokaValue;
+
+mod config;
+pub use config::MokaConfig;

data/core/src/services/moka/writer.rs

@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::{MokaCore, MokaValue};
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MokaWriter {
+    core: Arc<MokaCore>,
+    path: String,
+    op: OpWrite,
+    buffer: oio::QueueBuf,
+}
+
+impl MokaWriter {
+    pub fn new(core: Arc<MokaCore>, path: String, op: OpWrite) -> Self {
+        Self {
+            core,
+            path,
+            op,
+            buffer: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for MokaWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buffer.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buffer.clone().collect();
+        let length = buf.len() as u64;
+
+        // Build metadata with write options
+        let mut metadata =
+            Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
+
+        if let Some(content_type) = self.op.content_type() {
+            metadata.set_content_type(content_type);
+        }
+        if let Some(content_disposition) = self.op.content_disposition() {
+            metadata.set_content_disposition(content_disposition);
+        }
+        if let Some(cache_control) = self.op.cache_control() {
+            metadata.set_cache_control(cache_control);
+        }
+        if let Some(content_encoding) = self.op.content_encoding() {
+            metadata.set_content_encoding(content_encoding);
+        }
+
+        let value = MokaValue {
+            metadata: metadata.clone(),
+            content: buf,
+        };
+
+        self.core.set(&self.path, value).await?;
+        Ok(metadata)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buffer.clear();
+        Ok(())
+    }
+}

data/core/src/services/mongodb/backend.rs

@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use mongodb::bson::doc;
+use mongodb::bson::Binary;
+use mongodb::bson::Document;
+use mongodb::options::ClientOptions;
+use tokio::sync::OnceCell;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::MongodbConfig;
+use crate::*;
+
+impl Configurator for MongodbConfig {
+    type Builder = MongodbBuilder;
+    fn into_builder(self) -> Self::Builder {
+        MongodbBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct MongodbBuilder {
+    config: MongodbConfig,
+}
+
+impl Debug for MongodbBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MongodbBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl MongodbBuilder {
+    /// Set the connection_string of the MongoDB service.
+    ///
+    /// This connection string is used to connect to the MongoDB service. It typically follows the format:
+    ///
+    /// ## Format
+    ///
+    /// `mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]`
+    ///
+    /// Examples:
+    ///
+    /// - Connecting to a local MongoDB instance: `mongodb://localhost:27017`
+    /// - Using authentication: `mongodb://myUser:myPassword@localhost:27017/myAuthDB`
+    /// - Specifying authentication mechanism: `mongodb://myUser:myPassword@localhost:27017/myAuthDB?authMechanism=SCRAM-SHA-256`
+    ///
+    /// ## Options
+    ///
+    /// - `authMechanism`: Specifies the authentication method to use. Examples include `SCRAM-SHA-1`, `SCRAM-SHA-256`, and `MONGODB-AWS`.
+    /// - ... (any other options you wish to highlight)
+    ///
+    /// For more information, please refer to [MongoDB Connection String URI Format](https://docs.mongodb.com/manual/reference/connection-string/).
+    pub fn connection_string(mut self, v: &str) -> Self {
+        if !v.is_empty() {
+            self.config.connection_string = Some(v.to_string());
+        }
+        self
+    }
+    /// Set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the database name of the MongoDB service to read/write.
+    pub fn database(mut self, database: &str) -> Self {
+        if !database.is_empty() {
+            self.config.database = Some(database.to_string());
+        }
+        self
+    }
+
+    /// Set the collection name of the MongoDB service to read/write.
+    pub fn collection(mut self, collection: &str) -> Self {
+        if !collection.is_empty() {
+            self.config.collection = Some(collection.to_string());
+        }
+        self
+    }
+
+    /// Set the key field name of the MongoDB service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub fn key_field(mut self, key_field: &str) -> Self {
+        if !key_field.is_empty() {
+            self.config.key_field = Some(key_field.to_string());
+        }
+        self
+    }
+
+    /// Set the value field name of the MongoDB service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub fn value_field(mut self, value_field: &str) -> Self {
+        if !value_field.is_empty() {
+            self.config.value_field = Some(value_field.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for MongodbBuilder {
+    type Config = MongodbConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let conn = match &self.config.connection_string.clone() {
+            Some(v) => v.clone(),
+            None => {
+                return Err(
+                    Error::new(ErrorKind::ConfigInvalid, "connection_string is required")
+                        .with_context("service", Scheme::Mongodb),
+                )
+            }
+        };
+        let database = match &self.config.database.clone() {
+            Some(v) => v.clone(),
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "database is required")
+                    .with_context("service", Scheme::Mongodb))
+            }
+        };
+        let collection = match &self.config.collection.clone() {
+            Some(v) => v.clone(),
+            None => {
+                return Err(
+                    Error::new(ErrorKind::ConfigInvalid, "collection is required")
+                        .with_context("service", Scheme::Mongodb),
+                )
+            }
+        };
+        let key_field = match &self.config.key_field.clone() {
+            Some(v) => v.clone(),
+            None => "key".to_string(),
+        };
+        let value_field = match &self.config.value_field.clone() {
+            Some(v) => v.clone(),
+            None => "value".to_string(),
+        };
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+        Ok(MongodbBackend::new(Adapter {
+            connection_string: conn,
+            database,
+            collection,
+            collection_instance: OnceCell::new(),
+            key_field,
+            value_field,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+pub type MongodbBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    connection_string: String,
+    database: String,
+    collection: String,
+    collection_instance: OnceCell<mongodb::Collection<Document>>,
+    key_field: String,
+    value_field: String,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Adapter")
+            .field("connection_string", &self.connection_string)
+            .field("database", &self.database)
+            .field("collection", &self.collection)
+            .field("key_field", &self.key_field)
+            .field("value_field", &self.value_field)
+            .finish()
+    }
+}
+
+impl Adapter {
+    async fn get_collection(&self) -> Result<&mongodb::Collection<Document>> {
+        self.collection_instance
+            .get_or_try_init(|| async {
+                let client_options = ClientOptions::parse(&self.connection_string)
+                    .await
+                    .map_err(parse_mongodb_error)?;
+                let client =
+                    mongodb::Client::with_options(client_options).map_err(parse_mongodb_error)?;
+                let database = client.database(&self.database);
+                let collection = database.collection(&self.collection);
+                Ok(collection)
+            })
+            .await
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Mongodb,
+            &format!("{}/{}", self.database, self.collection),
+            Capability {
+                read: true,
+                write: true,
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let collection = self.get_collection().await?;
+        let filter = doc! {self.key_field.as_str():path};
+        let result = collection
+            .find_one(filter)
+            .await
+            .map_err(parse_mongodb_error)?;
+        match result {
+            Some(doc) => {
+                let value = doc
+                    .get_binary_generic(&self.value_field)
+                    .map_err(parse_bson_error)?;
+                Ok(Some(Buffer::from(value.to_vec())))
+            }
+            None => Ok(None),
+        }
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let collection = self.get_collection().await?;
+        let filter = doc! { self.key_field.as_str(): path };
+        let update = doc! { "$set": { self.value_field.as_str(): Binary { subtype: mongodb::bson::spec::BinarySubtype::Generic, bytes: value.to_vec() } } };
+        collection
+            .update_one(filter, update)
+            .upsert(true)
+            .await
+            .map_err(parse_mongodb_error)?;
+
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let collection = self.get_collection().await?;
+        let filter = doc! {self.key_field.as_str():path};
+        collection
+            .delete_one(filter)
+            .await
+            .map_err(parse_mongodb_error)?;
+        Ok(())
+    }
+}
+
+fn parse_mongodb_error(err: mongodb::error::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "mongodb error").set_source(err)
+}
+
+fn parse_bson_error(err: mongodb::bson::document::ValueAccessError) -> Error {
+    Error::new(ErrorKind::Unexpected, "bson error").set_source(err)
+}

data/core/src/services/mongodb/config.rs

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Mongodb service support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct MongodbConfig {
+    /// connection string of this backend
+    pub connection_string: Option<String>,
+    /// database of this backend
+    pub database: Option<String>,
+    /// collection of this backend
+    pub collection: Option<String>,
+    /// root of this backend
+    pub root: Option<String>,
+    /// key field of this backend
+    pub key_field: Option<String>,
+    /// value field of this backend
+    pub value_field: Option<String>,
+}
+
+impl Debug for MongodbConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MongodbConfig")
+            .field("connection_string", &self.connection_string)
+            .field("database", &self.database)
+            .field("collection", &self.collection)
+            .field("root", &self.root)
+            .field("key_field", &self.key_field)
+            .field("value_field", &self.value_field)
+            .finish()
+    }
+}

data/core/src/services/mongodb/docs.md

@@ -0,0 +1,49 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `connection_string`: Set the connection string of mongodb server
+- `database`: Set the database of mongodb
+- `collection`: Set the collection of mongodb
+- `key_field`: Set the key field of mongodb
+- `value_field`: Set the value field of mongodb
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Mongodb;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Mongodb::default()
+        .root("/")
+        .connection_string("mongodb://myUser:myPassword@localhost:27017/myAuthDB")
+        .database("your_database")
+        .collection("your_collection")
+        // key field type in the table should be compatible with Rust's &str like text
+        .key_field("key")
+        // value field type in the table should be compatible with Rust's Vec<u8> like bytea
+        .value_field("value");
+
+    let op = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/mongodb/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-mongodb")]
+mod backend;
+#[cfg(feature = "services-mongodb")]
+pub use backend::MongodbBuilder as Mongodb;
+
+mod config;
+pub use config::MongodbConfig;