opendal 0.1.6.pre.rc.1

data/core/src/services/sled/backend.rs

@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::str;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::SledConfig;
+use crate::Builder;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Scheme;
+use crate::*;
+
+// https://github.com/spacejam/sled/blob/69294e59c718289ab3cb6bd03ac3b9e1e072a1e7/src/db.rs#L5
+const DEFAULT_TREE_ID: &str = r#"__sled__default"#;
+
+impl Configurator for SledConfig {
+    type Builder = SledBuilder;
+    fn into_builder(self) -> Self::Builder {
+        SledBuilder { config: self }
+    }
+}
+
+/// Sled services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct SledBuilder {
+    config: SledConfig,
+}
+
+impl Debug for SledBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SledBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl SledBuilder {
+    /// Set the path to the sled data directory. Will create if not exists.
+    pub fn datadir(mut self, path: &str) -> Self {
+        self.config.datadir = Some(path.into());
+        self
+    }
+
+    /// Set the root for sled.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the tree for sled.
+    pub fn tree(mut self, tree: &str) -> Self {
+        self.config.tree = Some(tree.into());
+        self
+    }
+}
+
+impl Builder for SledBuilder {
+    type Config = SledConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let datadir_path = self.config.datadir.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set")
+                .with_context("service", Scheme::Sled)
+        })?;
+
+        let db = sled::open(&datadir_path).map_err(|e| {
+            Error::new(ErrorKind::ConfigInvalid, "open db")
+                .with_context("service", Scheme::Sled)
+                .with_context("datadir", datadir_path.clone())
+                .set_source(e)
+        })?;
+
+        // use "default" tree if not set
+        let tree_name = self
+            .config
+            .tree
+            .unwrap_or_else(|| DEFAULT_TREE_ID.to_string());
+
+        let tree = db.open_tree(&tree_name).map_err(|e| {
+            Error::new(ErrorKind::ConfigInvalid, "open tree")
+                .with_context("service", Scheme::Sled)
+                .with_context("datadir", datadir_path.clone())
+                .with_context("tree", tree_name.clone())
+                .set_source(e)
+        })?;
+
+        Ok(SledBackend::new(Adapter {
+            datadir: datadir_path,
+            tree,
+        })
+        .with_root(self.config.root.as_deref().unwrap_or("/")))
+    }
+}
+
+/// Backend for sled services.
+pub type SledBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    datadir: String,
+    tree: sled::Tree,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+        ds.field("path", &self.datadir);
+        ds.finish()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = kv::Scanner;
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Sled,
+            &self.datadir,
+            Capability {
+                read: true,
+                write: true,
+                list: true,
+                shared: false,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        Ok(self
+            .tree
+            .get(path)
+            .map_err(parse_error)?
+            .map(|v| Buffer::from(v.to_vec())))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        self.tree
+            .insert(path, value.to_vec())
+            .map_err(parse_error)?;
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        self.tree.remove(path).map_err(parse_error)?;
+
+        Ok(())
+    }
+
+    async fn scan(&self, path: &str) -> Result<Self::Scanner> {
+        let it = self.tree.scan_prefix(path).keys();
+        let mut res = Vec::default();
+
+        for i in it {
+            let bs = i.map_err(parse_error)?.to_vec();
+            let v = String::from_utf8(bs).map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "store key is not valid utf-8 string")
+                    .set_source(err)
+            })?;
+
+            res.push(v);
+        }
+
+        Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
+    }
+}
+
+fn parse_error(err: sled::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from sled").set_source(err)
+}

data/core/src/services/sled/config.rs

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Sled services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct SledConfig {
+    /// That path to the sled data directory.
+    pub datadir: Option<String>,
+    /// The root for sled.
+    pub root: Option<String>,
+    /// The tree for sled.
+    pub tree: Option<String>,
+}
+
+impl Debug for SledConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SledConfig")
+            .field("datadir", &self.datadir)
+            .field("root", &self.root)
+            .field("tree", &self.tree)
+            .finish()
+    }
+}

data/core/src/services/sled/docs.md

@@ -0,0 +1,39 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [x] blocking
+
+## Configuration
+
+- `datadir`: Set the path to the sled data directory
+
+You can refer to [`SledBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Sled;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Sled::default()
+        .datadir("/tmp/opendal/sled");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/sled/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-sled")]
+mod backend;
+#[cfg(feature = "services-sled")]
+pub use backend::SledBuilder as Sled;
+
+mod config;
+pub use config::SledConfig;

data/core/src/services/sqlite/backend.rs

@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::task::Context;
+use std::task::Poll;
+
+use futures::stream::BoxStream;
+use futures::Stream;
+use futures::StreamExt;
+use ouroboros::self_referencing;
+use sqlx::sqlite::SqliteConnectOptions;
+use sqlx::SqlitePool;
+use tokio::sync::OnceCell;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::SqliteConfig;
+use crate::*;
+
+impl Configurator for SqliteConfig {
+    type Builder = SqliteBuilder;
+    fn into_builder(self) -> Self::Builder {
+        SqliteBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct SqliteBuilder {
+    config: SqliteConfig,
+}
+
+impl Debug for SqliteBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("SqliteBuilder");
+
+        ds.field("config", &self.config);
+        ds.finish()
+    }
+}
+
+impl SqliteBuilder {
+    /// Set the connection_string of the sqlite service.
+    ///
+    /// This connection string is used to connect to the sqlite service. There are url based formats:
+    ///
+    /// ## Url
+    ///
+    /// This format resembles the url format of the sqlite client:
+    ///
+    /// - `sqlite::memory:`
+    /// - `sqlite:data.db`
+    /// - `sqlite://data.db`
+    ///
+    /// For more information, please visit <https://docs.rs/sqlx/latest/sqlx/sqlite/struct.SqliteConnectOptions.html>.
+    pub fn connection_string(mut self, v: &str) -> Self {
+        if !v.is_empty() {
+            self.config.connection_string = Some(v.to_string());
+        }
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the table name of the sqlite service to read/write.
+    pub fn table(mut self, table: &str) -> Self {
+        if !table.is_empty() {
+            self.config.table = Some(table.to_string());
+        }
+        self
+    }
+
+    /// Set the key field name of the sqlite service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub fn key_field(mut self, key_field: &str) -> Self {
+        if !key_field.is_empty() {
+            self.config.key_field = Some(key_field.to_string());
+        }
+        self
+    }
+
+    /// Set the value field name of the sqlite service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub fn value_field(mut self, value_field: &str) -> Self {
+        if !value_field.is_empty() {
+            self.config.value_field = Some(value_field.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for SqliteBuilder {
+    type Config = SqliteConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let conn = match self.config.connection_string {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "connection_string is required but not set",
+                )
+                .with_context("service", Scheme::Sqlite));
+            }
+        };
+
+        let config = SqliteConnectOptions::from_str(&conn).map_err(|err| {
+            Error::new(ErrorKind::ConfigInvalid, "connection_string is invalid")
+                .with_context("service", Scheme::Sqlite)
+                .set_source(err)
+        })?;
+
+        let table = match self.config.table {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "table is empty")
+                    .with_context("service", Scheme::Sqlite));
+            }
+        };
+
+        let key_field = self.config.key_field.unwrap_or_else(|| "key".to_string());
+
+        let value_field = self
+            .config
+            .value_field
+            .unwrap_or_else(|| "value".to_string());
+
+        let root = normalize_root(self.config.root.as_deref().unwrap_or("/"));
+
+        Ok(SqliteBackend::new(Adapter {
+            pool: OnceCell::new(),
+            config,
+            table,
+            key_field,
+            value_field,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+pub type SqliteBackend = kv::Backend<Adapter>;
+
+#[derive(Debug, Clone)]
+pub struct Adapter {
+    pool: OnceCell<SqlitePool>,
+    config: SqliteConnectOptions,
+
+    table: String,
+    key_field: String,
+    value_field: String,
+}
+
+impl Adapter {
+    async fn get_client(&self) -> Result<&SqlitePool> {
+        self.pool
+            .get_or_try_init(|| async {
+                let pool = SqlitePool::connect_with(self.config.clone())
+                    .await
+                    .map_err(parse_sqlite_error)?;
+                Ok(pool)
+            })
+            .await
+    }
+}
+
+#[self_referencing]
+pub struct SqliteScanner {
+    pool: SqlitePool,
+    query: String,
+
+    #[borrows(pool, query)]
+    #[covariant]
+    stream: BoxStream<'this, Result<String>>,
+}
+
+impl Stream for SqliteScanner {
+    type Item = Result<String>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.with_stream_mut(|s| s.poll_next_unpin(cx))
+    }
+}
+
+unsafe impl Sync for SqliteScanner {}
+
+impl kv::Scan for SqliteScanner {
+    async fn next(&mut self) -> Result<Option<String>> {
+        <Self as StreamExt>::next(self).await.transpose()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = SqliteScanner;
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Sqlite,
+            &self.table,
+            Capability {
+                read: true,
+                write: true,
+                delete: true,
+                list: true,
+                shared: false,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let pool = self.get_client().await?;
+
+        let value: Option<Vec<u8>> = sqlx::query_scalar(&format!(
+            "SELECT `{}` FROM `{}` WHERE `{}` = $1 LIMIT 1",
+            self.value_field, self.table, self.key_field
+        ))
+        .bind(path)
+        .fetch_optional(pool)
+        .await
+        .map_err(parse_sqlite_error)?;
+
+        Ok(value.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        sqlx::query(&format!(
+            "INSERT OR REPLACE INTO `{}` (`{}`, `{}`) VALUES ($1, $2)",
+            self.table, self.key_field, self.value_field,
+        ))
+        .bind(path)
+        .bind(value.to_vec())
+        .execute(pool)
+        .await
+        .map_err(parse_sqlite_error)?;
+
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        sqlx::query(&format!(
+            "DELETE FROM `{}` WHERE `{}` = $1",
+            self.table, self.key_field
+        ))
+        .bind(path)
+        .execute(pool)
+        .await
+        .map_err(parse_sqlite_error)?;
+
+        Ok(())
+    }
+
+    async fn scan(&self, path: &str) -> Result<Self::Scanner> {
+        let pool = self.get_client().await?;
+        let stream = SqliteScannerBuilder {
+            pool: pool.clone(),
+            query: format!(
+                "SELECT `{}` FROM `{}` WHERE `{}` LIKE $1",
+                self.key_field, self.table, self.key_field
+            ),
+            stream_builder: |pool, query| {
+                sqlx::query_scalar(query)
+                    .bind(format!("{path}%"))
+                    .fetch(pool)
+                    .map(|v| v.map_err(parse_sqlite_error))
+                    .boxed()
+            },
+        }
+        .build();
+
+        Ok(stream)
+    }
+}
+
+fn parse_sqlite_error(err: sqlx::Error) -> Error {
+    let is_temporary = matches!(
+        &err,
+        sqlx::Error::Database(db_err) if db_err.code().is_some_and(|c| c == "5" || c == "6")
+    );
+
+    let message = if is_temporary {
+        "database is locked or busy"
+    } else {
+        "unhandled error from sqlite"
+    };
+
+    let mut error = Error::new(ErrorKind::Unexpected, message).set_source(err);
+    if is_temporary {
+        error = error.set_temporary();
+    }
+    error
+}

data/core/src/services/sqlite/config.rs

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Sqlite support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct SqliteConfig {
+    /// Set the connection_string of the sqlite service.
+    ///
+    /// This connection string is used to connect to the sqlite service.
+    ///
+    /// The format of connect string resembles the url format of the sqlite client:
+    ///
+    /// - `sqlite::memory:`
+    /// - `sqlite:data.db`
+    /// - `sqlite://data.db`
+    ///
+    /// For more information, please visit <https://docs.rs/sqlx/latest/sqlx/sqlite/struct.SqliteConnectOptions.html>.
+    pub connection_string: Option<String>,
+
+    /// Set the table name of the sqlite service to read/write.
+    pub table: Option<String>,
+    /// Set the key field name of the sqlite service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub key_field: Option<String>,
+    /// Set the value field name of the sqlite service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub value_field: Option<String>,
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub root: Option<String>,
+}
+
+impl Debug for SqliteConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("SqliteConfig");
+
+        d.field("connection_string", &self.connection_string)
+            .field("table", &self.table)
+            .field("key_field", &self.key_field)
+            .field("value_field", &self.value_field)
+            .field("root", &self.root);
+
+        d.finish_non_exhaustive()
+    }
+}