opendal 0.1.6.pre.rc.1

data/core/src/services/postgresql/backend.rs

@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::str::FromStr;
+
+use sqlx::postgres::PgConnectOptions;
+use sqlx::PgPool;
+use tokio::sync::OnceCell;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::PostgresqlConfig;
+use crate::*;
+
+impl Configurator for PostgresqlConfig {
+    type Builder = PostgresqlBuilder;
+    fn into_builder(self) -> Self::Builder {
+        PostgresqlBuilder { config: self }
+    }
+}
+
+/// [PostgreSQL](https://www.postgresql.org/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct PostgresqlBuilder {
+    config: PostgresqlConfig,
+}
+
+impl Debug for PostgresqlBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("PostgresqlBuilder");
+
+        d.field("config", &self.config);
+        d.finish()
+    }
+}
+
+impl PostgresqlBuilder {
+    /// Set the connection url string of the postgresql service.
+    ///
+    /// The URL should be with a scheme of either `postgres://` or `postgresql://`.
+    ///
+    /// - `postgresql://user@localhost`
+    /// - `postgresql://user:password@%2Fvar%2Flib%2Fpostgresql/mydb?connect_timeout=10`
+    /// - `postgresql://user@host1:1234,host2,host3:5678?target_session_attrs=read-write`
+    /// - `postgresql:///mydb?user=user&host=/var/lib/postgresql`
+    ///
+    /// For more information, please visit <https://docs.rs/sqlx/latest/sqlx/postgres/struct.PgConnectOptions.html>.
+    pub fn connection_string(mut self, v: &str) -> Self {
+        if !v.is_empty() {
+            self.config.connection_string = Some(v.to_string());
+        }
+        self
+    }
+
+    /// Set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the table name of the postgresql service to read/write.
+    pub fn table(mut self, table: &str) -> Self {
+        if !table.is_empty() {
+            self.config.table = Some(table.to_string());
+        }
+        self
+    }
+
+    /// Set the key field name of the postgresql service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub fn key_field(mut self, key_field: &str) -> Self {
+        if !key_field.is_empty() {
+            self.config.key_field = Some(key_field.to_string());
+        }
+        self
+    }
+
+    /// Set the value field name of the postgresql service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub fn value_field(mut self, value_field: &str) -> Self {
+        if !value_field.is_empty() {
+            self.config.value_field = Some(value_field.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for PostgresqlBuilder {
+    type Config = PostgresqlConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let conn = match self.config.connection_string {
+            Some(v) => v,
+            None => {
+                return Err(
+                    Error::new(ErrorKind::ConfigInvalid, "connection_string is empty")
+                        .with_context("service", Scheme::Postgresql),
+                )
+            }
+        };
+
+        let config = PgConnectOptions::from_str(&conn).map_err(|err| {
+            Error::new(ErrorKind::ConfigInvalid, "connection_string is invalid")
+                .with_context("service", Scheme::Postgresql)
+                .set_source(err)
+        })?;
+
+        let table = match self.config.table {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "table is empty")
+                    .with_context("service", Scheme::Postgresql))
+            }
+        };
+
+        let key_field = self.config.key_field.unwrap_or_else(|| "key".to_string());
+
+        let value_field = self
+            .config
+            .value_field
+            .unwrap_or_else(|| "value".to_string());
+
+        let root = normalize_root(self.config.root.unwrap_or_else(|| "/".to_string()).as_str());
+
+        Ok(PostgresqlBackend::new(Adapter {
+            pool: OnceCell::new(),
+            config,
+            table,
+            key_field,
+            value_field,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+/// Backend for Postgresql service
+pub type PostgresqlBackend = kv::Backend<Adapter>;
+
+#[derive(Debug, Clone)]
+pub struct Adapter {
+    pool: OnceCell<PgPool>,
+    config: PgConnectOptions,
+
+    table: String,
+    key_field: String,
+    value_field: String,
+}
+
+impl Adapter {
+    async fn get_client(&self) -> Result<&PgPool> {
+        self.pool
+            .get_or_try_init(|| async {
+                let pool = PgPool::connect_with(self.config.clone())
+                    .await
+                    .map_err(parse_postgres_error)?;
+                Ok(pool)
+            })
+            .await
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Postgresql,
+            &self.table,
+            Capability {
+                read: true,
+                write: true,
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let pool = self.get_client().await?;
+
+        let value: Option<Vec<u8>> = sqlx::query_scalar(&format!(
+            r#"SELECT "{}" FROM "{}" WHERE "{}" = $1 LIMIT 1"#,
+            self.value_field, self.table, self.key_field
+        ))
+        .bind(path)
+        .fetch_optional(pool)
+        .await
+        .map_err(parse_postgres_error)?;
+
+        Ok(value.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        let table = &self.table;
+        let key_field = &self.key_field;
+        let value_field = &self.value_field;
+        sqlx::query(&format!(
+            r#"INSERT INTO "{table}" ("{key_field}", "{value_field}")
+                VALUES ($1, $2)
+                ON CONFLICT ("{key_field}")
+                    DO UPDATE SET "{value_field}" = EXCLUDED."{value_field}""#,
+        ))
+        .bind(path)
+        .bind(value.to_vec())
+        .execute(pool)
+        .await
+        .map_err(parse_postgres_error)?;
+
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        sqlx::query(&format!(
+            "DELETE FROM {} WHERE {} = $1",
+            self.table, self.key_field
+        ))
+        .bind(path)
+        .execute(pool)
+        .await
+        .map_err(parse_postgres_error)?;
+
+        Ok(())
+    }
+}
+
+fn parse_postgres_error(err: sqlx::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "unhandled error from postgresql").set_source(err)
+}

data/core/src/services/postgresql/config.rs

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for PostgreSQL services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct PostgresqlConfig {
+    /// Root of this backend.
+    ///
+    /// All operations will happen under this root.
+    ///
+    /// Default to `/` if not set.
+    pub root: Option<String>,
+    /// The URL should be with a scheme of either `postgres://` or `postgresql://`.
+    ///
+    /// - `postgresql://user@localhost`
+    /// - `postgresql://user:password@%2Fvar%2Flib%2Fpostgresql/mydb?connect_timeout=10`
+    /// - `postgresql://user@host1:1234,host2,host3:5678?target_session_attrs=read-write`
+    /// - `postgresql:///mydb?user=user&host=/var/lib/postgresql`
+    ///
+    /// For more information, please visit <https://docs.rs/sqlx/latest/sqlx/postgres/struct.PgConnectOptions.html>.
+    pub connection_string: Option<String>,
+    /// the table of postgresql
+    pub table: Option<String>,
+    /// the key field of postgresql
+    pub key_field: Option<String>,
+    /// the value field of postgresql
+    pub value_field: Option<String>,
+}
+
+impl Debug for PostgresqlConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("PostgresqlConfig");
+
+        if self.connection_string.is_some() {
+            d.field("connection_string", &"<redacted>");
+        }
+
+        d.field("root", &self.root)
+            .field("table", &self.table)
+            .field("key_field", &self.key_field)
+            .field("value_field", &self.value_field)
+            .finish()
+    }
+}

data/core/src/services/postgresql/docs.md

@@ -0,0 +1,47 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `connection_string`: Set the connection string of postgres server
+- `table`: Set the table of postgresql
+- `key_field`: Set the key field of postgresql
+- `value_field`: Set the value field of postgresql
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Postgresql;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Postgresql::default()
+        .root("/")
+        .connection_string("postgresql://you_username:your_password@127.0.0.1:5432/your_database")
+        .table("your_table")
+        // key field type in the table should be compatible with Rust's &str like text
+        .key_field("key")
+        // value field type in the table should be compatible with Rust's Vec<u8> like bytea
+        .value_field("value");
+
+    let op = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/postgresql/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-postgresql")]
+mod backend;
+#[cfg(feature = "services-postgresql")]
+pub use backend::PostgresqlBuilder as Postgresql;
+
+mod config;
+pub use config::PostgresqlConfig;

data/core/src/services/redb/backend.rs

@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::RedbConfig;
+use crate::Builder;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Scheme;
+use crate::*;
+
+impl Configurator for RedbConfig {
+    type Builder = RedbBuilder;
+    fn into_builder(self) -> Self::Builder {
+        RedbBuilder {
+            config: self,
+            database: None,
+        }
+    }
+}
+
+/// Redb service support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Debug)]
+pub struct RedbBuilder {
+    config: RedbConfig,
+
+    database: Option<Arc<redb::Database>>,
+}
+
+impl RedbBuilder {
+    /// Set the database for Redb.
+    ///
+    /// This method should be called when you want to
+    /// use multiple tables of one database because
+    /// Redb doesn't allow opening a database that have been opened.
+    ///
+    /// <div class="warning">
+    ///
+    /// `datadir` and `database` should not be set simultaneously.
+    /// If both are set, `database` will take precedence.
+    ///
+    /// </div>
+    pub fn database(mut self, db: Arc<redb::Database>) -> Self {
+        self.database = Some(db);
+        self
+    }
+
+    /// Set the path to the redb data directory. Will create if not exists.
+    ///
+    ///
+    /// <div class="warning">
+    ///
+    /// Opening redb database via `datadir` takes away the ability to access multiple redb tables.
+    /// If you need to access multiple redb tables, the correct solution is to
+    /// create an `Arc<redb::database>` beforehand and then share it via [`database`]
+    /// with multiple builders where every builder will open one redb table.
+    ///
+    /// </div>
+    ///
+    /// [`database`]: RedbBuilder::database
+    pub fn datadir(mut self, path: &str) -> Self {
+        self.config.datadir = Some(path.into());
+        self
+    }
+
+    /// Set the table name for Redb. Will create if not exists.
+    pub fn table(mut self, table: &str) -> Self {
+        self.config.table = Some(table.into());
+        self
+    }
+
+    /// Set the root for Redb.
+    pub fn root(mut self, path: &str) -> Self {
+        self.config.root = Some(path.into());
+        self
+    }
+}
+
+impl Builder for RedbBuilder {
+    type Config = RedbConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let table_name = self.config.table.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "table is required but not set")
+                .with_context("service", Scheme::Redb)
+        })?;
+
+        let (datadir, db) = if let Some(db) = self.database {
+            (None, db)
+        } else {
+            let datadir = self.config.datadir.ok_or_else(|| {
+                Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set")
+                    .with_context("service", Scheme::Redb)
+            })?;
+
+            let db = redb::Database::create(&datadir)
+                .map_err(parse_database_error)?
+                .into();
+
+            (Some(datadir), db)
+        };
+
+        create_table(&db, &table_name)?;
+
+        Ok(RedbBackend::new(Adapter {
+            datadir,
+            table: table_name,
+            db,
+        })
+        .with_root(self.config.root.as_deref().unwrap_or_default()))
+    }
+}
+
+/// Backend for Redb services.
+pub type RedbBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    datadir: Option<String>,
+    table: String,
+    db: Arc<redb::Database>,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+        ds.field("path", &self.datadir);
+        ds.finish()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Redb,
+            &self.table,
+            Capability {
+                read: true,
+                write: true,
+                shared: false,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let read_txn = self.db.begin_read().map_err(parse_transaction_error)?;
+
+        let table_define: redb::TableDefinition<&str, &[u8]> =
+            redb::TableDefinition::new(&self.table);
+
+        let table = read_txn
+            .open_table(table_define)
+            .map_err(parse_table_error)?;
+
+        let result = match table.get(path) {
+            Ok(Some(v)) => Ok(Some(v.value().to_vec())),
+            Ok(None) => Ok(None),
+            Err(e) => Err(parse_storage_error(e)),
+        }?;
+        Ok(result.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let write_txn = self.db.begin_write().map_err(parse_transaction_error)?;
+
+        let table_define: redb::TableDefinition<&str, &[u8]> =
+            redb::TableDefinition::new(&self.table);
+
+        {
+            let mut table = write_txn
+                .open_table(table_define)
+                .map_err(parse_table_error)?;
+
+            table
+                .insert(path, &*value.to_vec())
+                .map_err(parse_storage_error)?;
+        }
+
+        write_txn.commit().map_err(parse_commit_error)?;
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let write_txn = self.db.begin_write().map_err(parse_transaction_error)?;
+
+        let table_define: redb::TableDefinition<&str, &[u8]> =
+            redb::TableDefinition::new(&self.table);
+
+        {
+            let mut table = write_txn
+                .open_table(table_define)
+                .map_err(parse_table_error)?;
+
+            table.remove(path).map_err(parse_storage_error)?;
+        }
+
+        write_txn.commit().map_err(parse_commit_error)?;
+        Ok(())
+    }
+}
+
+fn parse_transaction_error(e: redb::TransactionError) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)
+}
+
+fn parse_table_error(e: redb::TableError) -> Error {
+    match e {
+        redb::TableError::TableDoesNotExist(_) => {
+            Error::new(ErrorKind::NotFound, "error from redb").set_source(e)
+        }
+        _ => Error::new(ErrorKind::Unexpected, "error from redb").set_source(e),
+    }
+}
+
+fn parse_storage_error(e: redb::StorageError) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)
+}
+
+fn parse_database_error(e: redb::DatabaseError) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)
+}
+
+fn parse_commit_error(e: redb::CommitError) -> Error {
+    Error::new(ErrorKind::Unexpected, "error from redb").set_source(e)
+}
+
+/// Check if a table exists, otherwise create it.
+fn create_table(db: &redb::Database, table: &str) -> Result<()> {
+    // Only one `WriteTransaction` is permitted at same time,
+    // applying new one will block until it available.
+    //
+    // So we first try checking table existence via `ReadTransaction`.
+    {
+        let read_txn = db.begin_read().map_err(parse_transaction_error)?;
+
+        let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(table);
+
+        match read_txn.open_table(table_define) {
+            Ok(_) => return Ok(()),
+            Err(redb::TableError::TableDoesNotExist(_)) => (),
+            Err(e) => return Err(parse_table_error(e)),
+        }
+    }
+
+    {
+        let write_txn = db.begin_write().map_err(parse_transaction_error)?;
+
+        let table_define: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new(table);
+
+        write_txn
+            .open_table(table_define)
+            .map_err(parse_table_error)?;
+        write_txn.commit().map_err(parse_commit_error)?;
+    }
+
+    Ok(())
+}