opendal 0.1.6.pre.rc.1

data/core/src/services/monoiofs/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for monoiofs service.
+#[cfg(feature = "services-monoiofs")]
+pub(super) const DEFAULT_SCHEME: &str = "monoiofs";
+#[cfg(feature = "services-monoiofs")]
+mod core;
+#[cfg(feature = "services-monoiofs")]
+mod delete;
+#[cfg(feature = "services-monoiofs")]
+mod reader;
+#[cfg(feature = "services-monoiofs")]
+mod writer;
+
+#[cfg(feature = "services-monoiofs")]
+mod backend;
+#[cfg(feature = "services-monoiofs")]
+pub use backend::MonoiofsBuilder as Monoiofs;
+
+mod config;
+pub use config::MonoiofsConfig;

data/core/src/services/monoiofs/reader.rs

@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use bytes::BytesMut;
+use futures::channel::mpsc;
+use futures::channel::oneshot;
+use futures::SinkExt;
+use futures::StreamExt;
+use monoio::fs::OpenOptions;
+
+use super::core::MonoiofsCore;
+use super::core::BUFFER_SIZE;
+use crate::raw::*;
+use crate::*;
+
+enum ReaderRequest {
+    Read {
+        pos: u64,
+        buf: BytesMut,
+        tx: oneshot::Sender<Result<BytesMut>>,
+    },
+}
+
+pub struct MonoiofsReader {
+    core: Arc<MonoiofsCore>,
+    tx: mpsc::UnboundedSender<ReaderRequest>,
+    pos: u64,
+    end_pos: Option<u64>,
+}
+
+impl MonoiofsReader {
+    pub async fn new(core: Arc<MonoiofsCore>, path: PathBuf, range: BytesRange) -> Result<Self> {
+        let (open_result_tx, open_result_rx) = oneshot::channel();
+        let (tx, rx) = mpsc::unbounded();
+        core.spawn(move || Self::worker_entrypoint(path, rx, open_result_tx))
+            .await;
+        core.unwrap(open_result_rx.await)?;
+        Ok(Self {
+            core,
+            tx,
+            pos: range.offset(),
+            end_pos: range.size().map(|size| range.offset() + size),
+        })
+    }
+
+    /// entrypoint of worker task that runs in context of monoio
+    async fn worker_entrypoint(
+        path: PathBuf,
+        mut rx: mpsc::UnboundedReceiver<ReaderRequest>,
+        open_result_tx: oneshot::Sender<Result<()>>,
+    ) {
+        let result = OpenOptions::new().read(true).open(path).await;
+        // [`monoio::fs::File`] is non-Send, hence it is kept within
+        // worker thread
+        let file = match result {
+            Ok(file) => {
+                let Ok(()) = open_result_tx.send(Ok(())) else {
+                    // MonoiofsReader::new is cancelled, exit worker task
+                    return;
+                };
+                file
+            }
+            Err(e) => {
+                // discard the result if send failed due to MonoiofsReader::new
+                // cancelled since we are going to exit anyway
+                let _ = open_result_tx.send(Err(new_std_io_error(e)));
+                return;
+            }
+        };
+        // wait for read request and send back result to main thread
+        loop {
+            let Some(req) = rx.next().await else {
+                // MonoiofsReader is dropped, exit worker task
+                break;
+            };
+            match req {
+                ReaderRequest::Read { pos, buf, tx } => {
+                    let (result, buf) = file.read_at(buf, pos).await;
+                    // buf.len() will be set to n by monoio if read
+                    // successfully, so n is dropped
+                    let result = result.map(move |_| buf).map_err(new_std_io_error);
+                    // discard the result if send failed due to
+                    // MonoiofsReader::read cancelled
+                    let _ = tx.send(result);
+                }
+            }
+        }
+    }
+}
+
+impl oio::Read for MonoiofsReader {
+    /// Send read request to worker thread and wait for result. Actual
+    /// read happens in [`MonoiofsReader::worker_entrypoint`] running
+    /// on worker thread.
+    async fn read(&mut self) -> Result<Buffer> {
+        if let Some(end_pos) = self.end_pos {
+            if self.pos >= end_pos {
+                return Ok(Buffer::new());
+            }
+        }
+
+        // allocate and resize buffer
+        let mut buf = self.core.buf_pool.get();
+        let size = self
+            .end_pos
+            .map_or(BUFFER_SIZE, |end_pos| (end_pos - self.pos) as usize);
+        // set capacity of buf to exact size to avoid excessive read
+        buf.reserve(size);
+        let _ = buf.split_off(size);
+
+        // send read request to worker thread and wait for result
+        let (tx, rx) = oneshot::channel();
+        self.core.unwrap(
+            self.tx
+                .send(ReaderRequest::Read {
+                    pos: self.pos,
+                    buf,
+                    tx,
+                })
+                .await,
+        );
+        let mut buf = self.core.unwrap(rx.await)?;
+
+        // advance cursor if read successfully
+        self.pos += buf.len() as u64;
+        let buffer = Buffer::from(buf.split().freeze());
+        self.core.buf_pool.put(buf);
+        Ok(buffer)
+    }
+}

data/core/src/services/monoiofs/writer.rs

@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use bytes::Buf;
+use bytes::Bytes;
+use chrono::DateTime;
+use futures::channel::mpsc;
+use futures::channel::oneshot;
+use futures::SinkExt;
+use futures::StreamExt;
+use monoio::fs::OpenOptions;
+
+use super::core::MonoiofsCore;
+use crate::raw::*;
+use crate::*;
+
+enum WriterRequest {
+    Write {
+        pos: u64,
+        buf: Bytes,
+        tx: oneshot::Sender<Result<()>>,
+    },
+    Stat {
+        tx: oneshot::Sender<Result<monoio::fs::Metadata>>,
+    },
+    Close {
+        tx: oneshot::Sender<Result<()>>,
+    },
+}
+
+pub struct MonoiofsWriter {
+    core: Arc<MonoiofsCore>,
+    tx: mpsc::UnboundedSender<WriterRequest>,
+    pos: u64,
+}
+
+impl MonoiofsWriter {
+    pub async fn new(core: Arc<MonoiofsCore>, path: PathBuf, append: bool) -> Result<Self> {
+        let (open_result_tx, open_result_rx) = oneshot::channel();
+        let (tx, rx) = mpsc::unbounded();
+        core.spawn(move || Self::worker_entrypoint(path, append, rx, open_result_tx))
+            .await;
+        core.unwrap(open_result_rx.await)?;
+        Ok(Self { core, tx, pos: 0 })
+    }
+
+    /// entrypoint of worker task that runs in context of monoio
+    async fn worker_entrypoint(
+        path: PathBuf,
+        append: bool,
+        mut rx: mpsc::UnboundedReceiver<WriterRequest>,
+        open_result_tx: oneshot::Sender<Result<()>>,
+    ) {
+        let result = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .append(append)
+            .truncate(!append)
+            .open(path)
+            .await;
+        // [`monoio::fs::File`] is non-Send, hence it is kept within
+        // worker thread
+        let file = match result {
+            Ok(file) => {
+                let Ok(()) = open_result_tx.send(Ok(())) else {
+                    // MonoiofsWriter::new is cancelled, exit worker task
+                    return;
+                };
+                file
+            }
+            Err(e) => {
+                // discard the result if send failed due to MonoiofsWriter::new
+                // cancelled since we are going to exit anyway
+                let _ = open_result_tx.send(Err(new_std_io_error(e)));
+                return;
+            }
+        };
+        // wait for write or close request and send back result to main thread
+        loop {
+            let Some(req) = rx.next().await else {
+                // MonoiofsWriter is dropped, exit worker task
+                break;
+            };
+            match req {
+                WriterRequest::Write { pos, buf, tx } => {
+                    let (result, _) = file.write_all_at(buf, pos).await;
+                    // discard the result if send failed due to
+                    // MonoiofsWriter::write cancelled
+                    let _ = tx.send(result.map_err(new_std_io_error));
+                }
+                WriterRequest::Stat { tx } => {
+                    let result = file.metadata().await;
+                    let _ = tx.send(result.map_err(new_std_io_error));
+                }
+                WriterRequest::Close { tx } => {
+                    let result = file.sync_all().await;
+                    // discard the result if send failed due to
+                    // MonoiofsWriter::close cancelled
+                    let _ = tx.send(result.map_err(new_std_io_error));
+                    // file is closed in background and result is useless
+                    let _ = file.close().await;
+                    break;
+                }
+            }
+        }
+    }
+}
+
+impl oio::Write for MonoiofsWriter {
+    /// Send write request to worker thread and wait for result. Actual
+    /// write happens in [`MonoiofsWriter::worker_entrypoint`] running
+    /// on worker thread.
+    async fn write(&mut self, mut bs: Buffer) -> Result<()> {
+        while bs.has_remaining() {
+            let buf = bs.current();
+            let n = buf.len();
+            let (tx, rx) = oneshot::channel();
+            self.core.unwrap(
+                self.tx
+                    .send(WriterRequest::Write {
+                        pos: self.pos,
+                        buf,
+                        tx,
+                    })
+                    .await,
+            );
+            self.core.unwrap(rx.await)?;
+            self.pos += n as u64;
+            bs.advance(n);
+        }
+        Ok(())
+    }
+
+    /// Send close request to worker thread and wait for result. Actual
+    /// close happens in [`MonoiofsWriter::worker_entrypoint`] running
+    /// on worker thread.
+    async fn close(&mut self) -> Result<Metadata> {
+        let (tx, rx) = oneshot::channel();
+        self.core
+            .unwrap(self.tx.send(WriterRequest::Stat { tx }).await);
+        let file_meta = self.core.unwrap(rx.await)?;
+
+        let (tx, rx) = oneshot::channel();
+        self.core
+            .unwrap(self.tx.send(WriterRequest::Close { tx }).await);
+        self.core.unwrap(rx.await)?;
+
+        let mode = if file_meta.is_dir() {
+            EntryMode::DIR
+        } else if file_meta.is_file() {
+            EntryMode::FILE
+        } else {
+            EntryMode::Unknown
+        };
+        let meta = Metadata::new(mode)
+            .with_content_length(file_meta.len())
+            .with_last_modified(
+                file_meta
+                    .modified()
+                    .map(DateTime::from)
+                    .map_err(new_std_io_error)?,
+            );
+        Ok(meta)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "Monoiofs doesn't support abort",
+        ))
+    }
+}

data/core/src/services/mysql/backend.rs

@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::str::FromStr;
+
+use sqlx::mysql::MySqlConnectOptions;
+use sqlx::MySqlPool;
+use tokio::sync::OnceCell;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::MysqlConfig;
+use crate::*;
+
+impl Configurator for MysqlConfig {
+    type Builder = MysqlBuilder;
+    fn into_builder(self) -> Self::Builder {
+        MysqlBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct MysqlBuilder {
+    config: MysqlConfig,
+}
+
+impl Debug for MysqlBuilder {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("MysqlBuilder");
+
+        d.field("config", &self.config).finish()
+    }
+}
+
+impl MysqlBuilder {
+    /// Set the connection_string of the mysql service.
+    ///
+    /// This connection string is used to connect to the mysql service. There are url based formats:
+    ///
+    /// ## Url
+    ///
+    /// This format resembles the url format of the mysql client. The format is: `[scheme://][user[:[password]]@]host[:port][/schema][?attribute1=value1&attribute2=value2...`
+    ///
+    /// - `mysql://user@localhost`
+    /// - `mysql://user:password@localhost`
+    /// - `mysql://user:password@localhost:3306`
+    /// - `mysql://user:password@localhost:3306/db`
+    ///
+    /// For more information, please refer to <https://docs.rs/sqlx/latest/sqlx/mysql/struct.MySqlConnectOptions.html>.
+    pub fn connection_string(mut self, v: &str) -> Self {
+        if !v.is_empty() {
+            self.config.connection_string = Some(v.to_string());
+        }
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the table name of the mysql service to read/write.
+    pub fn table(mut self, table: &str) -> Self {
+        if !table.is_empty() {
+            self.config.table = Some(table.to_string());
+        }
+        self
+    }
+
+    /// Set the key field name of the mysql service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub fn key_field(mut self, key_field: &str) -> Self {
+        if !key_field.is_empty() {
+            self.config.key_field = Some(key_field.to_string());
+        }
+        self
+    }
+
+    /// Set the value field name of the mysql service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub fn value_field(mut self, value_field: &str) -> Self {
+        if !value_field.is_empty() {
+            self.config.value_field = Some(value_field.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for MysqlBuilder {
+    type Config = MysqlConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let conn = match self.config.connection_string {
+            Some(v) => v,
+            None => {
+                return Err(
+                    Error::new(ErrorKind::ConfigInvalid, "connection_string is empty")
+                        .with_context("service", Scheme::Mysql),
+                )
+            }
+        };
+
+        let config = MySqlConnectOptions::from_str(&conn).map_err(|err| {
+            Error::new(ErrorKind::ConfigInvalid, "connection_string is invalid")
+                .with_context("service", Scheme::Mysql)
+                .set_source(err)
+        })?;
+
+        let table = match self.config.table {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "table is empty")
+                    .with_context("service", Scheme::Mysql))
+            }
+        };
+
+        let key_field = self.config.key_field.unwrap_or_else(|| "key".to_string());
+
+        let value_field = self
+            .config
+            .value_field
+            .unwrap_or_else(|| "value".to_string());
+
+        let root = normalize_root(self.config.root.unwrap_or_else(|| "/".to_string()).as_str());
+
+        Ok(MySqlBackend::new(Adapter {
+            pool: OnceCell::new(),
+            config,
+            table,
+            key_field,
+            value_field,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+/// Backend for mysql service
+pub type MySqlBackend = kv::Backend<Adapter>;
+
+#[derive(Debug, Clone)]
+pub struct Adapter {
+    pool: OnceCell<MySqlPool>,
+    config: MySqlConnectOptions,
+
+    table: String,
+    key_field: String,
+    value_field: String,
+}
+
+impl Adapter {
+    async fn get_client(&self) -> Result<&MySqlPool> {
+        self.pool
+            .get_or_try_init(|| async {
+                let pool = MySqlPool::connect_with(self.config.clone())
+                    .await
+                    .map_err(parse_mysql_error)?;
+                Ok(pool)
+            })
+            .await
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Mysql,
+            &self.table,
+            Capability {
+                read: true,
+                write: true,
+                delete: true,
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let pool = self.get_client().await?;
+
+        let value: Option<Vec<u8>> = sqlx::query_scalar(&format!(
+            "SELECT `{}` FROM `{}` WHERE `{}` = ? LIMIT 1",
+            self.value_field, self.table, self.key_field
+        ))
+        .bind(path)
+        .fetch_optional(pool)
+        .await
+        .map_err(parse_mysql_error)?;
+
+        Ok(value.map(Buffer::from))
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        sqlx::query(&format!(
+            r#"INSERT INTO `{}` (`{}`, `{}`) VALUES (?, ?)
+            ON DUPLICATE KEY UPDATE `{}` = VALUES({})"#,
+            self.table, self.key_field, self.value_field, self.value_field, self.value_field
+        ))
+        .bind(path)
+        .bind(value.to_vec())
+        .execute(pool)
+        .await
+        .map_err(parse_mysql_error)?;
+
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let pool = self.get_client().await?;
+
+        sqlx::query(&format!(
+            "DELETE FROM `{}` WHERE `{}` = ?",
+            self.table, self.key_field
+        ))
+        .bind(path)
+        .execute(pool)
+        .await
+        .map_err(parse_mysql_error)?;
+
+        Ok(())
+    }
+}
+
+fn parse_mysql_error(err: sqlx::Error) -> Error {
+    Error::new(ErrorKind::Unexpected, "unhandled error from mysql").set_source(err)
+}

data/core/src/services/mysql/config.rs

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Mysql services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct MysqlConfig {
+    /// This connection string is used to connect to the mysql service. There are url based formats.
+    ///
+    /// The format of connect string resembles the url format of the mysql client.
+    /// The format is: `[scheme://][user[:[password]]@]host[:port][/schema][?attribute1=value1&attribute2=value2...`
+    ///
+    /// - `mysql://user@localhost`
+    /// - `mysql://user:password@localhost`
+    /// - `mysql://user:password@localhost:3306`
+    /// - `mysql://user:password@localhost:3306/db`
+    ///
+    /// For more information, please refer to <https://docs.rs/sqlx/latest/sqlx/mysql/struct.MySqlConnectOptions.html>.
+    pub connection_string: Option<String>,
+
+    /// The table name for mysql.
+    pub table: Option<String>,
+    /// The key field name for mysql.
+    pub key_field: Option<String>,
+    /// The value field name for mysql.
+    pub value_field: Option<String>,
+    /// The root for mysql.
+    pub root: Option<String>,
+}
+
+impl Debug for MysqlConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("MysqlConfig");
+
+        if self.connection_string.is_some() {
+            d.field("connection_string", &"<redacted>");
+        }
+
+        d.field("root", &self.root)
+            .field("table", &self.table)
+            .field("key_field", &self.key_field)
+            .field("value_field", &self.value_field)
+            .finish()
+    }
+}