opendal 0.1.6.pre.rc.1

data/core/src/services/memcached/backend.rs

@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::time::Duration;
+
+use bb8::RunError;
+use tokio::net::TcpStream;
+use tokio::sync::OnceCell;
+
+use super::binary;
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::MemcachedConfig;
+use crate::*;
+
+impl Configurator for MemcachedConfig {
+    type Builder = MemcachedBuilder;
+    fn into_builder(self) -> Self::Builder {
+        MemcachedBuilder { config: self }
+    }
+}
+
+/// [Memcached](https://memcached.org/) service support.
+#[doc = include_str!("docs.md")]
+#[derive(Clone, Default)]
+pub struct MemcachedBuilder {
+    config: MemcachedConfig,
+}
+
+impl MemcachedBuilder {
+    /// set the network address of memcached service.
+    ///
+    /// For example: "tcp://localhost:11211"
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        if !endpoint.is_empty() {
+            self.config.endpoint = Some(endpoint.to_owned());
+        }
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// set the username.
+    pub fn username(mut self, username: &str) -> Self {
+        self.config.username = Some(username.to_string());
+        self
+    }
+
+    /// set the password.
+    pub fn password(mut self, password: &str) -> Self {
+        self.config.password = Some(password.to_string());
+        self
+    }
+
+    /// Set the default ttl for memcached services.
+    pub fn default_ttl(mut self, ttl: Duration) -> Self {
+        self.config.default_ttl = Some(ttl);
+        self
+    }
+}
+
+impl Builder for MemcachedBuilder {
+    type Config = MemcachedConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let endpoint = self.config.endpoint.clone().ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                .with_context("service", Scheme::Memcached)
+        })?;
+        let uri = http::Uri::try_from(&endpoint).map_err(|err| {
+            Error::new(ErrorKind::ConfigInvalid, "endpoint is invalid")
+                .with_context("service", Scheme::Memcached)
+                .with_context("endpoint", &endpoint)
+                .set_source(err)
+        })?;
+
+        match uri.scheme_str() {
+            // If scheme is none, we will use tcp by default.
+            None => (),
+            Some(scheme) => {
+                // We only support tcp by now.
+                if scheme != "tcp" {
+                    return Err(Error::new(
+                        ErrorKind::ConfigInvalid,
+                        "endpoint is using invalid scheme",
+                    )
+                    .with_context("service", Scheme::Memcached)
+                    .with_context("endpoint", &endpoint)
+                    .with_context("scheme", scheme.to_string()));
+                }
+            }
+        };
+
+        let host = if let Some(host) = uri.host() {
+            host.to_string()
+        } else {
+            return Err(
+                Error::new(ErrorKind::ConfigInvalid, "endpoint doesn't have host")
+                    .with_context("service", Scheme::Memcached)
+                    .with_context("endpoint", &endpoint),
+            );
+        };
+        let port = if let Some(port) = uri.port_u16() {
+            port
+        } else {
+            return Err(
+                Error::new(ErrorKind::ConfigInvalid, "endpoint doesn't have port")
+                    .with_context("service", Scheme::Memcached)
+                    .with_context("endpoint", &endpoint),
+            );
+        };
+        let endpoint = format!("{host}:{port}",);
+
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+
+        let conn = OnceCell::new();
+        Ok(MemcachedBackend::new(Adapter {
+            endpoint,
+            username: self.config.username.clone(),
+            password: self.config.password.clone(),
+            conn,
+            default_ttl: self.config.default_ttl,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+/// Backend for memcached services.
+pub type MemcachedBackend = kv::Backend<Adapter>;
+
+#[derive(Clone, Debug)]
+pub struct Adapter {
+    endpoint: String,
+    username: Option<String>,
+    password: Option<String>,
+    default_ttl: Option<Duration>,
+    conn: OnceCell<bb8::Pool<MemcacheConnectionManager>>,
+}
+
+impl Adapter {
+    async fn conn(&self) -> Result<bb8::PooledConnection<'_, MemcacheConnectionManager>> {
+        let pool = self
+            .conn
+            .get_or_try_init(|| async {
+                let mgr = MemcacheConnectionManager::new(
+                    &self.endpoint,
+                    self.username.clone(),
+                    self.password.clone(),
+                );
+
+                bb8::Pool::builder().build(mgr).await.map_err(|err| {
+                    Error::new(ErrorKind::ConfigInvalid, "connect to memecached failed")
+                        .set_source(err)
+                })
+            })
+            .await?;
+
+        pool.get().await.map_err(|err| match err {
+            RunError::TimedOut => {
+                Error::new(ErrorKind::Unexpected, "get connection from pool failed").set_temporary()
+            }
+            RunError::User(err) => err,
+        })
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Memcached,
+            "memcached",
+            Capability {
+                read: true,
+                write: true,
+                shared: true,
+
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, key: &str) -> Result<Option<Buffer>> {
+        let mut conn = self.conn().await?;
+        let result = conn.get(&percent_encode_path(key)).await?;
+        Ok(result.map(Buffer::from))
+    }
+
+    async fn set(&self, key: &str, value: Buffer) -> Result<()> {
+        let mut conn = self.conn().await?;
+
+        conn.set(
+            &percent_encode_path(key),
+            &value.to_vec(),
+            // Set expiration to 0 if ttl not set.
+            self.default_ttl
+                .map(|v| v.as_secs() as u32)
+                .unwrap_or_default(),
+        )
+        .await
+    }
+
+    async fn delete(&self, key: &str) -> Result<()> {
+        let mut conn = self.conn().await?;
+
+        conn.delete(&percent_encode_path(key)).await
+    }
+}
+
+/// A `bb8::ManageConnection` for `memcache_async::ascii::Protocol`.
+#[derive(Clone, Debug)]
+struct MemcacheConnectionManager {
+    address: String,
+    username: Option<String>,
+    password: Option<String>,
+}
+
+impl MemcacheConnectionManager {
+    fn new(address: &str, username: Option<String>, password: Option<String>) -> Self {
+        Self {
+            address: address.to_string(),
+            username,
+            password,
+        }
+    }
+}
+
+impl bb8::ManageConnection for MemcacheConnectionManager {
+    type Connection = binary::Connection;
+    type Error = Error;
+
+    /// TODO: Implement unix stream support.
+    async fn connect(&self) -> Result<Self::Connection, Self::Error> {
+        let conn = TcpStream::connect(&self.address)
+            .await
+            .map_err(new_std_io_error)?;
+        let mut conn = binary::Connection::new(conn);
+
+        if let (Some(username), Some(password)) = (self.username.as_ref(), self.password.as_ref()) {
+            conn.auth(username, password).await?;
+        }
+        Ok(conn)
+    }
+
+    async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> {
+        conn.version().await.map(|_| ())
+    }
+
+    fn has_broken(&self, _: &mut Self::Connection) -> bool {
+        false
+    }
+}

data/core/src/services/memcached/binary.rs

@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use tokio::io::AsyncReadExt;
+use tokio::io::AsyncWriteExt;
+use tokio::io::BufReader;
+use tokio::io::{self};
+use tokio::net::TcpStream;
+
+use crate::raw::*;
+use crate::*;
+
+pub(super) mod constants {
+    pub const OK_STATUS: u16 = 0x0;
+    pub const KEY_NOT_FOUND: u16 = 0x1;
+}
+
+pub enum Opcode {
+    Get = 0x00,
+    Set = 0x01,
+    Delete = 0x04,
+    Version = 0x0b,
+    StartAuth = 0x21,
+}
+
+pub enum Magic {
+    Request = 0x80,
+}
+
+#[derive(Debug)]
+pub struct StoreExtras {
+    pub flags: u32,
+    pub expiration: u32,
+}
+
+#[derive(Debug, Default)]
+pub struct PacketHeader {
+    pub magic: u8,
+    pub opcode: u8,
+    pub key_length: u16,
+    pub extras_length: u8,
+    pub data_type: u8,
+    pub vbucket_id_or_status: u16,
+    pub total_body_length: u32,
+    pub opaque: u32,
+    pub cas: u64,
+}
+
+impl PacketHeader {
+    pub async fn write(self, writer: &mut TcpStream) -> io::Result<()> {
+        writer.write_u8(self.magic).await?;
+        writer.write_u8(self.opcode).await?;
+        writer.write_u16(self.key_length).await?;
+        writer.write_u8(self.extras_length).await?;
+        writer.write_u8(self.data_type).await?;
+        writer.write_u16(self.vbucket_id_or_status).await?;
+        writer.write_u32(self.total_body_length).await?;
+        writer.write_u32(self.opaque).await?;
+        writer.write_u64(self.cas).await?;
+        Ok(())
+    }
+
+    pub async fn read(reader: &mut TcpStream) -> Result<PacketHeader, io::Error> {
+        let header = PacketHeader {
+            magic: reader.read_u8().await?,
+            opcode: reader.read_u8().await?,
+            key_length: reader.read_u16().await?,
+            extras_length: reader.read_u8().await?,
+            data_type: reader.read_u8().await?,
+            vbucket_id_or_status: reader.read_u16().await?,
+            total_body_length: reader.read_u32().await?,
+            opaque: reader.read_u32().await?,
+            cas: reader.read_u64().await?,
+        };
+        Ok(header)
+    }
+}
+
+pub struct Response {
+    header: PacketHeader,
+    _key: Vec<u8>,
+    _extras: Vec<u8>,
+    value: Vec<u8>,
+}
+
+pub struct Connection {
+    io: BufReader<TcpStream>,
+}
+
+impl Connection {
+    pub fn new(io: TcpStream) -> Self {
+        Self {
+            io: BufReader::new(io),
+        }
+    }
+
+    pub async fn auth(&mut self, username: &str, password: &str) -> Result<()> {
+        let writer = self.io.get_mut();
+        let key = "PLAIN";
+        let request_header = PacketHeader {
+            magic: Magic::Request as u8,
+            opcode: Opcode::StartAuth as u8,
+            key_length: key.len() as u16,
+            total_body_length: (key.len() + username.len() + password.len() + 2) as u32,
+            ..Default::default()
+        };
+        request_header
+            .write(writer)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_all(key.as_bytes())
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_all(format!("\x00{username}\x00{password}").as_bytes())
+            .await
+            .map_err(new_std_io_error)?;
+        writer.flush().await.map_err(new_std_io_error)?;
+        parse_response(writer).await?;
+        Ok(())
+    }
+
+    pub async fn version(&mut self) -> Result<String> {
+        let writer = self.io.get_mut();
+        let request_header = PacketHeader {
+            magic: Magic::Request as u8,
+            opcode: Opcode::Version as u8,
+            ..Default::default()
+        };
+        request_header
+            .write(writer)
+            .await
+            .map_err(new_std_io_error)?;
+        writer.flush().await.map_err(new_std_io_error)?;
+        let response = parse_response(writer).await?;
+        let version = String::from_utf8(response.value);
+        match version {
+            Ok(version) => Ok(version),
+            Err(e) => {
+                Err(Error::new(ErrorKind::Unexpected, "unexpected data received").set_source(e))
+            }
+        }
+    }
+
+    pub async fn get(&mut self, key: &str) -> Result<Option<Vec<u8>>> {
+        let writer = self.io.get_mut();
+        let request_header = PacketHeader {
+            magic: Magic::Request as u8,
+            opcode: Opcode::Get as u8,
+            key_length: key.len() as u16,
+            total_body_length: key.len() as u32,
+            ..Default::default()
+        };
+        request_header
+            .write(writer)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_all(key.as_bytes())
+            .await
+            .map_err(new_std_io_error)?;
+        writer.flush().await.map_err(new_std_io_error)?;
+        match parse_response(writer).await {
+            Ok(response) => {
+                if response.header.vbucket_id_or_status == 0x1 {
+                    return Ok(None);
+                }
+                Ok(Some(response.value))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    pub async fn set(&mut self, key: &str, val: &[u8], expiration: u32) -> Result<()> {
+        let writer = self.io.get_mut();
+        let request_header = PacketHeader {
+            magic: Magic::Request as u8,
+            opcode: Opcode::Set as u8,
+            key_length: key.len() as u16,
+            extras_length: 8,
+            total_body_length: (8 + key.len() + val.len()) as u32,
+            ..Default::default()
+        };
+        let extras = StoreExtras {
+            flags: 0,
+            expiration,
+        };
+        request_header
+            .write(writer)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_u32(extras.flags)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_u32(extras.expiration)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_all(key.as_bytes())
+            .await
+            .map_err(new_std_io_error)?;
+        writer.write_all(val).await.map_err(new_std_io_error)?;
+        writer.flush().await.map_err(new_std_io_error)?;
+
+        parse_response(writer).await?;
+        Ok(())
+    }
+
+    pub async fn delete(&mut self, key: &str) -> Result<()> {
+        let writer = self.io.get_mut();
+        let request_header = PacketHeader {
+            magic: Magic::Request as u8,
+            opcode: Opcode::Delete as u8,
+            key_length: key.len() as u16,
+            total_body_length: key.len() as u32,
+            ..Default::default()
+        };
+        request_header
+            .write(writer)
+            .await
+            .map_err(new_std_io_error)?;
+        writer
+            .write_all(key.as_bytes())
+            .await
+            .map_err(new_std_io_error)?;
+        writer.flush().await.map_err(new_std_io_error)?;
+        parse_response(writer).await?;
+        Ok(())
+    }
+}
+
+pub async fn parse_response(reader: &mut TcpStream) -> Result<Response> {
+    let header = PacketHeader::read(reader).await.map_err(new_std_io_error)?;
+
+    if header.vbucket_id_or_status != constants::OK_STATUS
+        && header.vbucket_id_or_status != constants::KEY_NOT_FOUND
+    {
+        return Err(
+            Error::new(ErrorKind::Unexpected, "unexpected status received")
+                .with_context("message", format!("{}", header.vbucket_id_or_status)),
+        );
+    }
+
+    let mut extras = vec![0x0; header.extras_length as usize];
+    reader
+        .read_exact(extras.as_mut_slice())
+        .await
+        .map_err(new_std_io_error)?;
+
+    let mut key = vec![0x0; header.key_length as usize];
+    reader
+        .read_exact(key.as_mut_slice())
+        .await
+        .map_err(new_std_io_error)?;
+
+    let mut value = vec![
+        0x0;
+        (header.total_body_length - u32::from(header.key_length) - u32::from(header.extras_length))
+            as usize
+    ];
+    reader
+        .read_exact(value.as_mut_slice())
+        .await
+        .map_err(new_std_io_error)?;
+
+    Ok(Response {
+        header,
+        _key: key,
+        _extras: extras,
+        value,
+    })
+}

data/core/src/services/memcached/config.rs

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::time::Duration;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for MemCached services support
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct MemcachedConfig {
+    /// network address of the memcached service.
+    ///
+    /// For example: "tcp://localhost:11211"
+    pub endpoint: Option<String>,
+    /// the working directory of the service. Can be "/path/to/dir"
+    ///
+    /// default is "/"
+    pub root: Option<String>,
+    /// Memcached username, optional.
+    pub username: Option<String>,
+    /// Memcached password, optional.
+    pub password: Option<String>,
+    /// The default ttl for put operations.
+    pub default_ttl: Option<Duration>,
+}

data/core/src/services/memcached/docs.md

@@ -0,0 +1,47 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [ ] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `username`: Set the username for authentication.
+- `password`: Set the password for authentication.
+- `endpoint`: Set the network address of memcached server
+- `default_ttl`: Set the ttl for memcached service.
+
+You can refer to [`MemcachedBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Memcached;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create memcached backend builder
+    let mut builder = Memcached::default()
+        .endpoint("tcp://127.0.0.1:11211");
+        // if you enable authentication, set username and password for authentication
+        // builder.username("admin")
+        // builder.password("password");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/memcached/mod.rs

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-memcached")]
+mod binary;
+
+#[cfg(feature = "services-memcached")]
+mod backend;
+#[cfg(feature = "services-memcached")]
+pub use backend::MemcachedBuilder as Memcached;
+
+mod config;
+pub use config::MemcachedConfig;