opendal 0.1.6.pre.rc.1

data/core/src/services/mini_moka/backend.rs

@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+use std::time::Duration;
+
+use log::debug;
+
+use super::core::*;
+use super::delete::MiniMokaDeleter;
+use super::lister::MiniMokaLister;
+use super::writer::MiniMokaWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::oio;
+use crate::raw::oio::HierarchyLister;
+use crate::raw::*;
+use crate::services::MiniMokaConfig;
+use crate::*;
+impl Configurator for MiniMokaConfig {
+    type Builder = MiniMokaBuilder;
+    fn into_builder(self) -> Self::Builder {
+        MiniMokaBuilder { config: self }
+    }
+}
+
+/// [mini-moka](https://github.com/moka-rs/mini-moka) backend support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct MiniMokaBuilder {
+    config: MiniMokaConfig,
+}
+
+impl Debug for MiniMokaBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MiniMokaBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl MiniMokaBuilder {
+    /// Create a [`MiniMokaBuilder`] with default configuration.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Sets the max capacity of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity)
+    pub fn max_capacity(mut self, v: u64) -> Self {
+        if v != 0 {
+            self.config.max_capacity = Some(v);
+        }
+        self
+    }
+
+    /// Sets the time to live of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)
+    pub fn time_to_live(mut self, v: Duration) -> Self {
+        if !v.is_zero() {
+            self.config.time_to_live = Some(v);
+        }
+        self
+    }
+
+    /// Sets the time to idle of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle)
+    pub fn time_to_idle(mut self, v: Duration) -> Self {
+        if !v.is_zero() {
+            self.config.time_to_idle = Some(v);
+        }
+        self
+    }
+
+    /// Set root path of this backend
+    pub fn root(mut self, path: &str) -> Self {
+        self.config.root = if path.is_empty() {
+            None
+        } else {
+            Some(path.to_string())
+        };
+
+        self
+    }
+}
+
+impl Builder for MiniMokaBuilder {
+    type Config = MiniMokaConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let mut builder: mini_moka::sync::CacheBuilder<String, MiniMokaValue, _> =
+            mini_moka::sync::Cache::builder();
+
+        // Use entries' bytes as capacity weigher.
+        builder = builder.weigher(|k, v| (k.len() + v.content.len()) as u32);
+
+        if let Some(v) = self.config.max_capacity {
+            builder = builder.max_capacity(v);
+        }
+        if let Some(v) = self.config.time_to_live {
+            builder = builder.time_to_live(v);
+        }
+        if let Some(v) = self.config.time_to_idle {
+            builder = builder.time_to_idle(v);
+        }
+
+        let cache = builder.build();
+
+        let root = normalize_root(self.config.root.as_deref().unwrap_or("/"));
+
+        let core = Arc::new(MiniMokaCore { cache });
+
+        debug!("backend build finished: {root}");
+        Ok(MiniMokaBackend::new(core, root))
+    }
+}
+
+#[derive(Debug)]
+struct MiniMokaBackend {
+    core: Arc<MiniMokaCore>,
+    root: String,
+}
+
+impl MiniMokaBackend {
+    fn new(core: Arc<MiniMokaCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl Access for MiniMokaBackend {
+    type Reader = Buffer;
+    type Writer = MiniMokaWriter;
+    type Lister = HierarchyLister<MiniMokaLister>;
+    type Deleter = oio::OneShotDeleter<MiniMokaDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME)
+            .set_root(&self.root)
+            .set_native_capability(Capability {
+                stat: true,
+                read: true,
+                write: true,
+                write_can_empty: true,
+                delete: true,
+                list: true,
+
+                ..Default::default()
+            });
+
+        Arc::new(info)
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let p = build_abs_path(&self.root, path);
+
+        // Check if path exists directly in cache
+        match self.core.get(&p) {
+            Some(value) => {
+                let mut metadata = value.metadata.clone();
+                if p.ends_with('/') {
+                    metadata.set_mode(EntryMode::DIR);
+                } else {
+                    metadata.set_mode(EntryMode::FILE);
+                }
+                Ok(RpStat::new(metadata))
+            }
+            None => {
+                if p.ends_with('/') {
+                    let is_prefix = self
+                        .core
+                        .cache
+                        .iter()
+                        .any(|entry| entry.key().starts_with(&p) && entry.key() != &p);
+
+                    if is_prefix {
+                        let mut metadata = Metadata::default();
+                        metadata.set_mode(EntryMode::DIR);
+                        return Ok(RpStat::new(metadata));
+                    }
+                }
+
+                Err(Error::new(ErrorKind::NotFound, "path not found"))
+            }
+        }
+    }
+
+    async fn read(&self, path: &str, op: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let p = build_abs_path(&self.root, path);
+
+        match self.core.get(&p) {
+            Some(value) => {
+                let range = op.range();
+
+                // If range is full, return the content buffer directly
+                if range.is_full() {
+                    return Ok((RpRead::new(), value.content));
+                }
+
+                let offset = range.offset() as usize;
+                if offset >= value.content.len() {
+                    return Err(Error::new(
+                        ErrorKind::RangeNotSatisfied,
+                        "range start offset exceeds content length",
+                    ));
+                }
+
+                let size = range.size().map(|s| s as usize);
+                let end = size.map_or(value.content.len(), |s| {
+                    (offset + s).min(value.content.len())
+                });
+                let sliced_content = value.content.slice(offset..end);
+
+                Ok((RpRead::new(), sliced_content))
+            }
+            None => Err(Error::new(ErrorKind::NotFound, "path not found")),
+        }
+    }
+
+    async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let p = build_abs_path(&self.root, path);
+        let writer = MiniMokaWriter::new(self.core.clone(), p, op);
+        Ok((RpWrite::new(), writer))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        let deleter =
+            oio::OneShotDeleter::new(MiniMokaDeleter::new(self.core.clone(), self.root.clone()));
+        Ok((RpDelete::default(), deleter))
+    }
+
+    async fn list(&self, path: &str, op: OpList) -> Result<(RpList, Self::Lister)> {
+        let p = build_abs_path(&self.root, path);
+
+        let mini_moka_lister = MiniMokaLister::new(self.core.clone(), self.root.clone(), p);
+        let lister = HierarchyLister::new(mini_moka_lister, path, op.recursive());
+
+        Ok((RpList::default(), lister))
+    }
+}

data/core/src/services/mini_moka/config.rs

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::time::Duration;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for mini-moka support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct MiniMokaConfig {
+    /// Sets the max capacity of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::max_capacity`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.max_capacity)
+    pub max_capacity: Option<u64>,
+    /// Sets the time to live of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)
+    pub time_to_live: Option<Duration>,
+    /// Sets the time to idle of the cache.
+    ///
+    /// Refer to [`mini-moka::sync::CacheBuilder::time_to_idle`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_idle)
+    pub time_to_idle: Option<Duration>,
+
+    /// root path of this backend
+    pub root: Option<String>,
+}
+
+impl Debug for MiniMokaConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MiniMokaConfig")
+            .field("max_capacity", &self.max_capacity)
+            .field("time_to_live", &self.time_to_live)
+            .field("time_to_idle", &self.time_to_idle)
+            .field("root", &self.root)
+            .finish()
+    }
+}

data/core/src/services/mini_moka/core.rs

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use mini_moka::sync::Cache;
+
+use crate::*;
+
+/// Value stored in mini-moka cache containing both metadata and content
+#[derive(Clone)]
+pub struct MiniMokaValue {
+    /// Stored metadata in mini-moka cache.
+    pub metadata: Metadata,
+    /// Stored content in mini-moka cache.
+    pub content: Buffer,
+}
+
+#[derive(Clone)]
+pub struct MiniMokaCore {
+    pub cache: Cache<String, MiniMokaValue>,
+}
+
+impl Debug for MiniMokaCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MiniMokaCore")
+            .field("size", &self.cache.weighted_size())
+            .field("count", &self.cache.entry_count())
+            .finish()
+    }
+}
+
+impl MiniMokaCore {
+    pub fn get(&self, key: &str) -> Option<MiniMokaValue> {
+        self.cache.get(&key.to_string())
+    }
+}

data/core/src/services/mini_moka/delete.rs

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::MiniMokaCore;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MiniMokaDeleter {
+    core: Arc<MiniMokaCore>,
+    root: String,
+}
+
+impl MiniMokaDeleter {
+    pub fn new(core: Arc<MiniMokaCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl oio::OneShotDelete for MiniMokaDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = build_abs_path(&self.root, &path);
+        self.core.cache.invalidate(&p);
+        Ok(())
+    }
+}

data/core/src/services/mini_moka/docs.md

@@ -0,0 +1,19 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] presign
+- [ ] blocking
+
+## Notes
+
+To better assist you in choosing the right cache for your use case,
+Here's a comparison table with [moka](https://github.com/moka-rs/moka#choosing-the-right-cache-for-your-use-case)

data/core/src/services/mini_moka/lister.rs

@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::vec::IntoIter;
+
+use super::core::MiniMokaCore;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MiniMokaLister {
+    root: String,
+    keys: IntoIter<String>,
+}
+
+impl MiniMokaLister {
+    pub fn new(core: Arc<MiniMokaCore>, root: String, _path: String) -> Self {
+        // Get all keys from the cache
+        let keys: Vec<String> = core
+            .cache
+            .iter()
+            .map(|entry| entry.key().to_string())
+            .collect();
+
+        Self {
+            root,
+            keys: keys.into_iter(),
+        }
+    }
+}
+
+impl oio::List for MiniMokaLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        match self.keys.next() {
+            Some(key) => {
+                // Convert absolute path to relative path
+                let rel_path = build_rel_path(&self.root, &key);
+
+                // Determine if it's a file or directory based on trailing slash
+                let mode = if key.ends_with('/') {
+                    EntryMode::DIR
+                } else {
+                    EntryMode::FILE
+                };
+
+                let metadata = Metadata::new(mode);
+
+                Ok(Some(oio::Entry::new(&rel_path, metadata)))
+            }
+            None => Ok(None),
+        }
+    }
+}

data/core/src/services/mini_moka/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for mini_moka service.
+#[cfg(feature = "services-mini-moka")]
+pub(super) const DEFAULT_SCHEME: &str = "mini_moka";
+#[cfg(feature = "services-mini-moka")]
+mod backend;
+#[cfg(feature = "services-mini-moka")]
+mod core;
+#[cfg(feature = "services-mini-moka")]
+mod delete;
+#[cfg(feature = "services-mini-moka")]
+mod lister;
+#[cfg(feature = "services-mini-moka")]
+mod writer;
+
+#[cfg(feature = "services-mini-moka")]
+pub use backend::MiniMokaBuilder as MiniMoka;
+
+mod config;
+pub use config::MiniMokaConfig;

data/core/src/services/mini_moka/writer.rs

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::{MiniMokaCore, MiniMokaValue};
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MiniMokaWriter {
+    core: Arc<MiniMokaCore>,
+    path: String,
+    op: OpWrite,
+    buffer: oio::QueueBuf,
+}
+
+impl MiniMokaWriter {
+    pub fn new(core: Arc<MiniMokaCore>, path: String, op: OpWrite) -> Self {
+        Self {
+            core,
+            path,
+            op,
+            buffer: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for MiniMokaWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buffer.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buffer.clone().collect();
+
+        let mut md = Metadata::new(EntryMode::from_path(&self.path));
+        md.set_content_length(buf.len() as u64);
+        md.set_last_modified(chrono::Utc::now());
+
+        // Set metadata from OpWrite
+        if let Some(content_type) = self.op.content_type() {
+            md.set_content_type(content_type);
+        }
+        if let Some(content_disposition) = self.op.content_disposition() {
+            md.set_content_disposition(content_disposition);
+        }
+        if let Some(content_encoding) = self.op.content_encoding() {
+            md.set_content_encoding(content_encoding);
+        }
+        if let Some(cache_control) = self.op.cache_control() {
+            md.set_cache_control(cache_control);
+        }
+
+        let value = MiniMokaValue {
+            metadata: md.clone(),
+            content: buf,
+        };
+
+        self.core.cache.insert(self.path.clone(), value);
+
+        Ok(md)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buffer.clear();
+        Ok(())
+    }
+}