opendal 0.1.6.pre.rc.1

data/core/src/services/cacache/backend.rs

@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use chrono::DateTime;
+
+use crate::raw::*;
+use crate::services::CacacheConfig;
+use crate::*;
+
+use super::core::CacacheCore;
+use super::delete::CacacheDeleter;
+use super::writer::CacacheWriter;
+use super::DEFAULT_SCHEME;
+impl Configurator for CacacheConfig {
+    type Builder = CacacheBuilder;
+    fn into_builder(self) -> Self::Builder {
+        CacacheBuilder { config: self }
+    }
+}
+
+/// cacache service support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct CacacheBuilder {
+    config: CacacheConfig,
+}
+
+impl CacacheBuilder {
+    /// Set the path to the cacache data directory. Will create if not exists.
+    pub fn datadir(mut self, path: &str) -> Self {
+        self.config.datadir = Some(path.into());
+        self
+    }
+}
+
+impl Builder for CacacheBuilder {
+    type Config = CacacheConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let datadir_path = self.config.datadir.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set")
+                .with_context("service", Scheme::Cacache)
+        })?;
+
+        let core = CacacheCore {
+            path: datadir_path.clone(),
+        };
+
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME);
+        info.set_name(&datadir_path);
+        info.set_root("/");
+        info.set_native_capability(Capability {
+            read: true,
+            write: true,
+            delete: true,
+            stat: true,
+            rename: false,
+            list: false,
+            shared: false,
+            ..Default::default()
+        });
+
+        Ok(CacacheAccessor {
+            core: Arc::new(core),
+            info: Arc::new(info),
+        })
+    }
+}
+
+/// Backend for cacache services.
+#[derive(Debug, Clone)]
+pub struct CacacheAccessor {
+    core: Arc<CacacheCore>,
+    info: Arc<AccessorInfo>,
+}
+
+impl Access for CacacheAccessor {
+    type Reader = Buffer;
+    type Writer = CacacheWriter;
+    type Lister = ();
+    type Deleter = oio::OneShotDeleter<CacacheDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let metadata = self.core.metadata(path).await?;
+
+        match metadata {
+            Some(meta) => {
+                let mut md = Metadata::new(EntryMode::FILE);
+                md.set_content_length(meta.size as u64);
+                // Convert u128 milliseconds to DateTime<Utc>
+                let millis = meta.time;
+                let secs = (millis / 1000) as i64;
+                let nanos = ((millis % 1000) * 1_000_000) as u32;
+                if let Some(dt) = DateTime::from_timestamp(secs, nanos) {
+                    md.set_last_modified(dt);
+                }
+                Ok(RpStat::new(md))
+            }
+            None => Err(Error::new(ErrorKind::NotFound, "entry not found")),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let data = self.core.get(path).await?;
+
+        match data {
+            Some(bytes) => {
+                let range = args.range();
+                let buffer = if range.is_full() {
+                    Buffer::from(bytes)
+                } else {
+                    let start = range.offset() as usize;
+                    let end = match range.size() {
+                        Some(size) => (range.offset() + size) as usize,
+                        None => bytes.len(),
+                    };
+                    Buffer::from(bytes.slice(start..end.min(bytes.len())))
+                };
+                Ok((RpRead::new(), buffer))
+            }
+            None => Err(Error::new(ErrorKind::NotFound, "entry not found")),
+        }
+    }
+
+    async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        Ok((
+            RpWrite::new(),
+            CacacheWriter::new(self.core.clone(), path.to_string()),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(CacacheDeleter::new(self.core.clone())),
+        ))
+    }
+}

data/core/src/services/cacache/config.rs

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// cacache service support.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct CacacheConfig {
+    /// That path to the cacache data directory.
+    pub datadir: Option<String>,
+}

data/core/src/services/cacache/core.rs

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use crate::*;
+
+#[derive(Clone)]
+pub struct CacacheCore {
+    pub path: String,
+}
+
+impl Debug for CacacheCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CacacheCore")
+            .field("path", &self.path)
+            .finish()
+    }
+}
+
+impl CacacheCore {
+    pub async fn get(&self, key: &str) -> Result<Option<bytes::Bytes>> {
+        let cache_path = self.path.clone();
+        let cache_key = key.to_string();
+
+        let data = cacache::read(&cache_path, &cache_key).await;
+
+        match data {
+            Ok(bs) => Ok(Some(bytes::Bytes::from(bs))),
+            Err(cacache::Error::EntryNotFound(_, _)) => Ok(None),
+            Err(err) => Err(Error::new(ErrorKind::Unexpected, "cacache get failed")
+                .with_operation("CacacheCore::get")
+                .set_source(err)),
+        }
+    }
+
+    pub async fn set(&self, key: &str, value: bytes::Bytes) -> Result<()> {
+        let cache_path = self.path.clone();
+        let cache_key = key.to_string();
+
+        cacache::write(&cache_path, &cache_key, value.to_vec())
+            .await
+            .map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "cacache set failed")
+                    .with_operation("CacacheCore::set")
+                    .set_source(err)
+            })?;
+
+        Ok(())
+    }
+
+    pub async fn delete(&self, key: &str) -> Result<()> {
+        let cache_path = self.path.clone();
+        let cache_key = key.to_string();
+
+        cacache::remove(&cache_path, &cache_key)
+            .await
+            .map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "cacache delete failed")
+                    .with_operation("CacacheCore::delete")
+                    .set_source(err)
+            })?;
+
+        Ok(())
+    }
+
+    pub async fn metadata(&self, key: &str) -> Result<Option<cacache::Metadata>> {
+        let cache_path = self.path.clone();
+        let cache_key = key.to_string();
+
+        let metadata = cacache::metadata(&cache_path, &cache_key).await;
+
+        match metadata {
+            Ok(Some(md)) => Ok(Some(md)),
+            Ok(None) => Ok(None),
+            Err(cacache::Error::EntryNotFound(_, _)) => Ok(None),
+            Err(err) => Err(Error::new(ErrorKind::Unexpected, "cacache metadata failed")
+                .with_operation("CacacheCore::metadata")
+                .set_source(err)),
+        }
+    }
+}

data/core/src/services/cacache/delete.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use crate::raw::*;
+use crate::*;
+
+use super::core::CacacheCore;
+
+pub struct CacacheDeleter {
+    core: Arc<CacacheCore>,
+}
+
+impl CacacheDeleter {
+    pub fn new(core: Arc<CacacheCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for CacacheDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        self.core.delete(&path).await
+    }
+}

data/core/src/services/cacache/docs.md

@@ -0,0 +1,38 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] ~~presign~~
+- [x] blocking
+
+## Configuration
+
+- `datadir`: Set the path to the cacache data directory
+
+You can refer to [`CacacheBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Cacache;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Cacache::default().datadir("/tmp/opendal/cacache");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/cacache/mod.rs

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for cacache service.
+#[cfg(feature = "services-cacache")]
+pub(super) const DEFAULT_SCHEME: &str = "cacache";
+#[cfg(feature = "services-cacache")]
+mod backend;
+#[cfg(feature = "services-cacache")]
+mod core;
+#[cfg(feature = "services-cacache")]
+mod delete;
+#[cfg(feature = "services-cacache")]
+mod writer;
+
+#[cfg(feature = "services-cacache")]
+pub use backend::CacacheBuilder as Cacache;
+
+mod config;
+pub use config::CacacheConfig;

data/core/src/services/cacache/writer.rs

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use crate::raw::*;
+use crate::*;
+
+use super::core::CacacheCore;
+
+pub struct CacacheWriter {
+    core: Arc<CacacheCore>,
+    path: String,
+    buffer: oio::QueueBuf,
+}
+
+impl CacacheWriter {
+    pub fn new(core: Arc<CacacheCore>, path: String) -> Self {
+        Self {
+            core,
+            path,
+            buffer: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for CacacheWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buffer.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buffer.clone().collect();
+        let length = buf.len() as u64;
+
+        self.core.set(&self.path, buf.to_bytes()).await?;
+
+        let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
+        Ok(meta)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buffer.clear();
+        Ok(())
+    }
+}