opendal 0.1.6.pre.rc.1

data/core/src/services/memory/backend.rs

@@ -0,0 +1,205 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use http::Uri;
+use percent_encoding::percent_decode_str;
+
+use super::core::*;
+use super::delete::MemoryDeleter;
+use super::lister::MemoryLister;
+use super::writer::MemoryWriter;
+use super::MEMORY_SCHEME;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::services::MemoryConfig;
+use crate::*;
+impl Configurator for MemoryConfig {
+    type Builder = MemoryBuilder;
+    fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+        let mut map = options.clone();
+
+        if !map.contains_key("root") {
+            let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+            if !path.is_empty() && path != "/" {
+                map.insert("root".to_string(), path.trim_start_matches('/').to_string());
+            }
+        }
+
+        Self::from_iter(map)
+    }
+    fn into_builder(self) -> Self::Builder {
+        MemoryBuilder { config: self }
+    }
+}
+
+/// In memory service support. (BTreeMap Based)
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct MemoryBuilder {
+    config: MemoryConfig,
+}
+
+impl MemoryBuilder {
+    /// Set the root for BTreeMap.
+    pub fn root(mut self, path: &str) -> Self {
+        self.config.root = Some(path.into());
+        self
+    }
+}
+
+impl Builder for MemoryBuilder {
+    type Config = MemoryConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let root = normalize_root(self.config.root.as_deref().unwrap_or("/"));
+
+        let core = MemoryCore::new();
+        Ok(MemoryAccessor::new(core).with_normalized_root(root))
+    }
+}
+
+/// MemoryAccessor implements Access trait directly
+#[derive(Debug, Clone)]
+pub struct MemoryAccessor {
+    core: Arc<MemoryCore>,
+    root: String,
+    info: Arc<AccessorInfo>,
+}
+
+impl MemoryAccessor {
+    fn new(core: MemoryCore) -> Self {
+        let info = AccessorInfo::default();
+        info.set_scheme(MEMORY_SCHEME);
+        info.set_name(&format!("{:p}", Arc::as_ptr(&core.data)));
+        info.set_root("/");
+        info.set_native_capability(Capability {
+            read: true,
+            write: true,
+            write_can_empty: true,
+            write_with_cache_control: true,
+            write_with_content_type: true,
+            write_with_content_disposition: true,
+            write_with_content_encoding: true,
+            delete: true,
+            stat: true,
+            list: true,
+            list_with_recursive: true,
+            shared: false,
+            ..Default::default()
+        });
+
+        Self {
+            core: Arc::new(core),
+            root: "/".to_string(),
+            info: Arc::new(info),
+        }
+    }
+
+    fn with_normalized_root(mut self, root: String) -> Self {
+        self.info.set_root(&root);
+        self.root = root;
+        self
+    }
+}
+
+impl Access for MemoryAccessor {
+    type Reader = Buffer;
+    type Writer = MemoryWriter;
+    type Lister = oio::HierarchyLister<MemoryLister>;
+    type Deleter = oio::OneShotDeleter<MemoryDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let p = build_abs_path(&self.root, path);
+
+        if p == build_abs_path(&self.root, "") {
+            Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+        } else {
+            match self.core.get(&p)? {
+                Some(value) => Ok(RpStat::new(value.metadata)),
+                None => Err(Error::new(
+                    ErrorKind::NotFound,
+                    "memory doesn't have this path",
+                )),
+            }
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let p = build_abs_path(&self.root, path);
+
+        let value = match self.core.get(&p)? {
+            Some(value) => value,
+            None => {
+                return Err(Error::new(
+                    ErrorKind::NotFound,
+                    "memory doesn't have this path",
+                ))
+            }
+        };
+
+        Ok((
+            RpRead::new(),
+            value.content.slice(args.range().to_range_as_usize()),
+        ))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let p = build_abs_path(&self.root, path);
+        Ok((
+            RpWrite::new(),
+            MemoryWriter::new(self.core.clone(), p, args),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(MemoryDeleter::new(self.core.clone(), self.root.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let p = build_abs_path(&self.root, path);
+        let keys = self.core.scan(&p)?;
+        let lister = MemoryLister::new(&self.root, keys);
+        let lister = oio::HierarchyLister::new(lister, path, args.recursive());
+
+        Ok((RpList::default(), lister))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_accessor_metadata_name() {
+        let b1 = MemoryBuilder::default().build().unwrap();
+        assert_eq!(b1.info().name(), b1.info().name());
+
+        let b2 = MemoryBuilder::default().build().unwrap();
+        assert_ne!(b1.info().name(), b2.info().name())
+    }
+}

data/core/src/services/memory/config.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for memory.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct MemoryConfig {
+    /// root of the backend.
+    pub root: Option<String>,
+}

data/core/src/services/memory/core.rs

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use crate::*;
+
+/// Value stored in memory containing both metadata and content
+#[derive(Clone)]
+pub struct MemoryValue {
+    pub metadata: Metadata,
+    pub content: Buffer,
+}
+
+#[derive(Clone)]
+pub struct MemoryCore {
+    pub data: Arc<Mutex<BTreeMap<String, MemoryValue>>>,
+}
+
+impl Debug for MemoryCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MemoryCore").finish_non_exhaustive()
+    }
+}
+
+impl MemoryCore {
+    pub fn new() -> Self {
+        Self {
+            data: Arc::new(Mutex::new(BTreeMap::new())),
+        }
+    }
+
+    pub fn get(&self, key: &str) -> Result<Option<MemoryValue>> {
+        Ok(self.data.lock().unwrap().get(key).cloned())
+    }
+
+    pub fn set(&self, key: &str, value: MemoryValue) -> Result<()> {
+        self.data.lock().unwrap().insert(key.to_string(), value);
+        Ok(())
+    }
+
+    pub fn delete(&self, key: &str) -> Result<()> {
+        self.data.lock().unwrap().remove(key);
+        Ok(())
+    }
+
+    pub fn scan(&self, prefix: &str) -> Result<Vec<String>> {
+        let data = self.data.lock().unwrap();
+
+        if prefix.is_empty() {
+            return Ok(data.keys().cloned().collect());
+        }
+
+        let mut keys = Vec::new();
+        for (key, _) in data.range(prefix.to_string()..) {
+            if !key.starts_with(prefix) {
+                break;
+            }
+            keys.push(key.clone());
+        }
+        Ok(keys)
+    }
+}

data/core/src/services/memory/delete.rs

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MemoryDeleter {
+    core: Arc<MemoryCore>,
+    root: String,
+}
+
+impl MemoryDeleter {
+    pub fn new(core: Arc<MemoryCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl oio::OneShotDelete for MemoryDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = build_abs_path(&self.root, &path);
+        self.core.delete(&p)?;
+        Ok(())
+    }
+}

data/core/src/services/memory/docs.md

@@ -0,0 +1,36 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [ ] list
+- [ ] presign
+- [ ] blocking
+
+## Example
+
+### Via Builder
+
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Memory;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Memory::default().root("/tmp");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/memory/lister.rs

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::vec::IntoIter;
+
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MemoryLister {
+    root: String,
+    keys: IntoIter<String>,
+}
+
+impl MemoryLister {
+    pub fn new(root: &str, keys: Vec<String>) -> Self {
+        Self {
+            root: root.to_string(),
+            keys: keys.into_iter(),
+        }
+    }
+}
+
+impl oio::List for MemoryLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        match self.keys.next() {
+            Some(key) => {
+                let mode = if key.ends_with('/') {
+                    EntryMode::DIR
+                } else {
+                    EntryMode::FILE
+                };
+                let mut path = build_rel_path(&self.root, &key);
+                if path.is_empty() {
+                    path = "/".to_string();
+                }
+                Ok(Some(oio::Entry::new(&path, Metadata::new(mode))))
+            }
+            None => Ok(None),
+        }
+    }
+}

data/core/src/services/memory/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for memory service.
+#[cfg(feature = "services-memory")]
+pub const MEMORY_SCHEME: &str = "memory";
+#[cfg(feature = "services-memory")]
+mod backend;
+#[cfg(feature = "services-memory")]
+pub use backend::MemoryBuilder as Memory;
+
+mod config;
+pub use config::MemoryConfig;
+
+#[cfg(feature = "services-memory")]
+mod core;
+#[cfg(feature = "services-memory")]
+mod delete;
+#[cfg(feature = "services-memory")]
+mod lister;
+#[cfg(feature = "services-memory")]
+mod writer;

data/core/src/services/memory/writer.rs

@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct MemoryWriter {
+    core: Arc<MemoryCore>,
+    path: String,
+    op: OpWrite,
+    buf: Option<oio::QueueBuf>,
+}
+
+impl MemoryWriter {
+    pub fn new(core: Arc<MemoryCore>, path: String, op: OpWrite) -> Self {
+        Self {
+            core,
+            path,
+            op,
+            buf: None,
+        }
+    }
+}
+
+impl oio::Write for MemoryWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        let mut buf = self.buf.take().unwrap_or_default();
+        buf.push(bs);
+        self.buf = Some(buf);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let buf = self.buf.take().unwrap_or_default();
+        let content = buf.collect();
+
+        let mut metadata = Metadata::new(EntryMode::FILE);
+        metadata.set_content_length(content.len() as u64);
+
+        if let Some(v) = self.op.cache_control() {
+            metadata.set_cache_control(v);
+        }
+        if let Some(v) = self.op.content_disposition() {
+            metadata.set_content_disposition(v);
+        }
+        if let Some(v) = self.op.content_type() {
+            metadata.set_content_type(v);
+        }
+        if let Some(v) = self.op.content_encoding() {
+            metadata.set_content_encoding(v);
+        }
+
+        let value = MemoryValue {
+            metadata: metadata.clone(),
+            content,
+        };
+
+        self.core.set(&self.path, value)?;
+
+        Ok(metadata)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buf = None;
+        Ok(())
+    }
+}