opendal 0.1.6.pre.rc.1

data/core/src/services/d1/error.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use serde_json::de;
+
+use super::model::D1Error;
+use super::model::D1Response;
+use crate::raw::*;
+use crate::*;
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        // Some services (like owncloud) return 403 while file locked.
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
+        // Allowing retry for resource locked.
+        StatusCode::LOCKED => (ErrorKind::Unexpected, true),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, d1_err) = de::from_reader::<_, D1Response>(bs.clone().reader())
+        .map(|d1_err| (format!("{d1_err:?}"), Some(d1_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(d1_err) = d1_err {
+        (kind, retryable) = parse_d1_error_code(d1_err.errors).unwrap_or((kind, retryable));
+    }
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+pub fn parse_d1_error_code(errors: Vec<D1Error>) -> Option<(ErrorKind, bool)> {
+    if errors.is_empty() {
+        return None;
+    }
+
+    match errors[0].code {
+        // The request is malformed: failed to decode id.
+        7400 => Some((ErrorKind::Unexpected, false)),
+        // no such column: Xxxx.
+        7500 => Some((ErrorKind::NotFound, false)),
+        // Authentication error.
+        10000 => Some((ErrorKind::PermissionDenied, false)),
+        _ => None,
+    }
+}

data/core/src/services/d1/mod.rs

@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-d1")]
+mod error;
+#[cfg(feature = "services-d1")]
+mod model;
+
+#[cfg(feature = "services-d1")]
+mod backend;
+#[cfg(feature = "services-d1")]
+pub use backend::D1Builder as D1;
+
+mod config;
+pub use config::D1Config;

data/core/src/services/d1/model.rs

@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use bytes::Bytes;
+use serde::Deserialize;
+use serde::Serialize;
+use serde_json::Map;
+use serde_json::Value;
+
+use crate::Buffer;
+use crate::Error;
+
+/// response data from d1
+#[derive(Deserialize, Debug)]
+pub struct D1Response {
+    pub result: Vec<D1Result>,
+    pub success: bool,
+    pub errors: Vec<D1Error>,
+}
+
+impl D1Response {
+    pub fn parse(bs: &Bytes) -> Result<D1Response, Error> {
+        let response: D1Response = serde_json::from_slice(bs).map_err(|e| {
+            Error::new(
+                crate::ErrorKind::Unexpected,
+                format!("failed to parse error response: {e}"),
+            )
+        })?;
+
+        if !response.success {
+            return Err(Error::new(
+                crate::ErrorKind::Unexpected,
+                String::from_utf8_lossy(bs),
+            ));
+        }
+        Ok(response)
+    }
+
+    pub fn get_result(&self, key: &str) -> Option<Buffer> {
+        if self.result.is_empty() || self.result[0].results.is_empty() {
+            return None;
+        }
+        let result = &self.result[0].results[0];
+        let value = result.get(key);
+
+        match value {
+            Some(Value::Array(s)) => {
+                let mut v = Vec::new();
+                for i in s {
+                    if let Value::Number(n) = i {
+                        v.push(n.as_u64().unwrap() as u8);
+                    }
+                }
+                Some(Buffer::from(v))
+            }
+            _ => None,
+        }
+    }
+}
+
+#[derive(Deserialize, Debug)]
+pub struct D1Result {
+    pub results: Vec<Map<String, Value>>,
+}
+
+#[derive(Clone, Deserialize, Debug, Serialize)]
+pub struct D1Error {
+    pub message: String,
+    pub code: i32,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_deserialize_get_object_json_response() {
+        let data = r#"
+        {
+            "result": [
+                {
+                    "results": [
+                        {
+                            "CustomerId": "4",
+                            "CompanyName": "Around the Horn",
+                            "ContactName": "Thomas Hardy"
+                        }
+                    ],
+                    "success": true,
+                    "meta": {
+                        "served_by": "v3-prod",
+                        "duration": 0.2147,
+                        "changes": 0,
+                        "last_row_id": 0,
+                        "changed_db": false,
+                        "size_after": 2162688,
+                        "rows_read": 3,
+                        "rows_written": 2
+                    }
+                }
+            ],
+            "success": true,
+            "errors": [],
+            "messages": []
+        }"#;
+        let response: D1Response = serde_json::from_str(data).unwrap();
+        println!("{:?}", response.result[0].results[0]);
+    }
+}

data/core/src/services/dashmap/backend.rs

@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use dashmap::DashMap;
+use log::debug;
+
+use super::core::DashmapCore;
+use super::delete::DashmapDeleter;
+use super::lister::DashmapLister;
+use super::writer::DashmapWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::services::DashmapConfig;
+use crate::*;
+impl Configurator for DashmapConfig {
+    type Builder = DashmapBuilder;
+    fn into_builder(self) -> Self::Builder {
+        DashmapBuilder { config: self }
+    }
+}
+
+/// [dashmap](https://github.com/xacrimon/dashmap) backend support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct DashmapBuilder {
+    config: DashmapConfig,
+}
+
+impl Debug for DashmapBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DashmapBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl DashmapBuilder {
+    /// Set the root for dashmap.
+    pub fn root(mut self, path: &str) -> Self {
+        self.config.root = if path.is_empty() {
+            None
+        } else {
+            Some(path.to_string())
+        };
+
+        self
+    }
+}
+
+impl Builder for DashmapBuilder {
+    type Config = DashmapConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(
+            self.config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+
+        debug!("backend build finished: {:?}", self.config);
+
+        let core = DashmapCore {
+            cache: DashMap::new(),
+        };
+
+        Ok(DashmapAccessor::new(core, root))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct DashmapAccessor {
+    core: Arc<DashmapCore>,
+    root: String,
+    info: Arc<AccessorInfo>,
+}
+
+impl DashmapAccessor {
+    fn new(core: DashmapCore, root: String) -> Self {
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME);
+        info.set_name("dashmap");
+        info.set_root(&root);
+        info.set_native_capability(Capability {
+            read: true,
+
+            write: true,
+            write_can_empty: true,
+            write_with_cache_control: true,
+            write_with_content_type: true,
+            write_with_content_disposition: true,
+            write_with_content_encoding: true,
+
+            delete: true,
+            stat: true,
+            list: true,
+            shared: false,
+            ..Default::default()
+        });
+
+        Self {
+            core: Arc::new(core),
+            root,
+            info: Arc::new(info),
+        }
+    }
+}
+
+impl Access for DashmapAccessor {
+    type Reader = Buffer;
+    type Writer = DashmapWriter;
+    type Lister = oio::HierarchyLister<DashmapLister>;
+    type Deleter = oio::OneShotDeleter<DashmapDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let p = build_abs_path(&self.root, path);
+
+        match self.core.get(&p)? {
+            Some(value) => {
+                let metadata = value.metadata;
+                Ok(RpStat::new(metadata))
+            }
+            None => {
+                if p.ends_with('/') {
+                    let has_children = self.core.cache.iter().any(|kv| kv.key().starts_with(&p));
+                    if has_children {
+                        return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+                    }
+                }
+                Err(Error::new(ErrorKind::NotFound, "key not found in dashmap"))
+            }
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let p = build_abs_path(&self.root, path);
+
+        match self.core.get(&p)? {
+            Some(value) => {
+                let buffer = if args.range().is_full() {
+                    value.content
+                } else {
+                    let range = args.range();
+                    let start = range.offset() as usize;
+                    let end = match range.size() {
+                        Some(size) => (range.offset() + size) as usize,
+                        None => value.content.len(),
+                    };
+                    value.content.slice(start..end.min(value.content.len()))
+                };
+                Ok((RpRead::new(), buffer))
+            }
+            None => Err(Error::new(ErrorKind::NotFound, "key not found in dashmap")),
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let p = build_abs_path(&self.root, path);
+        Ok((
+            RpWrite::new(),
+            DashmapWriter::new(self.core.clone(), p, args),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(DashmapDeleter::new(self.core.clone(), self.root.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let lister = DashmapLister::new(self.core.clone(), self.root.clone(), path.to_string());
+        let lister = oio::HierarchyLister::new(lister, path, args.recursive());
+        Ok((RpList::default(), lister))
+    }
+}

data/core/src/services/dashmap/config.rs

@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+use std::fmt::Debug;
+
+/// Config for Dashmap services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct DashmapConfig {
+    /// root path of this backend
+    pub root: Option<String>,
+}
+
+impl Debug for DashmapConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DashmapConfig")
+            .field("root", &self.root)
+            .finish_non_exhaustive()
+    }
+}

data/core/src/services/dashmap/core.rs

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use dashmap::DashMap;
+
+use crate::*;
+
+/// Value stored in dashmap cache containing both metadata and content
+#[derive(Clone)]
+pub struct DashmapValue {
+    /// Stored metadata in dashmap cache.
+    pub metadata: Metadata,
+    /// Stored content in dashmap cache.
+    pub content: Buffer,
+}
+
+#[derive(Clone)]
+pub struct DashmapCore {
+    pub cache: DashMap<String, DashmapValue>,
+}
+
+impl Debug for DashmapCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DashmapCore")
+            .field("size", &self.cache.len())
+            .finish()
+    }
+}
+
+impl DashmapCore {
+    pub fn get(&self, key: &str) -> Result<Option<DashmapValue>> {
+        Ok(self.cache.get(key).map(|v| v.value().clone()))
+    }
+
+    pub fn set(&self, key: &str, value: DashmapValue) -> Result<()> {
+        self.cache.insert(key.to_string(), value);
+        Ok(())
+    }
+
+    pub fn delete(&self, key: &str) -> Result<()> {
+        self.cache.remove(key);
+        Ok(())
+    }
+}

data/core/src/services/dashmap/delete.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::DashmapCore;
+use crate::raw::{build_abs_path, oio, OpDelete};
+use crate::*;
+
+pub struct DashmapDeleter {
+    core: Arc<DashmapCore>,
+    root: String,
+}
+
+impl DashmapDeleter {
+    pub fn new(core: Arc<DashmapCore>, root: String) -> Self {
+        Self { core, root }
+    }
+}
+
+impl oio::OneShotDelete for DashmapDeleter {
+    async fn delete_once(&self, path: String, _op: OpDelete) -> Result<()> {
+        let p = build_abs_path(&self.root, &path);
+        self.core.delete(&p)
+    }
+}

data/core/src/services/dashmap/docs.md

@@ -0,0 +1,38 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [x] list
+- [ ] presign
+
+## Configuration
+
+- `root`: Set the root path for this dashmap instance.
+
+You can refer to [`DashmapBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Dashmap;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Dashmap::default()
+        .root("/");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/dashmap/lister.rs

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::core::DashmapCore;
+use crate::raw::oio::Entry;
+use crate::raw::{build_abs_path, build_rel_path, oio};
+use crate::*;
+use std::sync::Arc;
+use std::vec::IntoIter;
+
+pub struct DashmapLister {
+    root: String,
+    path: String,
+    iter: IntoIter<String>,
+}
+
+impl DashmapLister {
+    pub fn new(core: Arc<DashmapCore>, root: String, path: String) -> Self {
+        let entries: Vec<_> = core.cache.iter().map(|item| item.key().clone()).collect();
+        let path = build_abs_path(&root, &path);
+
+        Self {
+            root,
+            path,
+            iter: entries.into_iter(),
+        }
+    }
+}
+
+impl oio::List for DashmapLister {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        for key in self.iter.by_ref() {
+            if key.starts_with(&self.path) {
+                let path = build_rel_path(&self.root, &key);
+
+                // Determine if it's a file or directory based on trailing slash
+                let mode = if key.ends_with('/') {
+                    EntryMode::DIR
+                } else {
+                    EntryMode::FILE
+                };
+                let entry = Entry::new(&path, Metadata::new(mode));
+                return Ok(Some(entry));
+            }
+        }
+
+        Ok(None)
+    }
+}