opendal 0.1.6.pre.rc.1

data/core/src/services/dashmap/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for dashmap service.
+#[cfg(feature = "services-dashmap")]
+pub(super) const DEFAULT_SCHEME: &str = "dashmap";
+#[cfg(feature = "services-dashmap")]
+mod backend;
+#[cfg(feature = "services-dashmap")]
+mod core;
+#[cfg(feature = "services-dashmap")]
+mod delete;
+#[cfg(feature = "services-dashmap")]
+mod lister;
+#[cfg(feature = "services-dashmap")]
+mod writer;
+
+#[cfg(feature = "services-dashmap")]
+pub use backend::DashmapBuilder as Dashmap;
+
+mod config;
+pub use config::DashmapConfig;

data/core/src/services/dashmap/writer.rs

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::time::SystemTime;
+
+use super::core::DashmapCore;
+use super::core::DashmapValue;
+use crate::raw::{oio, OpWrite};
+use crate::*;
+
+pub struct DashmapWriter {
+    core: Arc<DashmapCore>,
+    path: String,
+    op: OpWrite,
+
+    buf: oio::QueueBuf,
+}
+
+impl DashmapWriter {
+    pub fn new(core: Arc<DashmapCore>, path: String, op: OpWrite) -> Self {
+        DashmapWriter {
+            core,
+            path,
+            op,
+            buf: oio::QueueBuf::new(),
+        }
+    }
+}
+
+impl oio::Write for DashmapWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.buf.push(bs);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let content = self.buf.clone().collect();
+
+        let entry_mode = EntryMode::from_path(&self.path);
+        let mut meta = Metadata::new(entry_mode);
+        meta.set_content_length(content.len() as u64);
+        meta.set_last_modified(SystemTime::now().into());
+
+        if let Some(v) = self.op.content_type() {
+            meta.set_content_type(v);
+        }
+        if let Some(v) = self.op.content_disposition() {
+            meta.set_content_disposition(v);
+        }
+        if let Some(v) = self.op.cache_control() {
+            meta.set_cache_control(v);
+        }
+        if let Some(v) = self.op.content_encoding() {
+            meta.set_content_encoding(v);
+        }
+
+        self.core.set(
+            &self.path,
+            DashmapValue {
+                metadata: meta.clone(),
+                content,
+            },
+        )?;
+
+        Ok(meta)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.buf.clear();
+        Ok(())
+    }
+}

data/core/src/services/dbfs/backend.rs

@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+
+use super::core::DbfsCore;
+use super::delete::DbfsDeleter;
+use super::error::parse_error;
+use super::lister::DbfsLister;
+use super::writer::DbfsWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::DbfsConfig;
+use crate::*;
+impl Configurator for DbfsConfig {
+    type Builder = DbfsBuilder;
+    fn into_builder(self) -> Self::Builder {
+        DbfsBuilder { config: self }
+    }
+}
+
+/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Clone)]
+pub struct DbfsBuilder {
+    config: DbfsConfig,
+}
+
+impl Debug for DbfsBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("DbfsBuilder");
+
+        ds.field("config", &self.config);
+
+        ds.finish()
+    }
+}
+
+impl DbfsBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set endpoint of this backend.
+    ///
+    /// Endpoint must be full uri, e.g.
+    ///
+    /// - Azure: `https://adb-1234567890123456.78.azuredatabricks.net`
+    /// - Aws: `https://dbc-123a5678-90bc.cloud.databricks.com`
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        self.config.endpoint = if endpoint.is_empty() {
+            None
+        } else {
+            Some(endpoint.trim_end_matches('/').to_string())
+        };
+        self
+    }
+
+    /// Set the token of this backend.
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for DbfsBuilder {
+    type Config = DbfsConfig;
+
+    /// Build a DbfsBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let endpoint = match &self.config.endpoint {
+            Some(endpoint) => Ok(endpoint.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Dbfs)),
+        }?;
+        debug!("backend use endpoint: {}", &endpoint);
+
+        let token = match self.config.token {
+            Some(token) => token,
+            None => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "missing token for Dbfs",
+                ));
+            }
+        };
+
+        let client = HttpClient::new()?;
+        Ok(DbfsBackend {
+            core: Arc::new(DbfsCore {
+                root,
+                endpoint: endpoint.to_string(),
+                token,
+                client,
+            }),
+        })
+    }
+}
+
+/// Backend for DBFS service
+#[derive(Debug, Clone)]
+pub struct DbfsBackend {
+    core: Arc<DbfsCore>,
+}
+
+impl Access for DbfsBackend {
+    type Reader = ();
+    type Writer = oio::OneShotWriter<DbfsWriter>;
+    type Lister = oio::PageLister<DbfsLister>;
+    type Deleter = oio::OneShotDeleter<DbfsDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        let am = AccessorInfo::default();
+        am.set_scheme(DEFAULT_SCHEME)
+            .set_root(&self.core.root)
+            .set_native_capability(Capability {
+                stat: true,
+
+                write: true,
+                create_dir: true,
+                delete: true,
+                rename: true,
+
+                list: true,
+
+                shared: true,
+
+                ..Default::default()
+            });
+        am.into()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        let resp = self.core.dbfs_create_dir(path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        // Stat root always returns a DIR.
+        if path == "/" {
+            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+        }
+
+        let resp = self.core.dbfs_get_status(path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let mut meta = parse_into_metadata(path, resp.headers())?;
+                let bs = resp.into_body();
+                let decoded_response: DbfsStatus =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+                meta.set_last_modified(parse_datetime_from_from_timestamp_millis(
+                    decoded_response.modification_time,
+                )?);
+                match decoded_response.is_dir {
+                    true => meta.set_mode(EntryMode::DIR),
+                    false => {
+                        meta.set_mode(EntryMode::FILE);
+                        meta.set_content_length(decoded_response.file_size as u64)
+                    }
+                };
+                Ok(RpStat::new(meta))
+            }
+            StatusCode::NOT_FOUND if path.ends_with('/') => {
+                Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        Ok((
+            RpWrite::default(),
+            oio::OneShotWriter::new(DbfsWriter::new(self.core.clone(), args, path.to_string())),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(DbfsDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = DbfsLister::new(self.core.clone(), path.to_string());
+
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        self.core.dbfs_ensure_parent_path(to).await?;
+
+        let resp = self.core.dbfs_rename(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(RpRename::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct DbfsStatus {
+    // Not used fields.
+    // path: String,
+    is_dir: bool,
+    file_size: i64,
+    modification_time: i64,
+}

data/core/src/services/dbfs/config.rs

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct DbfsConfig {
+    /// The root for dbfs.
+    pub root: Option<String>,
+    /// The endpoint for dbfs.
+    pub endpoint: Option<String>,
+    /// The token for dbfs.
+    pub token: Option<String>,
+}
+
+impl Debug for DbfsConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("DbfsConfig");
+
+        ds.field("root", &self.root);
+        ds.field("endpoint", &self.endpoint);
+
+        if self.token.is_some() {
+            ds.field("token", &"<redacted>");
+        }
+
+        ds.finish()
+    }
+}

data/core/src/services/dbfs/core.rs

@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde_json::json;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DbfsCore {
+    pub root: String,
+    pub endpoint: String,
+    pub token: String,
+    pub client: HttpClient,
+}
+
+impl Debug for DbfsCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("DbfsCore")
+            .field("root", &self.root)
+            .field("endpoint", &self.endpoint)
+            .field("token", &self.token)
+            .finish_non_exhaustive()
+    }
+}
+
+impl DbfsCore {
+    pub async fn dbfs_create_dir(&self, path: &str) -> Result<Response<Buffer>> {
+        let url = format!("{}/api/2.0/dbfs/mkdirs", self.endpoint);
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let p = build_rooted_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let req_body = &json!({
+            "path": percent_encode_path(&p),
+        });
+        let body = Buffer::from(Bytes::from(req_body.to_string()));
+
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub async fn dbfs_delete(&self, path: &str) -> Result<Response<Buffer>> {
+        let url = format!("{}/api/2.0/dbfs/delete", self.endpoint);
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let p = build_rooted_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let request_body = &json!({
+            "path": percent_encode_path(&p),
+            // TODO: support recursive toggle, should we add a new field in OpDelete?
+            "recursive": true,
+        });
+
+        let body = Buffer::from(Bytes::from(request_body.to_string()));
+
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub async fn dbfs_rename(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
+        let source = build_rooted_abs_path(&self.root, from);
+        let target = build_rooted_abs_path(&self.root, to);
+
+        let url = format!("{}/api/2.0/dbfs/move", self.endpoint);
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let req_body = &json!({
+            "source_path": percent_encode_path(&source),
+            "destination_path": percent_encode_path(&target),
+        });
+
+        let body = Buffer::from(Bytes::from(req_body.to_string()));
+
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub async fn dbfs_list(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_rooted_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/2.0/dbfs/list?path={}",
+            self.endpoint,
+            percent_encode_path(&p)
+        );
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub fn dbfs_create_file_request(&self, path: &str, body: Bytes) -> Result<Request<Buffer>> {
+        let url = format!("{}/api/2.0/dbfs/put", self.endpoint);
+
+        let contents = BASE64_STANDARD.encode(body);
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let req_body = &json!({
+            "path": path,
+            "contents": contents,
+            "overwrite": true,
+        });
+
+        let body = Buffer::from(Bytes::from(req_body.to_string()));
+
+        req.body(body).map_err(new_request_build_error)
+    }
+
+    pub async fn dbfs_get_status(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_rooted_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/2.0/dbfs/get-status?path={}",
+            &self.endpoint,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format!("Bearer {}", self.token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub async fn dbfs_ensure_parent_path(&self, path: &str) -> Result<()> {
+        let resp = self.dbfs_get_status(path).await?;
+
+        match resp.status() {
+            StatusCode::OK => return Ok(()),
+            StatusCode::NOT_FOUND => {
+                self.dbfs_create_dir(path).await?;
+            }
+            _ => return Err(parse_error(resp)),
+        }
+        Ok(())
+    }
+}

data/core/src/services/dbfs/delete.rs

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DbfsDeleter {
+    core: Arc<DbfsCore>,
+}
+
+impl DbfsDeleter {
+    pub fn new(core: Arc<DbfsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for DbfsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.dbfs_delete(&path).await?;
+
+        let status = resp.status();
+
+        match status {
+            // NOTE: Server will return 200 even if the path doesn't exist.
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}