opendal 0.1.6.pre.rc.1

data/core/src/services/dbfs/docs.md

@@ -0,0 +1,57 @@
+This service will visit the [DBFS API](https://docs.databricks.com/api/azure/workspace/dbfs) supported by [Databricks File System](https://docs.databricks.com/en/dbfs/index.html).
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [x] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configurations
+
+- `root`: Set the work directory for backend.
+- `endpoint`: Set the endpoint for backend.
+- `token`: Databricks personal access token.
+
+Refer to [`DbfsBuilder`]'s public API docs for more information.
+
+## Examples
+
+### Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Dbfs;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Dbfs::default()
+        // set the root for Dbfs, all operations will happen under this root
+        //
+        // Note:
+        // if the root is not exists, the builder will automatically create the
+        // root directory for you
+        // if the root exists and is a directory, the builder will continue working
+        // if the root exists and is a folder, the builder will fail on building backend
+        .root("/path/to/dir")
+        // set the endpoint of Dbfs workspace
+        .endpoint("https://adb-1234567890123456.78.azuredatabricks.net")
+        // set the personal access token for builder
+        .token("access_token");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/dbfs/error.rs

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// DbfsError is the error returned by DBFS service.
+#[derive(Default, Deserialize)]
+struct DbfsError {
+    error_code: String,
+    message: String,
+}
+
+impl Debug for DbfsError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("DbfsError");
+        de.field("error_code", &self.error_code);
+        // replace `\n` to ` ` for better reading.
+        de.field("message", &self.message.replace('\n', " "));
+
+        de.finish()
+    }
+}
+
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match serde_json::from_slice::<DbfsError>(&bs) {
+        Ok(dbfs_error) => format!("{:?}", dbfs_error.message),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/src/services/dbfs/lister.rs

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+use serde::Deserialize;
+
+use super::core::DbfsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DbfsLister {
+    core: Arc<DbfsCore>,
+    path: String,
+}
+
+impl DbfsLister {
+    pub fn new(core: Arc<DbfsCore>, path: String) -> Self {
+        Self { core, path }
+    }
+}
+
+impl oio::PageList for DbfsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let response = self.core.dbfs_list(&self.path).await?;
+
+        let status_code = response.status();
+        if !status_code.is_success() {
+            if status_code == StatusCode::NOT_FOUND {
+                ctx.done = true;
+                return Ok(());
+            }
+            let error = parse_error(response);
+            return Err(error);
+        }
+
+        let bytes = response.into_body();
+        let decoded_response: DbfsOutputList =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        ctx.done = true;
+
+        for status in decoded_response.files {
+            let entry: oio::Entry = match status.is_dir {
+                true => {
+                    let normalized_path = format!("{}/", &status.path);
+                    let mut meta = Metadata::new(EntryMode::DIR);
+                    meta.set_last_modified(parse_datetime_from_from_timestamp_millis(
+                        status.modification_time,
+                    )?);
+                    oio::Entry::new(&normalized_path, meta)
+                }
+                false => {
+                    let mut meta = Metadata::new(EntryMode::FILE);
+                    meta.set_last_modified(parse_datetime_from_from_timestamp_millis(
+                        status.modification_time,
+                    )?);
+                    meta.set_content_length(status.file_size as u64);
+                    oio::Entry::new(&status.path, meta)
+                }
+            };
+            ctx.entries.push_back(entry);
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize)]
+struct DbfsOutputList {
+    files: Vec<DbfsStatus>,
+}
+
+#[derive(Debug, Deserialize)]
+struct DbfsStatus {
+    path: String,
+    is_dir: bool,
+    file_size: i64,
+    modification_time: i64,
+}

data/core/src/services/dbfs/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for dbfs service.
+#[cfg(feature = "services-dbfs")]
+pub(super) const DEFAULT_SCHEME: &str = "dbfs";
+#[cfg(feature = "services-dbfs")]
+mod core;
+#[cfg(feature = "services-dbfs")]
+mod delete;
+#[cfg(feature = "services-dbfs")]
+mod error;
+#[cfg(feature = "services-dbfs")]
+mod lister;
+#[cfg(feature = "services-dbfs")]
+mod writer;
+
+#[cfg(feature = "services-dbfs")]
+mod backend;
+#[cfg(feature = "services-dbfs")]
+pub use backend::DbfsBuilder as Dbfs;
+
+mod config;
+pub use config::DbfsConfig;

data/core/src/services/dbfs/writer.rs

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::DbfsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct DbfsWriter {
+    core: Arc<DbfsCore>,
+    path: String,
+}
+
+impl DbfsWriter {
+    const MAX_SIMPLE_SIZE: usize = 1024 * 1024;
+
+    pub fn new(core: Arc<DbfsCore>, _op: OpWrite, path: String) -> Self {
+        DbfsWriter { core, path }
+    }
+}
+
+impl oio::OneShotWrite for DbfsWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let size = bs.len();
+
+        // MAX_BLOCK_SIZE_EXCEEDED will be thrown if this limit(1MB) is exceeded.
+        if size >= Self::MAX_SIMPLE_SIZE {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "AppendWrite has not been implemented for Dbfs",
+            ));
+        }
+
+        let req = self
+            .core
+            .dbfs_create_file_request(&self.path, bs.to_bytes())?;
+
+        let resp = self.core.client.send(req).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/dropbox/backend.rs

@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+
+use super::core::*;
+use super::delete::DropboxDeleter;
+use super::error::*;
+use super::lister::DropboxLister;
+use super::writer::DropboxWriter;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Clone, Debug)]
+pub struct DropboxBackend {
+    pub core: Arc<DropboxCore>,
+}
+
+impl Access for DropboxBackend {
+    type Reader = HttpBody;
+    type Writer = oio::OneShotWriter<DropboxWriter>;
+    type Lister = oio::PageLister<DropboxLister>;
+    type Deleter = oio::OneShotDeleter<DropboxDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+        // Check if the folder already exists.
+        let resp = self.core.dropbox_get_metadata(path).await?;
+        if StatusCode::OK == resp.status() {
+            let bytes = resp.into_body();
+            let decoded_response: DropboxMetadataResponse =
+                serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+            if "folder" == decoded_response.tag {
+                return Ok(RpCreateDir::default());
+            }
+            if "file" == decoded_response.tag {
+                return Err(Error::new(
+                    ErrorKind::NotADirectory,
+                    format!("it's not a directory {path}"),
+                ));
+            }
+        }
+
+        let res = self.core.dropbox_create_folder(path).await?;
+        Ok(res)
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let resp = self.core.dropbox_get_metadata(path).await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let bytes = resp.into_body();
+                let decoded_response: DropboxMetadataResponse =
+                    serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+                let entry_mode: EntryMode = match decoded_response.tag.as_str() {
+                    "file" => EntryMode::FILE,
+                    "folder" => EntryMode::DIR,
+                    _ => EntryMode::Unknown,
+                };
+
+                let mut metadata = Metadata::new(entry_mode);
+                // Only set last_modified and size if entry_mode is FILE, because Dropbox API
+                // returns last_modified and size only for files.
+                // FYI: https://www.dropbox.com/developers/documentation/http/documentation#files-get_metadata
+                if entry_mode == EntryMode::FILE {
+                    let date_utc_last_modified =
+                        parse_datetime_from_rfc3339(&decoded_response.client_modified)?;
+                    metadata.set_last_modified(date_utc_last_modified);
+
+                    if let Some(size) = decoded_response.size {
+                        metadata.set_content_length(size);
+                    } else {
+                        return Err(Error::new(
+                            ErrorKind::Unexpected,
+                            format!("no size found for file {path}"),
+                        ));
+                    }
+                }
+                Ok(RpStat::new(metadata))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.dropbox_get(path, args.range(), &args).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        Ok((
+            RpWrite::default(),
+            oio::OneShotWriter::new(DropboxWriter::new(
+                self.core.clone(),
+                args,
+                String::from(path),
+            )),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(DropboxDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        Ok((
+            RpList::default(),
+            oio::PageLister::new(DropboxLister::new(
+                self.core.clone(),
+                path.to_string(),
+                args.recursive(),
+                args.limit(),
+            )),
+        ))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
+        let resp = self.core.dropbox_copy(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(RpCopy::default()),
+            _ => {
+                let err = parse_error(resp);
+                match err.kind() {
+                    ErrorKind::NotFound => Ok(RpCopy::default()),
+                    _ => Err(err),
+                }
+            }
+        }
+    }
+
+    async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result<RpRename> {
+        let resp = self.core.dropbox_move(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(RpRename::default()),
+            _ => {
+                let err = parse_error(resp);
+                match err.kind() {
+                    ErrorKind::NotFound => Ok(RpRename::default()),
+                    _ => Err(err),
+                }
+            }
+        }
+    }
+}