opendal 0.1.6.pre.rc.1

data/core/src/services/aliyun_drive/docs.md

@@ -0,0 +1,61 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [x] list
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- `access_token`: Set the access_token for backend.
+- `client_id`: Set the client_id for backend.
+- `client_secret`: Set the client_secret for backend.
+- `refresh_token`: Set the refresh_token for backend.
+- `drive_type`: Set the drive_type for backend.
+
+Refer to [`AliyunDriveBuilder`]`s  public API docs for more information.
+
+## Example
+
+### Basic Setup
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::AliyunDrive;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create aliyun-drive backend builder.
+    let mut builder = AliyunDrive::default()
+        // Set the root for aliyun-drive, all operations will happen under this root.
+        //
+        // NOTE: the root must be absolute path.
+        .root("/path/to/dir")
+        // Set the client_id. This is required.
+        .client_id("client_id")
+        // Set the client_secret. This is required.
+        .client_secret("client_secret")
+        // Set the refresh_token. This is required.
+        .refresh_token("refresh_token")
+        // Set the drive_type. This is required.
+        //
+        // Fallback to the default type if no other types found.
+        .drive_type("resource");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/aliyun_drive/error.rs

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use serde::Deserialize;
+
+use crate::*;
+
+#[derive(Default, Debug, Deserialize)]
+struct AliyunDriveError {
+    code: String,
+    message: String,
+}
+
+pub(super) fn parse_error(res: Response<Buffer>) -> Error {
+    let (parts, body) = res.into_parts();
+    let bs = body.to_bytes();
+    let (code, message) = serde_json::from_reader::<_, AliyunDriveError>(bs.clone().reader())
+        .map(|err| (Some(err.code), err.message))
+        .unwrap_or((None, String::from_utf8_lossy(&bs).into_owned()));
+    let (kind, retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        400 => match code {
+            Some(code) if code == "NotFound.File" => (ErrorKind::NotFound, false),
+            Some(code) if code == "AlreadyExist.File" => (ErrorKind::AlreadyExists, false),
+            Some(code) if code == "PreHashMatched" => (ErrorKind::IsSameFile, false),
+            _ => (ErrorKind::Unexpected, false),
+        },
+        409 => (ErrorKind::AlreadyExists, false),
+        429 => match code {
+            Some(code) if code == "TooManyRequests" => (ErrorKind::RateLimited, true),
+            _ => (ErrorKind::Unexpected, false),
+        },
+        _ => (ErrorKind::Unexpected, false),
+    };
+    let mut err = Error::new(kind, message);
+    if retryable {
+        err = err.set_temporary();
+    }
+    err
+}

data/core/src/services/aliyun_drive/lister.rs

@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use chrono::Utc;
+
+use self::oio::Entry;
+use super::core::AliyunDriveCore;
+use super::core::AliyunDriveFileList;
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Metadata;
+use crate::Result;
+
+pub struct AliyunDriveLister {
+    core: Arc<AliyunDriveCore>,
+
+    parent: Option<AliyunDriveParent>,
+    limit: Option<usize>,
+}
+
+pub struct AliyunDriveParent {
+    pub file_id: String,
+    pub path: String,
+    pub updated_at: String,
+}
+
+impl AliyunDriveLister {
+    pub fn new(
+        core: Arc<AliyunDriveCore>,
+        parent: Option<AliyunDriveParent>,
+        limit: Option<usize>,
+    ) -> Self {
+        AliyunDriveLister {
+            core,
+            parent,
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for AliyunDriveLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let Some(parent) = &self.parent else {
+            ctx.done = true;
+            return Ok(());
+        };
+
+        let offset = if ctx.token.is_empty() {
+            // Push self into the list result.
+            ctx.entries.push_back(Entry::new(
+                &parent.path,
+                Metadata::new(EntryMode::DIR).with_last_modified(
+                    parent
+                        .updated_at
+                        .parse::<chrono::DateTime<Utc>>()
+                        .map_err(|e| {
+                            Error::new(ErrorKind::Unexpected, "parse last modified time")
+                                .set_source(e)
+                        })?,
+                ),
+            ));
+            None
+        } else {
+            Some(ctx.token.clone())
+        };
+
+        let res = self.core.list(&parent.file_id, self.limit, offset).await;
+        let res = match res {
+            Err(err) if err.kind() == ErrorKind::NotFound => {
+                ctx.done = true;
+                None
+            }
+            Err(err) => return Err(err),
+            Ok(res) => Some(res),
+        };
+
+        let Some(res) = res else {
+            return Ok(());
+        };
+
+        let result: AliyunDriveFileList =
+            serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
+
+        for item in result.items {
+            let (path, mut md) = if item.path_type == "folder" {
+                let path = format!("{}{}/", &parent.path.trim_start_matches('/'), &item.name);
+                (path, Metadata::new(EntryMode::DIR))
+            } else {
+                let path = format!("{}{}", &parent.path.trim_start_matches('/'), &item.name);
+                (path, Metadata::new(EntryMode::FILE))
+            };
+
+            md = md.with_last_modified(item.updated_at.parse::<chrono::DateTime<Utc>>().map_err(
+                |e| Error::new(ErrorKind::Unexpected, "parse last modified time").set_source(e),
+            )?);
+            if let Some(v) = item.size {
+                md = md.with_content_length(v);
+            }
+            if let Some(v) = item.content_type {
+                md = md.with_content_type(v);
+            }
+
+            ctx.entries.push_back(Entry::new(&path, md));
+        }
+
+        let next_marker = result.next_marker.unwrap_or_default();
+        if next_marker.is_empty() {
+            ctx.done = true;
+        } else {
+            ctx.token = next_marker;
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/aliyun_drive/mod.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for aliyun_drive service.
+#[cfg(feature = "services-aliyun-drive")]
+pub(super) const DEFAULT_SCHEME: &str = "aliyun_drive";
+#[cfg(feature = "services-aliyun-drive")]
+mod core;
+
+#[cfg(feature = "services-aliyun-drive")]
+mod backend;
+#[cfg(feature = "services-aliyun-drive")]
+mod delete;
+#[cfg(feature = "services-aliyun-drive")]
+mod error;
+#[cfg(feature = "services-aliyun-drive")]
+mod lister;
+#[cfg(feature = "services-aliyun-drive")]
+mod writer;
+#[cfg(feature = "services-aliyun-drive")]
+pub use backend::AliyunDriveBuilder as AliyunDrive;
+
+mod config;
+
+pub use config::AliyunDriveConfig;

data/core/src/services/aliyun_drive/writer.rs

@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::AliyunDriveCore;
+use super::core::CheckNameMode;
+use super::core::CreateResponse;
+use super::core::CreateType;
+use crate::raw::*;
+use crate::*;
+
+pub struct AliyunDriveWriter {
+    core: Arc<AliyunDriveCore>,
+
+    _op: OpWrite,
+    parent_file_id: String,
+    name: String,
+
+    file_id: Option<String>,
+    upload_id: Option<String>,
+    part_number: usize,
+}
+
+impl AliyunDriveWriter {
+    pub fn new(core: Arc<AliyunDriveCore>, parent_file_id: &str, name: &str, op: OpWrite) -> Self {
+        AliyunDriveWriter {
+            core,
+            _op: op,
+            parent_file_id: parent_file_id.to_string(),
+            name: name.to_string(),
+            file_id: None,
+            upload_id: None,
+            part_number: 1, // must start from 1
+        }
+    }
+}
+
+impl oio::Write for AliyunDriveWriter {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        let (upload_id, file_id) = match (self.upload_id.as_ref(), self.file_id.as_ref()) {
+            (Some(upload_id), Some(file_id)) => (upload_id, file_id),
+            _ => {
+                let res = self
+                    .core
+                    .create(
+                        Some(&self.parent_file_id),
+                        &self.name,
+                        CreateType::File,
+                        CheckNameMode::Refuse,
+                    )
+                    .await?;
+                let output: CreateResponse =
+                    serde_json::from_reader(res.reader()).map_err(new_json_deserialize_error)?;
+                if output.exist.is_some_and(|x| x) {
+                    return Err(Error::new(ErrorKind::AlreadyExists, "file exists"));
+                }
+                self.upload_id = output.upload_id;
+                self.file_id = Some(output.file_id);
+                (
+                    self.upload_id.as_ref().expect("cannot find upload_id"),
+                    self.file_id.as_ref().expect("cannot find file_id"),
+                )
+            }
+        };
+
+        if let Err(err) = self
+            .core
+            .upload(file_id, upload_id, self.part_number, bs)
+            .await
+        {
+            if err.kind() != ErrorKind::AlreadyExists {
+                return Err(err);
+            }
+        };
+
+        self.part_number += 1;
+
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let (Some(upload_id), Some(file_id)) = (self.upload_id.as_ref(), self.file_id.as_ref())
+        else {
+            return Ok(Metadata::default());
+        };
+
+        self.core.complete(file_id, upload_id).await?;
+        Ok(Metadata::default())
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        let Some(file_id) = self.file_id.as_ref() else {
+            return Ok(());
+        };
+        self.core.delete_path(file_id).await
+    }
+}

data/core/src/services/alluxio/backend.rs

@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::Response;
+use log::debug;
+
+use super::core::AlluxioCore;
+use super::delete::AlluxioDeleter;
+use super::error::parse_error;
+use super::lister::AlluxioLister;
+use super::writer::AlluxioWriter;
+use super::writer::AlluxioWriters;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::AlluxioConfig;
+use crate::*;
+impl Configurator for AlluxioConfig {
+    type Builder = AlluxioBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        AlluxioBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// [Alluxio](https://www.alluxio.io/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct AlluxioBuilder {
+    config: AlluxioConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for AlluxioBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("AlluxioBuilder");
+
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl AlluxioBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// endpoint of this backend.
+    ///
+    /// Endpoint must be full uri, mostly like `http://127.0.0.1:39999`.
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        if !endpoint.is_empty() {
+            // Trim trailing `/` so that we can accept `http://127.0.0.1:39999/`
+            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string())
+        }
+
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+}
+
+impl Builder for AlluxioBuilder {
+    type Config = AlluxioConfig;
+
+    /// Builds the backend and returns the result of AlluxioBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+        debug!("backend use root {}", &root);
+
+        let endpoint = match &self.config.endpoint {
+            Some(endpoint) => Ok(endpoint.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Alluxio)),
+        }?;
+        debug!("backend use endpoint {}", &endpoint);
+
+        Ok(AlluxioBackend {
+            core: Arc::new(AlluxioCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_root(&root)
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            // FIXME:
+                            //
+                            // alluxio's read support is not implemented correctly
+                            // We need to refactor by use [page_read](https://github.com/Alluxio/alluxio-py/blob/main/alluxio/const.py#L18)
+                            read: false,
+
+                            write: true,
+                            write_can_multi: true,
+
+                            create_dir: true,
+                            delete: true,
+
+                            list: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+
+                    // allow deprecated api here for compatibility
+                    #[allow(deprecated)]
+                    if let Some(client) = self.http_client {
+                        am.update_http_client(|_| client);
+                    }
+
+                    am.into()
+                },
+                root,
+                endpoint,
+            }),
+        })
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AlluxioBackend {
+    core: Arc<AlluxioCore>,
+}
+
+impl Access for AlluxioBackend {
+    type Reader = HttpBody;
+    type Writer = AlluxioWriters;
+    type Lister = oio::PageLister<AlluxioLister>;
+    type Deleter = oio::OneShotDeleter<AlluxioDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        self.core.create_dir(path).await?;
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let file_info = self.core.get_status(path).await?;
+
+        Ok(RpStat::new(file_info.try_into()?))
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let stream_id = self.core.open_file(path).await?;
+
+        let resp = self.core.read(stream_id, args.range()).await?;
+        if !resp.status().is_success() {
+            let (part, mut body) = resp.into_parts();
+            let buf = body.to_buffer().await?;
+            return Err(parse_error(Response::from_parts(part, buf)));
+        }
+        Ok((RpRead::new(), resp.into_body()))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let w = AlluxioWriter::new(self.core.clone(), args.clone(), path.to_string());
+
+        Ok((RpWrite::default(), w))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(AlluxioDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = AlluxioLister::new(self.core.clone(), path);
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result<RpRename> {
+        self.core.rename(from, to).await?;
+
+        Ok(RpRename::default())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use super::*;
+
+    #[test]
+    fn test_builder_from_map() {
+        let mut map = HashMap::new();
+        map.insert("root".to_string(), "/".to_string());
+        map.insert("endpoint".to_string(), "http://127.0.0.1:39999".to_string());
+
+        let builder = AlluxioConfig::from_iter(map).unwrap();
+
+        assert_eq!(builder.root, Some("/".to_string()));
+        assert_eq!(builder.endpoint, Some("http://127.0.0.1:39999".to_string()));
+    }
+
+    #[test]
+    fn test_builder_build() {
+        let builder = AlluxioBuilder::default()
+            .root("/root")
+            .endpoint("http://127.0.0.1:39999")
+            .build();
+
+        assert!(builder.is_ok());
+    }
+}