opendal 0.1.6.pre.rc.1

data/core/src/services/yandex_disk/backend.rs

@@ -0,0 +1,267 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use log::debug;
+
+use super::core::*;
+use super::delete::YandexDiskDeleter;
+use super::error::parse_error;
+use super::lister::YandexDiskLister;
+use super::writer::YandexDiskWriter;
+use super::writer::YandexDiskWriters;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::YandexDiskConfig;
+use crate::*;
+impl Configurator for YandexDiskConfig {
+    type Builder = YandexDiskBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        YandexDiskBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// [YandexDisk](https://360.yandex.com/disk/) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct YandexDiskBuilder {
+    config: YandexDiskConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for YandexDiskBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("YandexDiskBuilder");
+
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl YandexDiskBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// yandex disk oauth access_token.
+    /// The valid token will looks like `y0_XXXXXXqihqIWAADLWwAAAAD3IXXXXXX0gtVeSPeIKM0oITMGhXXXXXX`.
+    /// We can fetch the debug token from <https://yandex.com/dev/disk/poligon>.
+    /// To use it in production, please register an app at <https://oauth.yandex.com> instead.
+    pub fn access_token(mut self, access_token: &str) -> Self {
+        self.config.access_token = access_token.to_string();
+
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+}
+
+impl Builder for YandexDiskBuilder {
+    type Config = YandexDiskConfig;
+
+    /// Builds the backend and returns the result of YandexDiskBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+        debug!("backend use root {}", &root);
+
+        // Handle oauth access_token.
+        if self.config.access_token.is_empty() {
+            return Err(
+                Error::new(ErrorKind::ConfigInvalid, "access_token is empty")
+                    .with_operation("Builder::build")
+                    .with_context("service", Scheme::YandexDisk),
+            );
+        }
+
+        Ok(YandexDiskBackend {
+            core: Arc::new(YandexDiskCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_root(&root)
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            create_dir: true,
+
+                            read: true,
+
+                            write: true,
+                            write_can_empty: true,
+
+                            delete: true,
+                            rename: true,
+                            copy: true,
+
+                            list: true,
+                            list_with_limit: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+
+                    // allow deprecated api here for compatibility
+                    #[allow(deprecated)]
+                    if let Some(client) = self.http_client {
+                        am.update_http_client(|_| client);
+                    }
+
+                    am.into()
+                },
+                root,
+                access_token: self.config.access_token.clone(),
+            }),
+        })
+    }
+}
+
+/// Backend for YandexDisk services.
+#[derive(Debug, Clone)]
+pub struct YandexDiskBackend {
+    core: Arc<YandexDiskCore>,
+}
+
+impl Access for YandexDiskBackend {
+    type Reader = HttpBody;
+    type Writer = YandexDiskWriters;
+    type Lister = oio::PageLister<YandexDiskLister>;
+    type Deleter = oio::OneShotDeleter<YandexDiskDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        self.core.ensure_dir_exists(path).await?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        self.core.ensure_dir_exists(to).await?;
+
+        let resp = self.core.move_object(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::CREATED => Ok(RpRename::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        self.core.ensure_dir_exists(to).await?;
+
+        let resp = self.core.copy(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::CREATED => Ok(RpCopy::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.download(path, args.range()).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let resp = self.core.metainformation(path, None, None).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let mf: MetainformationResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                parse_info(mf).map(RpStat::new)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let writer = YandexDiskWriter::new(self.core.clone(), path.to_string());
+
+        let w = oio::OneShotWriter::new(writer);
+
+        Ok((RpWrite::default(), w))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(YandexDiskDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = YandexDiskLister::new(self.core.clone(), path, args.limit());
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+}

data/core/src/services/yandex_disk/config.rs

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for YandexDisk services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct YandexDiskConfig {
+    /// root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub root: Option<String>,
+    /// yandex disk oauth access_token.
+    pub access_token: String,
+}
+
+impl Debug for YandexDiskConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Config");
+
+        ds.field("root", &self.root);
+
+        ds.finish()
+    }
+}

data/core/src/services/yandex_disk/core.rs

@@ -0,0 +1,340 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::header;
+use http::request;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Clone)]
+pub struct YandexDiskCore {
+    pub info: Arc<AccessorInfo>,
+    /// The root of this core.
+    pub root: String,
+    /// Yandex Disk oauth access_token.
+    pub access_token: String,
+}
+
+impl Debug for YandexDiskCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Backend")
+            .field("root", &self.root)
+            .finish_non_exhaustive()
+    }
+}
+
+impl YandexDiskCore {
+    #[inline]
+    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
+        self.info.http_client().send(req).await
+    }
+
+    #[inline]
+    pub fn sign(&self, req: request::Builder) -> request::Builder {
+        req.header(
+            header::AUTHORIZATION,
+            format!("OAuth {}", self.access_token),
+        )
+    }
+}
+
+impl YandexDiskCore {
+    /// Get upload url.
+    async fn get_upload_url(&self, path: &str) -> Result<String> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources/upload?path={}&overwrite=true",
+            percent_encode_path(&path)
+        );
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::Write);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bytes = resp.into_body();
+
+                let resp: GetUploadUrlResponse =
+                    serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(resp.href)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn upload(&self, path: &str, body: Buffer) -> Result<Response<Buffer>> {
+        let upload_url = self.get_upload_url(path).await?;
+        let req = Request::put(upload_url)
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    async fn get_download_url(&self, path: &str) -> Result<String> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources/download?path={}&overwrite=true",
+            percent_encode_path(&path)
+        );
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::Read);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bytes = resp.into_body();
+
+                let resp: GetUploadUrlResponse =
+                    serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(resp.href)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn download(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
+        let download_url = self.get_download_url(path).await?;
+        let req = Request::get(download_url)
+            .header(header::RANGE, range.to_header())
+            .extension(Operation::Read)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub async fn ensure_dir_exists(&self, path: &str) -> Result<()> {
+        let path = build_abs_path(&self.root, path);
+
+        let paths = path.split('/').collect::<Vec<&str>>();
+
+        for i in 0..paths.len() - 1 {
+            let path = paths[..i + 1].join("/");
+            let resp = self.create_dir(&path).await?;
+
+            let status = resp.status();
+
+            match status {
+                StatusCode::CREATED | StatusCode::CONFLICT => {}
+                _ => return Err(parse_error(resp)),
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn create_dir(&self, path: &str) -> Result<Response<Buffer>> {
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources?path=/{}",
+            percent_encode_path(path),
+        );
+
+        let req = Request::put(url);
+
+        let req = req.extension(Operation::CreateDir);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn copy(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
+        let from = build_rooted_abs_path(&self.root, from);
+        let to = build_rooted_abs_path(&self.root, to);
+
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources/copy?from={}&path={}&overwrite=true",
+            percent_encode_path(&from),
+            percent_encode_path(&to)
+        );
+
+        let req = Request::post(url);
+
+        let req = req.extension(Operation::Copy);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn move_object(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
+        let from = build_rooted_abs_path(&self.root, from);
+        let to = build_rooted_abs_path(&self.root, to);
+
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources/move?from={}&path={}&overwrite=true",
+            percent_encode_path(&from),
+            percent_encode_path(&to)
+        );
+
+        let req = Request::post(url);
+
+        let req = req.extension(Operation::Rename);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn delete(&self, path: &str) -> Result<Response<Buffer>> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources?path={}&permanently=true",
+            percent_encode_path(&path),
+        );
+
+        let req = Request::delete(url);
+
+        let req = req.extension(Operation::Delete);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn metainformation(
+        &self,
+        path: &str,
+        limit: Option<usize>,
+        offset: Option<String>,
+    ) -> Result<Response<Buffer>> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let mut url = format!(
+            "https://cloud-api.yandex.net/v1/disk/resources?path={}",
+            percent_encode_path(&path),
+        );
+
+        if let Some(limit) = limit {
+            url = format!("{url}&limit={limit}");
+        }
+
+        if let Some(offset) = offset {
+            url = format!("{url}&offset={offset}");
+        }
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::Stat);
+
+        let req = self.sign(req);
+
+        // Set body
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+}
+
+pub(super) fn parse_info(mf: MetainformationResponse) -> Result<Metadata> {
+    let mode = if mf.ty == "file" {
+        EntryMode::FILE
+    } else {
+        EntryMode::DIR
+    };
+
+    let mut m = Metadata::new(mode);
+
+    m.set_last_modified(parse_datetime_from_rfc3339(&mf.modified)?);
+
+    if let Some(md5) = mf.md5 {
+        m.set_content_md5(&md5);
+    }
+
+    if let Some(mime_type) = mf.mime_type {
+        m.set_content_type(&mime_type);
+    }
+
+    if let Some(size) = mf.size {
+        m.set_content_length(size);
+    }
+
+    Ok(m)
+}
+
+#[derive(Debug, Deserialize)]
+pub struct GetUploadUrlResponse {
+    pub href: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct MetainformationResponse {
+    #[serde(rename = "type")]
+    pub ty: String,
+    pub path: String,
+    pub modified: String,
+    pub md5: Option<String>,
+    pub mime_type: Option<String>,
+    pub size: Option<u64>,
+    #[serde(rename = "_embedded")]
+    pub embedded: Option<Embedded>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Embedded {
+    pub total: usize,
+    pub items: Vec<MetainformationResponse>,
+}

data/core/src/services/yandex_disk/delete.rs

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct YandexDiskDeleter {
+    core: Arc<YandexDiskCore>,
+}
+
+impl YandexDiskDeleter {
+    pub fn new(core: Arc<YandexDiskCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for YandexDiskDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.delete(&path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(()),
+            StatusCode::NO_CONTENT => Ok(()),
+            // Yandex Disk deleting a non-empty folder can take an unknown amount of time,
+            // So the API responds with the code 202 Accepted (the deletion process has started).
+            StatusCode::ACCEPTED => Ok(()),
+            // Allow 404 when deleting a non-existing object
+            StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}