opendal 0.1.6.pre.rc.1

data/core/src/services/gcs/writer.rs

@@ -0,0 +1,163 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::CompleteMultipartUploadRequestPart;
+use super::core::GcsCore;
+use super::core::InitiateMultipartUploadResult;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub type GcsWriters = oio::MultipartWriter<GcsWriter>;
+
+pub struct GcsWriter {
+    core: Arc<GcsCore>,
+    path: String,
+    op: OpWrite,
+}
+
+impl GcsWriter {
+    pub fn new(core: Arc<GcsCore>, path: &str, op: OpWrite) -> Self {
+        GcsWriter {
+            core,
+            path: path.to_string(),
+            op,
+        }
+    }
+}
+
+impl oio::MultipartWrite for GcsWriter {
+    async fn write_once(&self, _: u64, body: Buffer) -> Result<Metadata> {
+        let size = body.len() as u64;
+        let mut req = self.core.gcs_insert_object_request(
+            &percent_encode_path(&self.path),
+            Some(size),
+            &self.op,
+            body,
+        )?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => {
+                let metadata =
+                    GcsCore::build_metadata_from_object_response(&self.path, resp.into_body())?;
+                Ok(metadata)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self
+            .core
+            .gcs_initiate_multipart_upload(&percent_encode_path(&self.path), &self.op)
+            .await?;
+
+        if !resp.status().is_success() {
+            return Err(parse_error(resp));
+        }
+
+        let buf = resp.into_body();
+        let upload_id: InitiateMultipartUploadResult =
+            quick_xml::de::from_reader(buf.reader()).map_err(new_xml_deserialize_error)?;
+        Ok(upload_id.upload_id)
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<oio::MultipartPart> {
+        // Gcs requires part number must between [1..=10000]
+        let part_number = part_number + 1;
+
+        let resp = self
+            .core
+            .gcs_upload_part(&self.path, upload_id, part_number, size, body)
+            .await?;
+
+        if !resp.status().is_success() {
+            return Err(parse_error(resp));
+        }
+
+        let etag = parse_etag(resp.headers())?
+            .ok_or_else(|| {
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "ETag not present in returning response",
+                )
+            })?
+            .to_string();
+
+        Ok(oio::MultipartPart {
+            part_number,
+            etag,
+            checksum: None,
+        })
+    }
+
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        parts: &[oio::MultipartPart],
+    ) -> Result<Metadata> {
+        let parts = parts
+            .iter()
+            .map(|p| CompleteMultipartUploadRequestPart {
+                part_number: p.part_number,
+                etag: p.etag.clone(),
+            })
+            .collect();
+
+        let resp = self
+            .core
+            .gcs_complete_multipart_upload(&self.path, upload_id, parts)
+            .await?;
+
+        if !resp.status().is_success() {
+            return Err(parse_error(resp));
+        }
+        // we don't extract metadata from `CompleteMultipartUploadResult`, since we only need the `ETag` from it.
+        // However, the `ETag` differs from the `ETag` obtained through the `stat` operation.
+        // refer to: https://cloud.google.com/storage/docs/metadata#etags
+        Ok(Metadata::default())
+    }
+
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self
+            .core
+            .gcs_abort_multipart_upload(&self.path, upload_id)
+            .await?;
+        match resp.status() {
+            // gcs returns code 204 if abort succeeds.
+            StatusCode::NO_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/gdrive/backend.rs

@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use bytes::Buf;
+use chrono::Utc;
+use http::Response;
+use http::StatusCode;
+
+use super::core::GdriveCore;
+use super::core::GdriveFile;
+use super::delete::GdriveDeleter;
+use super::error::parse_error;
+use super::lister::GdriveLister;
+use super::writer::GdriveWriter;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Clone, Debug)]
+pub struct GdriveBackend {
+    pub core: Arc<GdriveCore>,
+}
+
+impl Access for GdriveBackend {
+    type Reader = HttpBody;
+    type Writer = oio::OneShotWriter<GdriveWriter>;
+    type Lister = oio::PageLister<GdriveLister>;
+    type Deleter = oio::OneShotDeleter<GdriveDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+        let path = build_abs_path(&self.core.root, path);
+        let _ = self.core.path_cache.ensure_dir(&path).await?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let resp = self.core.gdrive_stat(path).await?;
+
+        if resp.status() != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+        let gdrive_file: GdriveFile =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        let file_type = if gdrive_file.mime_type == "application/vnd.google-apps.folder" {
+            EntryMode::DIR
+        } else {
+            EntryMode::FILE
+        };
+        let mut meta = Metadata::new(file_type).with_content_type(gdrive_file.mime_type);
+        if let Some(v) = gdrive_file.size {
+            meta = meta.with_content_length(v.parse::<u64>().map_err(|e| {
+                Error::new(ErrorKind::Unexpected, "parse content length").set_source(e)
+            })?);
+        }
+        if let Some(v) = gdrive_file.modified_time {
+            meta = meta.with_last_modified(v.parse::<chrono::DateTime<Utc>>().map_err(|e| {
+                Error::new(ErrorKind::Unexpected, "parse last modified time").set_source(e)
+            })?);
+        }
+        Ok(RpStat::new(meta))
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.gdrive_get(path, args.range()).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let path = build_abs_path(&self.core.root, path);
+
+        // As Google Drive allows files have the same name, we need to check if the file exists.
+        // If the file exists, we will keep its ID and update it.
+        let file_id = self.core.path_cache.get(&path).await?;
+
+        Ok((
+            RpWrite::default(),
+            oio::OneShotWriter::new(GdriveWriter::new(self.core.clone(), path, file_id)),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(GdriveDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> {
+        let path = build_abs_path(&self.core.root, path);
+        let l = GdriveLister::new(path, self.core.clone());
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        let resp = self.core.gdrive_copy(from, to).await?;
+
+        match resp.status() {
+            StatusCode::OK => Ok(RpCopy::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        let source = build_abs_path(&self.core.root, from);
+        let target = build_abs_path(&self.core.root, to);
+
+        // rename will overwrite `to`, delete it if exist
+        if let Some(id) = self.core.path_cache.get(&target).await? {
+            let resp = self.core.gdrive_trash(&id).await?;
+            let status = resp.status();
+            if status != StatusCode::OK {
+                return Err(parse_error(resp));
+            }
+
+            self.core.path_cache.remove(&target).await;
+        }
+
+        let resp = self
+            .core
+            .gdrive_patch_metadata_request(&source, &target)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let meta: GdriveFile =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+
+                let cache = &self.core.path_cache;
+
+                cache.remove(&build_abs_path(&self.core.root, from)).await;
+                cache
+                    .insert(&build_abs_path(&self.core.root, to), &meta.id)
+                    .await;
+
+                Ok(RpRename::default())
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/gdrive/builder.rs

@@ -0,0 +1,228 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use chrono::DateTime;
+use chrono::Utc;
+use log::debug;
+use tokio::sync::Mutex;
+
+use super::backend::GdriveBackend;
+use super::core::GdriveCore;
+use super::core::GdrivePathQuery;
+use super::core::GdriveSigner;
+use super::DEFAULT_SCHEME;
+use crate::raw::normalize_root;
+use crate::raw::Access;
+use crate::raw::AccessorInfo;
+use crate::raw::HttpClient;
+use crate::raw::PathCacher;
+use crate::services::GdriveConfig;
+use crate::Scheme;
+use crate::*;
+impl Configurator for GdriveConfig {
+    type Builder = GdriveBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        GdriveBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// [GoogleDrive](https://drive.google.com/) backend support.
+#[derive(Default)]
+#[doc = include_str!("docs.md")]
+pub struct GdriveBuilder {
+    config: GdriveConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for GdriveBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Backend")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl GdriveBuilder {
+    /// Set root path of GoogleDrive folder.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Access token is used for temporary access to the GoogleDrive API.
+    ///
+    /// You can get the access token from [GoogleDrive App Console](https://console.cloud.google.com/apis/credentials)
+    /// or [GoogleDrive OAuth2 Playground](https://developers.google.com/oauthplayground/)
+    ///
+    /// # Note
+    ///
+    /// - An access token is valid for 1 hour.
+    /// - If you want to use the access token for a long time,
+    ///   you can use the refresh token to get a new access token.
+    pub fn access_token(mut self, access_token: &str) -> Self {
+        self.config.access_token = Some(access_token.to_string());
+        self
+    }
+
+    /// Refresh token is used for long term access to the GoogleDrive API.
+    ///
+    /// You can get the refresh token via OAuth 2.0 Flow of GoogleDrive API.
+    ///
+    /// OpenDAL will use this refresh token to get a new access token when the old one is expired.
+    pub fn refresh_token(mut self, refresh_token: &str) -> Self {
+        self.config.refresh_token = Some(refresh_token.to_string());
+        self
+    }
+
+    /// Set the client id for GoogleDrive.
+    ///
+    /// This is required for OAuth 2.0 Flow to refresh the access token.
+    pub fn client_id(mut self, client_id: &str) -> Self {
+        self.config.client_id = Some(client_id.to_string());
+        self
+    }
+
+    /// Set the client secret for GoogleDrive.
+    ///
+    /// This is required for OAuth 2.0 Flow with refresh the access token.
+    pub fn client_secret(mut self, client_secret: &str) -> Self {
+        self.config.client_secret = Some(client_secret.to_string());
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, http_client: HttpClient) -> Self {
+        self.http_client = Some(http_client);
+        self
+    }
+}
+
+impl Builder for GdriveBuilder {
+    type Config = GdriveConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME)
+            .set_root(&root)
+            .set_native_capability(Capability {
+                stat: true,
+
+                read: true,
+
+                list: true,
+
+                write: true,
+
+                create_dir: true,
+                delete: true,
+                rename: true,
+                copy: true,
+
+                shared: true,
+
+                ..Default::default()
+            });
+
+        // allow deprecated api here for compatibility
+        #[allow(deprecated)]
+        if let Some(client) = self.http_client {
+            info.update_http_client(|_| client);
+        }
+
+        let accessor_info = Arc::new(info);
+        let mut signer = GdriveSigner::new(accessor_info.clone());
+        match (self.config.access_token, self.config.refresh_token) {
+            (Some(access_token), None) => {
+                signer.access_token = access_token;
+                // We will never expire user specified access token.
+                signer.expires_in = DateTime::<Utc>::MAX_UTC;
+            }
+            (None, Some(refresh_token)) => {
+                let client_id = self.config.client_id.ok_or_else(|| {
+                    Error::new(
+                        ErrorKind::ConfigInvalid,
+                        "client_id must be set when refresh_token is set",
+                    )
+                    .with_context("service", Scheme::Gdrive)
+                })?;
+                let client_secret = self.config.client_secret.ok_or_else(|| {
+                    Error::new(
+                        ErrorKind::ConfigInvalid,
+                        "client_secret must be set when refresh_token is set",
+                    )
+                    .with_context("service", Scheme::Gdrive)
+                })?;
+
+                signer.refresh_token = refresh_token;
+                signer.client_id = client_id;
+                signer.client_secret = client_secret;
+            }
+            (Some(_), Some(_)) => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "access_token and refresh_token cannot be set at the same time",
+                )
+                .with_context("service", Scheme::Gdrive))
+            }
+            (None, None) => {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "access_token or refresh_token must be set",
+                )
+                .with_context("service", Scheme::Gdrive))
+            }
+        };
+
+        let signer = Arc::new(Mutex::new(signer));
+
+        Ok(GdriveBackend {
+            core: Arc::new(GdriveCore {
+                info: accessor_info.clone(),
+                root,
+                signer: signer.clone(),
+                path_cache: PathCacher::new(GdrivePathQuery::new(accessor_info, signer))
+                    .with_lock(),
+            }),
+        })
+    }
+}

data/core/src/services/gdrive/config.rs

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// [GoogleDrive](https://drive.google.com/) configuration.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct GdriveConfig {
+    /// The root for gdrive
+    pub root: Option<String>,
+    /// Access token for gdrive.
+    pub access_token: Option<String>,
+    /// Refresh token for gdrive.
+    pub refresh_token: Option<String>,
+    /// Client id for gdrive.
+    pub client_id: Option<String>,
+    /// Client secret for gdrive.
+    pub client_secret: Option<String>,
+}
+
+impl Debug for GdriveConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GdriveConfig")
+            .field("root", &self.root)
+            .finish_non_exhaustive()
+    }
+}