opendal 0.1.6.pre.rc.1

data/core/src/services/gdrive/core.rs

@@ -0,0 +1,499 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes;
+use bytes::Buf;
+use bytes::Bytes;
+use chrono::DateTime;
+use chrono::Utc;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::json;
+use tokio::sync::Mutex;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct GdriveCore {
+    pub info: Arc<AccessorInfo>,
+
+    pub root: String,
+
+    pub signer: Arc<Mutex<GdriveSigner>>,
+
+    /// Cache the mapping from path to file id
+    pub path_cache: PathCacher<GdrivePathQuery>,
+}
+
+impl Debug for GdriveCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("GdriveCore");
+        de.field("root", &self.root);
+        de.finish()
+    }
+}
+
+impl GdriveCore {
+    pub async fn gdrive_stat(&self, path: &str) -> Result<Response<Buffer>> {
+        let path = build_abs_path(&self.root, path);
+        let file_id = self.path_cache.get(&path).await?.ok_or(Error::new(
+            ErrorKind::NotFound,
+            format!("path not found: {path}"),
+        ))?;
+
+        // The file metadata in the Google Drive API is very complex.
+        // For now, we only need the file id, name, mime type and modified time.
+        let mut req = Request::get(format!(
+            "https://www.googleapis.com/drive/v3/files/{file_id}?fields=id,name,mimeType,size,modifiedTime"
+        ))
+            .extension(Operation::Stat)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn gdrive_get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
+        let path = build_abs_path(&self.root, path);
+        let path_id = self.path_cache.get(&path).await?.ok_or(Error::new(
+            ErrorKind::NotFound,
+            format!("path not found: {path}"),
+        ))?;
+
+        let url: String = format!("https://www.googleapis.com/drive/v3/files/{path_id}?alt=media");
+
+        let mut req = Request::get(&url)
+            .extension(Operation::Read)
+            .header(header::RANGE, range.to_header())
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+        self.sign(&mut req).await?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub async fn gdrive_list(
+        &self,
+        file_id: &str,
+        page_size: i32,
+        next_page_token: &str,
+    ) -> Result<Response<Buffer>> {
+        let q = format!("'{file_id}' in parents and trashed = false");
+        let url = "https://www.googleapis.com/drive/v3/files";
+        let mut url = QueryPairsWriter::new(url);
+        url = url.push("pageSize", &page_size.to_string());
+        url = url.push("q", &percent_encode_path(&q));
+        if !next_page_token.is_empty() {
+            url = url.push("pageToken", next_page_token);
+        };
+
+        let mut req = Request::get(url.finish())
+            .extension(Operation::List)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    // Update with content and metadata
+    pub async fn gdrive_patch_metadata_request(
+        &self,
+        source: &str,
+        target: &str,
+    ) -> Result<Response<Buffer>> {
+        let source_file_id = self.path_cache.get(source).await?.ok_or(Error::new(
+            ErrorKind::NotFound,
+            format!("source path not found: {source}"),
+        ))?;
+        let source_parent = get_parent(source);
+        let source_parent_id = self
+            .path_cache
+            .get(source_parent)
+            .await?
+            .expect("old parent must exist");
+
+        let target_parent_id = self.path_cache.ensure_dir(get_parent(target)).await?;
+        let target_file_name = get_basename(target);
+
+        let metadata = &json!({
+            "name": target_file_name,
+            "removeParents": [source_parent_id],
+            "addParents": [target_parent_id],
+        });
+
+        let url = format!("https://www.googleapis.com/drive/v3/files/{source_file_id}");
+        let mut req = Request::patch(url)
+            .extension(Operation::Rename)
+            .body(Buffer::from(Bytes::from(metadata.to_string())))
+            .map_err(new_request_build_error)?;
+
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn gdrive_trash(&self, file_id: &str) -> Result<Response<Buffer>> {
+        let url = format!("https://www.googleapis.com/drive/v3/files/{file_id}");
+
+        let body = serde_json::to_vec(&json!({
+            "trashed": true
+        }))
+        .map_err(new_json_serialize_error)?;
+
+        let mut req = Request::patch(&url)
+            .extension(Operation::Delete)
+            .body(Buffer::from(Bytes::from(body)))
+            .map_err(new_request_build_error)?;
+
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    /// Create a file with the content.
+    pub async fn gdrive_upload_simple_request(
+        &self,
+        path: &str,
+        size: u64,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let parent = self.path_cache.ensure_dir(get_parent(path)).await?;
+
+        let url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=multipart";
+
+        let file_name = get_basename(path);
+
+        let metadata = serde_json::to_vec(&json!({
+            "name": file_name,
+            "parents": [parent],
+        }))
+        .map_err(new_json_serialize_error)?;
+
+        let req = Request::post(url)
+            .header("X-Upload-Content-Length", size)
+            .extension(Operation::Write);
+
+        let multipart = Multipart::new()
+            .part(
+                FormDataPart::new("metadata")
+                    .header(
+                        header::CONTENT_TYPE,
+                        "application/json; charset=UTF-8".parse().unwrap(),
+                    )
+                    .content(metadata),
+            )
+            .part(
+                FormDataPart::new("file")
+                    .header(
+                        header::CONTENT_TYPE,
+                        "application/octet-stream".parse().unwrap(),
+                    )
+                    .content(body),
+            );
+
+        let mut req = multipart.apply(req)?;
+
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    /// Overwrite the file with the content.
+    ///
+    /// # Notes
+    ///
+    /// - The file id is required. Do not use this method to create a file.
+    pub async fn gdrive_upload_overwrite_simple_request(
+        &self,
+        file_id: &str,
+        size: u64,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let url =
+            format!("https://www.googleapis.com/upload/drive/v3/files/{file_id}?uploadType=media");
+
+        let mut req = Request::patch(url)
+            .header(header::CONTENT_TYPE, "application/octet-stream")
+            .header(header::CONTENT_LENGTH, size)
+            .header("X-Upload-Content-Length", size)
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
+        let mut signer = self.signer.lock().await;
+        signer.sign(req).await
+    }
+
+    pub async fn gdrive_copy(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
+        let from = build_abs_path(&self.root, from);
+
+        let from_file_id = self.path_cache.get(&from).await?.ok_or(Error::new(
+            ErrorKind::NotFound,
+            "the file to copy does not exist",
+        ))?;
+
+        let to_name = get_basename(to);
+        let to_path = build_abs_path(&self.root, to);
+        let to_parent_id = self.path_cache.ensure_dir(get_parent(&to_path)).await?;
+
+        // copy will overwrite `to`, delete it if exist
+        if let Some(id) = self.path_cache.get(&to_path).await? {
+            let resp = self.gdrive_trash(&id).await?;
+            let status = resp.status();
+            if status != StatusCode::OK {
+                return Err(parse_error(resp));
+            }
+
+            self.path_cache.remove(&to_path).await;
+        }
+
+        let url = format!("https://www.googleapis.com/drive/v3/files/{from_file_id}/copy");
+
+        let request_body = &json!({
+            "name": to_name,
+            "parents": [to_parent_id],
+        });
+        let body = Buffer::from(Bytes::from(request_body.to_string()));
+
+        let mut req = Request::post(&url)
+            .extension(Operation::Copy)
+            .body(body)
+            .map_err(new_request_build_error)?;
+        self.sign(&mut req).await?;
+
+        self.info.http_client().send(req).await
+    }
+}
+
+#[derive(Clone)]
+pub struct GdriveSigner {
+    pub info: Arc<AccessorInfo>,
+
+    pub client_id: String,
+    pub client_secret: String,
+    pub refresh_token: String,
+
+    pub access_token: String,
+    pub expires_in: DateTime<Utc>,
+}
+
+impl GdriveSigner {
+    /// Create a new signer.
+    pub fn new(info: Arc<AccessorInfo>) -> Self {
+        GdriveSigner {
+            info,
+
+            client_id: "".to_string(),
+            client_secret: "".to_string(),
+            refresh_token: "".to_string(),
+            access_token: "".to_string(),
+            expires_in: DateTime::<Utc>::MIN_UTC,
+        }
+    }
+
+    /// Sign a request.
+    pub async fn sign<T>(&mut self, req: &mut Request<T>) -> Result<()> {
+        if !self.access_token.is_empty() && self.expires_in > Utc::now() {
+            let value = format!("Bearer {}", self.access_token)
+                .parse()
+                .expect("access token must be valid header value");
+
+            req.headers_mut().insert(header::AUTHORIZATION, value);
+            return Ok(());
+        }
+
+        let url = format!(
+            "https://oauth2.googleapis.com/token?refresh_token={}&client_id={}&client_secret={}&grant_type=refresh_token",
+            self.refresh_token, self.client_id, self.client_secret
+        );
+
+        {
+            let req = Request::post(url)
+                .header(header::CONTENT_LENGTH, 0)
+                .body(Buffer::new())
+                .map_err(new_request_build_error)?;
+
+            let resp = self.info.http_client().send(req).await?;
+            let status = resp.status();
+
+            match status {
+                StatusCode::OK => {
+                    let resp_body = resp.into_body();
+                    let token: GdriveTokenResponse = serde_json::from_reader(resp_body.reader())
+                        .map_err(new_json_deserialize_error)?;
+                    self.access_token.clone_from(&token.access_token);
+                    self.expires_in = Utc::now()
+                        + chrono::TimeDelta::try_seconds(token.expires_in)
+                            .expect("expires_in must be valid seconds")
+                        - chrono::TimeDelta::try_seconds(120).expect("120 must be valid seconds");
+                }
+                _ => {
+                    return Err(parse_error(resp));
+                }
+            }
+        }
+
+        let auth_header_content = format!("Bearer {}", self.access_token);
+        req.headers_mut()
+            .insert(header::AUTHORIZATION, auth_header_content.parse().unwrap());
+
+        Ok(())
+    }
+}
+
+pub struct GdrivePathQuery {
+    pub info: Arc<AccessorInfo>,
+    pub signer: Arc<Mutex<GdriveSigner>>,
+}
+
+impl GdrivePathQuery {
+    pub fn new(info: Arc<AccessorInfo>, signer: Arc<Mutex<GdriveSigner>>) -> Self {
+        GdrivePathQuery { info, signer }
+    }
+}
+
+impl PathQuery for GdrivePathQuery {
+    async fn root(&self) -> Result<String> {
+        Ok("root".to_string())
+    }
+
+    async fn query(&self, parent_id: &str, name: &str) -> Result<Option<String>> {
+        let mut queries = vec![
+            // Make sure name has been replaced with escaped name.
+            //
+            // ref: <https://developers.google.com/drive/api/guides/ref-search-terms>
+            format!(
+                "name = '{}'",
+                name.replace('\'', "\\'").trim_end_matches('/')
+            ),
+            format!("'{}' in parents", parent_id),
+            "trashed = false".to_string(),
+        ];
+        if name.ends_with('/') {
+            queries.push("mimeType = 'application/vnd.google-apps.folder'".to_string());
+        }
+        let query = queries.join(" and ");
+
+        let url = format!(
+            "https://www.googleapis.com/drive/v3/files?q={}",
+            percent_encode_path(query.as_str())
+        );
+
+        let mut req = Request::get(&url)
+            .extension(Operation::Stat)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.signer.lock().await.sign(&mut req).await?;
+
+        let resp = self.info.http_client().send(req).await?;
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let meta: GdriveFileList =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+
+                if let Some(f) = meta.files.first() {
+                    Ok(Some(f.id.clone()))
+                } else {
+                    Ok(None)
+                }
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn create_dir(&self, parent_id: &str, name: &str) -> Result<String> {
+        let url = "https://www.googleapis.com/drive/v3/files";
+
+        let content = serde_json::to_vec(&json!({
+            "name": name.trim_end_matches('/'),
+            "mimeType": "application/vnd.google-apps.folder",
+            // If the parent is not provided, the folder will be created in the root folder.
+            "parents": [parent_id],
+        }))
+        .map_err(new_json_serialize_error)?;
+
+        let mut req = Request::post(url)
+            .extension(Operation::CreateDir)
+            .header(header::CONTENT_TYPE, "application/json")
+            .body(Buffer::from(Bytes::from(content)))
+            .map_err(new_request_build_error)?;
+
+        self.signer.lock().await.sign(&mut req).await?;
+
+        let resp = self.info.http_client().send(req).await?;
+        if !resp.status().is_success() {
+            return Err(parse_error(resp));
+        }
+
+        let body = resp.into_body();
+        let file: GdriveFile =
+            serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+        Ok(file.id)
+    }
+}
+
+#[derive(Deserialize)]
+pub struct GdriveTokenResponse {
+    access_token: String,
+    expires_in: i64,
+}
+
+/// This is the file struct returned by the Google Drive API.
+/// This is a complex struct, but we only add the fields we need.
+/// refer to https://developers.google.com/drive/api/reference/rest/v3/files#File
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct GdriveFile {
+    pub mime_type: String,
+    pub id: String,
+    pub name: String,
+    pub size: Option<String>,
+    // The modified time is not returned unless the `fields`
+    // query parameter contains `modifiedTime`.
+    // As we only need the modified time when we do `stat` operation,
+    // if other operations(such as search) do not specify the `fields` query parameter,
+    // try to access this field, it will be `None`.
+    pub modified_time: Option<String>,
+}
+
+/// refer to https://developers.google.com/drive/api/reference/rest/v3/files/list
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct GdriveFileList {
+    pub(crate) files: Vec<GdriveFile>,
+    pub(crate) next_page_token: Option<String>,
+}

data/core/src/services/gdrive/delete.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct GdriveDeleter {
+    core: Arc<GdriveCore>,
+}
+
+impl GdriveDeleter {
+    pub fn new(core: Arc<GdriveCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for GdriveDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let path = build_abs_path(&self.core.root, &path);
+        let file_id = self.core.path_cache.get(&path).await?;
+        let file_id = if let Some(id) = file_id {
+            id
+        } else {
+            return Ok(());
+        };
+
+        let resp = self.core.gdrive_trash(&file_id).await?;
+        let status = resp.status();
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        self.core.path_cache.remove(&path).await;
+
+        Ok(())
+    }
+}

data/core/src/services/gdrive/docs.md

@@ -0,0 +1,65 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] delete
+- [x] create_dir
+- [x] list
+- [x] copy
+- [x] rename
+- [ ] batch
+
+
+# Configuration
+
+- `root`: Set the work directory for backend
+
+### Credentials related
+
+#### Just provide Access Token (Temporary)
+
+- `access_token`: set the access_token for google drive api
+Please notice its expiration.
+
+#### Or provide Client ID and Client Secret and refresh token (Long Term)
+
+If you want to let OpenDAL to refresh the access token automatically,
+please provide the following fields:
+
+- `refresh_token`: set the refresh_token for google drive api
+- `client_id`: set the client_id for google drive api
+- `client_secret`: set the client_secret for google drive api
+
+OpenDAL is a library, it cannot do the first step of OAuth2 for you.
+You need to get authorization code from user by calling GoogleDrive's authorize url
+and exchange it for refresh token.
+
+Make sure you have enabled Google Drive API in your Google Cloud Console.
+And your OAuth scope contains `https://www.googleapis.com/auth/drive`.
+
+Please refer to [GoogleDrive OAuth2 Flow](https://developers.google.com/identity/protocols/oauth2/)
+for more information.
+
+You can refer to [`GdriveBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Gdrive;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Gdrive::default()
+        .root("/test")
+        .access_token("<token>");
+
+    Ok(())
+}
+

data/core/src/services/gdrive/error.rs

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+#[derive(Default, Debug, Deserialize)]
+struct GdriveError {
+    error: GdriveInnerError,
+}
+
+#[derive(Default, Debug, Deserialize)]
+struct GdriveInnerError {
+    message: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT
+        // Gdrive sometimes return METHOD_NOT_ALLOWED for our requests for abuse detection.
+        | StatusCode::METHOD_NOT_ALLOWED => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, gdrive_err) = serde_json::from_slice::<GdriveError>(bs.as_ref())
+        .map(|gdrive_err| (format!("{gdrive_err:?}"), Some(gdrive_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(gdrive_err) = gdrive_err {
+        (kind, retryable) =
+            parse_gdrive_error_code(gdrive_err.error.message.as_str()).unwrap_or((kind, retryable));
+    }
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+pub fn parse_gdrive_error_code(message: &str) -> Option<(ErrorKind, bool)> {
+    match message {
+        // > Please reduce your request rate.
+        //
+        // It's Ok to retry since later on the request rate may get reduced.
+        "User rate limit exceeded." => Some((ErrorKind::RateLimited, true)),
+        _ => None,
+    }
+}