opendal 0.1.6.pre.rc.1

data/core/src/services/gdrive/lister.rs

@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::GdriveCore;
+use super::core::GdriveFileList;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct GdriveLister {
+    path: String,
+    core: Arc<GdriveCore>,
+}
+
+impl GdriveLister {
+    pub fn new(path: String, core: Arc<GdriveCore>) -> Self {
+        Self { path, core }
+    }
+}
+
+impl oio::PageList for GdriveLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let file_id = self.core.path_cache.get(&self.path).await?;
+
+        let file_id = match file_id {
+            Some(file_id) => file_id,
+            None => {
+                ctx.done = true;
+                return Ok(());
+            }
+        };
+
+        let resp = self
+            .core
+            .gdrive_list(file_id.as_str(), 100, &ctx.token)
+            .await?;
+
+        let bytes = match resp.status() {
+            StatusCode::OK => resp.into_body().to_bytes(),
+            _ => return Err(parse_error(resp)),
+        };
+
+        // Google Drive returns an empty response when attempting to list a non-existent directory.
+        if bytes.is_empty() {
+            ctx.done = true;
+            return Ok(());
+        }
+
+        // Include the current directory itself when handling the first page of the listing.
+        if ctx.token.is_empty() && !ctx.done {
+            let path = build_rel_path(&self.core.root, &self.path);
+            let e = oio::Entry::new(&path, Metadata::new(EntryMode::DIR));
+            ctx.entries.push_back(e);
+        }
+
+        let decoded_response =
+            serde_json::from_slice::<GdriveFileList>(&bytes).map_err(new_json_deserialize_error)?;
+
+        if let Some(next_page_token) = decoded_response.next_page_token {
+            ctx.token = next_page_token;
+        } else {
+            ctx.done = true;
+        }
+
+        for mut file in decoded_response.files {
+            let file_type = if file.mime_type.as_str() == "application/vnd.google-apps.folder" {
+                if !file.name.ends_with('/') {
+                    file.name += "/";
+                }
+                EntryMode::DIR
+            } else {
+                EntryMode::FILE
+            };
+
+            let root = &self.core.root;
+            let path = format!("{}{}", &self.path, file.name);
+            let normalized_path = build_rel_path(root, &path);
+
+            // Update path cache when path doesn't exist.
+            // When Google Drive converts a format, for example, Microsoft PowerPoint,
+            // Google Drive keeps two entries with the same ID.
+            if let Ok(None) = self.core.path_cache.get(&path).await {
+                self.core.path_cache.insert(&path, &file.id).await;
+            }
+
+            let entry = oio::Entry::new(&normalized_path, Metadata::new(file_type));
+            ctx.entries.push_back(entry);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/gdrive/mod.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for gdrive service.
+#[cfg(feature = "services-gdrive")]
+pub(super) const DEFAULT_SCHEME: &str = "gdrive";
+#[cfg(feature = "services-gdrive")]
+mod backend;
+#[cfg(feature = "services-gdrive")]
+mod core;
+#[cfg(feature = "services-gdrive")]
+mod delete;
+#[cfg(feature = "services-gdrive")]
+mod error;
+#[cfg(feature = "services-gdrive")]
+mod lister;
+#[cfg(feature = "services-gdrive")]
+mod writer;
+
+#[cfg(feature = "services-gdrive")]
+mod builder;
+#[cfg(feature = "services-gdrive")]
+pub use builder::GdriveBuilder as Gdrive;
+
+mod config;
+pub use config::GdriveConfig;

data/core/src/services/gdrive/writer.rs

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::GdriveCore;
+use super::core::GdriveFile;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct GdriveWriter {
+    core: Arc<GdriveCore>,
+
+    path: String,
+
+    file_id: Option<String>,
+}
+
+impl GdriveWriter {
+    pub fn new(core: Arc<GdriveCore>, path: String, file_id: Option<String>) -> Self {
+        GdriveWriter {
+            core,
+            path,
+
+            file_id,
+        }
+    }
+}
+
+impl oio::OneShotWrite for GdriveWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let size = bs.len();
+
+        let resp = if let Some(file_id) = &self.file_id {
+            self.core
+                .gdrive_upload_overwrite_simple_request(file_id, size as u64, bs)
+                .await
+        } else {
+            self.core
+                .gdrive_upload_simple_request(&self.path, size as u64, bs)
+                .await
+        }?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::CREATED => {
+                // If we don't have the file id before, let's update the cache to avoid re-fetching.
+                if self.file_id.is_none() {
+                    let bs = resp.into_body();
+                    let file: GdriveFile =
+                        serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+                    self.core.path_cache.insert(&self.path, &file.id).await;
+                }
+                Ok(Metadata::default())
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/ghac/backend.rs

@@ -0,0 +1,285 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::env;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+use log::debug;
+use sha2::Digest;
+
+use super::core::*;
+use super::error::parse_error;
+use super::writer::GhacWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::ghac::core::GhacCore;
+use crate::services::GhacConfig;
+use crate::*;
+fn value_or_env(
+    explicit_value: Option<String>,
+    env_var_name: &str,
+    operation: &'static str,
+) -> Result<String> {
+    if let Some(value) = explicit_value {
+        return Ok(value);
+    }
+
+    env::var(env_var_name).map_err(|err| {
+        let text = format!("{env_var_name} not found, maybe not in github action environment?");
+        Error::new(ErrorKind::ConfigInvalid, text)
+            .with_operation(operation)
+            .set_source(err)
+    })
+}
+
+impl Configurator for GhacConfig {
+    type Builder = GhacBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        GhacBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// GitHub Action Cache Services support.
+#[doc = include_str!("docs.md")]
+#[derive(Debug, Default)]
+pub struct GhacBuilder {
+    config: GhacConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl GhacBuilder {
+    /// set the working directory root of backend
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// set the version that used by cache.
+    ///
+    /// The version is the unique value that provides namespacing.
+    /// It's better to make sure this value is only used by this backend.
+    ///
+    /// If not set, we will use `opendal` as default.
+    pub fn version(mut self, version: &str) -> Self {
+        if !version.is_empty() {
+            self.config.version = Some(version.to_string())
+        }
+
+        self
+    }
+
+    /// Set the endpoint for ghac service.
+    ///
+    /// For example, this is provided as the `ACTIONS_CACHE_URL` environment variable by the GHA runner.
+    ///
+    /// Default: the value of the `ACTIONS_CACHE_URL` environment variable.
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        if !endpoint.is_empty() {
+            self.config.endpoint = Some(endpoint.to_string())
+        }
+        self
+    }
+
+    /// Set the runtime token for ghac service.
+    ///
+    /// For example, this is provided as the `ACTIONS_RUNTIME_TOKEN` environment variable by the GHA
+    /// runner.
+    ///
+    /// Default: the value of the `ACTIONS_RUNTIME_TOKEN` environment variable.
+    pub fn runtime_token(mut self, runtime_token: &str) -> Self {
+        if !runtime_token.is_empty() {
+            self.config.runtime_token = Some(runtime_token.to_string())
+        }
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+}
+
+impl Builder for GhacBuilder {
+    type Config = GhacConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {self:?}");
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let service_version = get_cache_service_version();
+        debug!("backend use service version {service_version:?}");
+
+        let mut version = self
+            .config
+            .version
+            .clone()
+            .unwrap_or_else(|| "opendal".to_string());
+        debug!("backend use version {version}");
+        // ghac requires to use hex digest of Sha256 as version.
+        if matches!(service_version, GhacVersion::V2) {
+            let hash = sha2::Sha256::digest(&version);
+            version = format!("{hash:x}");
+        }
+
+        let cache_url = self
+            .config
+            .endpoint
+            .unwrap_or_else(|| get_cache_service_url(service_version));
+        if cache_url.is_empty() {
+            return Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "cache url for ghac not found, maybe not in github action environment?".to_string(),
+            ));
+        }
+
+        let core = GhacCore {
+            info: {
+                let am = AccessorInfo::default();
+                am.set_scheme(DEFAULT_SCHEME)
+                    .set_root(&root)
+                    .set_name(&version)
+                    .set_native_capability(Capability {
+                        stat: true,
+
+                        read: true,
+
+                        write: true,
+                        write_can_multi: true,
+
+                        shared: true,
+
+                        ..Default::default()
+                    });
+
+                // allow deprecated api here for compatibility
+                #[allow(deprecated)]
+                if let Some(client) = self.http_client {
+                    am.update_http_client(|_| client);
+                }
+
+                am.into()
+            },
+            root,
+
+            cache_url,
+            catch_token: value_or_env(
+                self.config.runtime_token,
+                ACTIONS_RUNTIME_TOKEN,
+                "Builder::build",
+            )?,
+            version,
+
+            service_version,
+        };
+
+        Ok(GhacBackend {
+            core: Arc::new(core),
+        })
+    }
+}
+
+/// Backend for github action cache services.
+#[derive(Debug, Clone)]
+pub struct GhacBackend {
+    core: Arc<GhacCore>,
+}
+
+impl Access for GhacBackend {
+    type Reader = HttpBody;
+    type Writer = GhacWriter;
+    type Lister = ();
+    type Deleter = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    /// Some self-hosted GHES instances are backed by AWS S3 services which only returns
+    /// signed url with `GET` method. So we will use `GET` with empty range to simulate
+    /// `HEAD` instead.
+    ///
+    /// In this way, we can support both self-hosted GHES and `github.com`.
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let resp = self.core.ghac_stat(path).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT | StatusCode::RANGE_NOT_SATISFIABLE => {
+                let mut meta = parse_into_metadata(path, resp.headers())?;
+                // Correct content length via returning content range.
+                meta.set_content_length(
+                    meta.content_range()
+                        .expect("content range must be valid")
+                        .size()
+                        .expect("content range must contains size"),
+                );
+
+                Ok(RpStat::new(meta))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.ghac_read(path, args.range()).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let url = self.core.ghac_get_upload_url(path).await?;
+
+        Ok((
+            RpWrite::default(),
+            GhacWriter::new(self.core.clone(), path.to_string(), url)?,
+        ))
+    }
+}

data/core/src/services/ghac/config.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for GitHub Action Cache Services support.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct GhacConfig {
+    /// The root path for ghac.
+    pub root: Option<String>,
+    /// The version that used by cache.
+    pub version: Option<String>,
+    /// The endpoint for ghac service.
+    pub endpoint: Option<String>,
+    /// The runtime token for ghac service.
+    pub runtime_token: Option<String>,
+}