opendal 0.1.6.pre.rc.1

data/core/src/services/gcs/delete.rs

@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::*;
+
+pub struct GcsDeleter {
+    core: Arc<GcsCore>,
+}
+
+impl GcsDeleter {
+    pub fn new(core: Arc<GcsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::BatchDelete for GcsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.gcs_delete_object(&path).await?;
+
+        // deleting not existing objects is ok
+        if resp.status().is_success() || resp.status() == StatusCode::NOT_FOUND {
+            Ok(())
+        } else {
+            Err(parse_error(resp))
+        }
+    }
+
+    async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+        let paths: Vec<String> = batch.into_iter().map(|(p, _)| p).collect();
+        let resp = self.core.gcs_delete_objects(paths.clone()).await?;
+
+        let status = resp.status();
+
+        // If the overall request isn't formatted correctly and Cloud Storage is unable to parse it into sub-requests, you receive a 400 error.
+        // Otherwise, Cloud Storage returns a 200 status code, even if some or all of the sub-requests fail.
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let boundary = parse_multipart_boundary(resp.headers())?.ok_or_else(|| {
+            Error::new(
+                ErrorKind::Unexpected,
+                "gcs batch delete response content type is empty",
+            )
+        })?;
+        let multipart: Multipart<MixedPart> = Multipart::new()
+            .with_boundary(boundary)
+            .parse(resp.into_body().to_bytes())?;
+        let parts = multipart.into_parts();
+
+        let mut batched_result = BatchDeleteResult::default();
+
+        for (i, part) in parts.into_iter().enumerate() {
+            let resp = part.into_response();
+            // TODO: maybe we can take it directly?
+            let path = paths[i].clone();
+
+            // deleting not existing objects is ok
+            if resp.status().is_success() || resp.status() == StatusCode::NOT_FOUND {
+                batched_result.succeeded.push((path, OpDelete::default()));
+            } else {
+                batched_result
+                    .failed
+                    .push((path, OpDelete::default(), parse_error(resp)));
+            }
+        }
+
+        // If no object is deleted, return directly.
+        if batched_result.succeeded.is_empty() {
+            let err = batched_result.failed.remove(0).2;
+            return Err(err);
+        }
+
+        Ok(batched_result)
+    }
+}

data/core/src/services/gcs/docs.md

@@ -0,0 +1,76 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [x] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work directory for backend
+- `bucket`: Set the container name for backend
+- `endpoint`: Customizable endpoint setting
+- `credential`: Service Account or External Account JSON, in base64
+- `credential_path`: local path to Service Account or External Account JSON file
+- `service_account`: name of Service Account
+- `predefined_acl`: Predefined ACL for GCS
+- `default_storage_class`: Default storage class for GCS
+
+Refer to public API docs for more information. For authentication related options, read on.
+
+## Options to authenticate to GCS
+
+OpenDAL supports the following authentication options:
+
+1. Provide a base64-ed JSON key string with `credential`
+2. Provide a JSON key file at explicit path with `credential_path`
+3. Provide a JSON key file at implicit path
+    - `GcsBackend` will attempt to load Service Account key from [ADC well-known places](https://cloud.google.com/docs/authentication/application-default-credentials).
+4. Fetch access token from [VM metadata](https://cloud.google.com/docs/authentication/rest#metadata-server)
+    - Only works when running inside Google Cloud.
+    - If a non-default Service Account name is required, set with `service_account`. Otherwise, nothing need to be set.
+5. A custom `TokenLoader` via `GcsBuilder.customized_token_loader()`
+
+Notes:
+
+- When a Service Account key is provided, it will be used to create access tokens (VM metadata will not be used).
+- Explicit Service Account key, in json or path, always take precedence over ADC-defined key paths.
+- Due to [limitation in GCS](https://cloud.google.com/storage/docs/authentication/signatures#signing-process), a private key is required to create Pre-signed URL. Currently, OpenDAL only supports Service Account key.
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Gcs;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create backend builder
+    let mut builder = Gcs::default()
+       // set the storage bucket for OpenDAL
+       .bucket("test")
+       // set the working directory root for GCS
+       // all operations will happen within it
+       .root("/path/to/dir")
+       // set the credentials with service account
+       .credential("service account JSON in base64")
+       // set the predefined ACL for GCS
+       .predefined_acl("publicRead")
+       // set the default storage class for GCS
+       .default_storage_class("STANDARD");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/gcs/error.rs

@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde_json::de;
+
+use crate::raw::*;
+use crate::*;
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+struct GcsErrorResponse {
+    error: GcsError,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+struct GcsError {
+    code: usize,
+    message: String,
+    errors: Vec<GcsErrorDetail>,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+struct GcsErrorDetail {
+    domain: String,
+    location: String,
+    location_type: String,
+    message: String,
+    reason: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match de::from_slice::<GcsErrorResponse>(&bs) {
+        Ok(gcs_err) => format!("{gcs_err:?}"),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
+{
+"error": {
+ "errors": [
+  {
+   "domain": "global",
+   "reason": "required",
+   "message": "Login Required",
+   "locationType": "header",
+   "location": "Authorization"
+  }
+ ],
+ "code": 401,
+ "message": "Login Required"
+ }
+}
+"#,
+        );
+
+        let out: GcsErrorResponse = de::from_slice(&bs).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.error.code, 401);
+        assert_eq!(out.error.message, "Login Required");
+        assert_eq!(out.error.errors[0].domain, "global");
+        assert_eq!(out.error.errors[0].reason, "required");
+        assert_eq!(out.error.errors[0].message, "Login Required");
+        assert_eq!(out.error.errors[0].location_type, "header");
+        assert_eq!(out.error.errors[0].location, "Authorization");
+    }
+}

data/core/src/services/gcs/lister.rs

@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use serde_json;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+/// GcsLister takes over task of listing objects and
+/// helps walking directory
+pub struct GcsLister {
+    core: Arc<GcsCore>,
+
+    path: String,
+    delimiter: &'static str,
+    limit: Option<usize>,
+
+    /// Filter results to objects whose names are lexicographically
+    /// **equal to or after** startOffset
+    start_after: Option<String>,
+}
+
+impl GcsLister {
+    /// Generate a new directory walker
+    pub fn new(
+        core: Arc<GcsCore>,
+        path: &str,
+        recursive: bool,
+        limit: Option<usize>,
+        start_after: Option<&str>,
+    ) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+        Self {
+            core,
+
+            path: path.to_string(),
+            delimiter,
+            limit,
+            start_after: start_after.map(String::from),
+        }
+    }
+}
+
+impl oio::PageList for GcsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .gcs_list_objects(
+                &self.path,
+                &ctx.token,
+                self.delimiter,
+                self.limit,
+                if ctx.token.is_empty() {
+                    self.start_after.clone()
+                } else {
+                    None
+                },
+            )
+            .await?;
+
+        if !resp.status().is_success() {
+            return Err(parse_error(resp));
+        }
+        let bytes = resp.into_body();
+
+        let output: ListResponse =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        if let Some(token) = &output.next_page_token {
+            ctx.token.clone_from(token);
+        } else {
+            ctx.done = true;
+        }
+
+        for prefix in output.prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix),
+                Metadata::new(EntryMode::DIR),
+            );
+
+            ctx.entries.push_back(de);
+        }
+
+        for object in output.items {
+            // exclude the inclusive start_after itself
+            let mut path = build_rel_path(&self.core.root, &object.name);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+            if self.start_after.as_ref() == Some(&path) {
+                continue;
+            }
+
+            let mut meta = Metadata::new(EntryMode::from_path(&path));
+
+            // set metadata fields
+            meta.set_content_md5(object.md5_hash.as_str());
+            meta.set_etag(object.etag.as_str());
+
+            let size = object.size.parse().map_err(|e| {
+                Error::new(ErrorKind::Unexpected, "parse u64 from list response").set_source(e)
+            })?;
+            meta.set_content_length(size);
+            if !object.content_type.is_empty() {
+                meta.set_content_type(&object.content_type);
+            }
+
+            meta.set_last_modified(parse_datetime_from_rfc3339(object.updated.as_str())?);
+
+            let de = oio::Entry::with(path, meta);
+
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/gcs/mod.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for gcs service.
+#[cfg(feature = "services-gcs")]
+pub(super) const DEFAULT_SCHEME: &str = "gcs";
+#[cfg(feature = "services-gcs")]
+mod core;
+#[cfg(feature = "services-gcs")]
+mod delete;
+#[cfg(feature = "services-gcs")]
+mod error;
+#[cfg(feature = "services-gcs")]
+mod lister;
+#[cfg(feature = "services-gcs")]
+mod uri;
+#[cfg(feature = "services-gcs")]
+mod writer;
+
+#[cfg(feature = "services-gcs")]
+mod backend;
+#[cfg(feature = "services-gcs")]
+pub use backend::GcsBuilder as Gcs;
+
+mod config;
+pub use config::GcsConfig;

data/core/src/services/gcs/uri.rs

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use percent_encoding::utf8_percent_encode;
+use percent_encoding::AsciiSet;
+use percent_encoding::NON_ALPHANUMERIC;
+
+/// PATH_ENCODE_SET is the encode set for http url path.
+///
+/// This set follows [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) which will encode all non-ASCII characters except `A-Z a-z 0-9 - _ . ! ~ * ' ( )`
+///
+/// Following characters is allowed in GCS, check "https://cloud.google.com/storage/docs/request-endpoints#encoding" for details
+static GCS_PATH_ENCODE_SET: AsciiSet = NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'*');
+
+/// percent_encode_path will do percent encoding for http encode path.
+///
+/// Follows [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent)
+/// which will encode all non-ASCII characters except `A-Z a-z 0-9 - _ . *`
+///
+/// GCS does not allow '/'s in paths, this should also be dealt with
+pub(super) fn percent_encode_path(path: &str) -> String {
+    utf8_percent_encode(path, &GCS_PATH_ENCODE_SET).to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_percent_encode_path() {
+        let cases = vec![
+            (
+                "Reserved Characters",
+                ";,/?:@&=+$",
+                "%3B%2C%2F%3F%3A%40%26%3D%2B%24",
+            ),
+            ("Unescaped Characters", "-_.*", "-_.*"),
+            ("Number Sign", "#", "%23"),
+            (
+                "Alphanumeric Characters + Space",
+                "ABC abc 123",
+                "ABC%20abc%20123",
+            ),
+            (
+                "Unicode",
+                "你好，世界！❤",
+                "%E4%BD%A0%E5%A5%BD%EF%BC%8C%E4%B8%96%E7%95%8C%EF%BC%81%E2%9D%A4",
+            ),
+        ];
+
+        for (name, input, expected) in cases {
+            let actual = percent_encode_path(input);
+
+            assert_eq!(actual, expected, "{name}");
+        }
+    }
+}