opendal 0.1.6.pre.rc.1

data/core/src/services/github/core.rs

@@ -0,0 +1,351 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use base64::Engine;
+use bytes::Buf;
+use bytes::Bytes;
+use http::header;
+use http::request;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde::Serialize;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+/// Core of [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support.
+#[derive(Clone)]
+pub struct GithubCore {
+    pub info: Arc<AccessorInfo>,
+    /// The root of this core.
+    pub root: String,
+    /// Github access_token.
+    pub token: Option<String>,
+    /// Github repo owner.
+    pub owner: String,
+    /// Github repo name.
+    pub repo: String,
+}
+
+impl Debug for GithubCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Backend")
+            .field("root", &self.root)
+            .field("owner", &self.owner)
+            .field("repo", &self.repo)
+            .finish_non_exhaustive()
+    }
+}
+
+impl GithubCore {
+    #[inline]
+    pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
+        self.info.http_client().send(req).await
+    }
+
+    pub fn sign(&self, req: request::Builder) -> Result<request::Builder> {
+        let mut req = req
+            .header(header::USER_AGENT, format!("opendal-{VERSION}"))
+            .header("X-GitHub-Api-Version", "2022-11-28");
+
+        // Github access_token is optional.
+        if let Some(token) = &self.token {
+            req = req.header(
+                header::AUTHORIZATION,
+                format_authorization_by_bearer(token)?,
+            )
+        }
+
+        Ok(req)
+    }
+}
+
+impl GithubCore {
+    pub async fn get_file_sha(&self, path: &str) -> Result<Option<String>> {
+        // if the token is not set, we should not try to get the sha of the file.
+        if self.token.is_none() {
+            return Err(Error::new(
+                ErrorKind::PermissionDenied,
+                "Github access_token is not set",
+            ));
+        }
+
+        let resp = self.stat(path).await?;
+
+        match resp.status() {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let resp: Entry =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(Some(resp.sha))
+            }
+            StatusCode::NOT_FOUND => Ok(None),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn stat(&self, path: &str) -> Result<Response<Buffer>> {
+        let path = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/contents/{}",
+            self.owner,
+            self.repo,
+            percent_encode_path(&path)
+        );
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::Stat);
+
+        let req = self.sign(req)?;
+
+        let req = req
+            .header("Accept", "application/vnd.github.object+json")
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
+        let path = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/contents/{}",
+            self.owner,
+            self.repo,
+            percent_encode_path(&path)
+        );
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::Read);
+
+        let req = self.sign(req)?;
+
+        let req = req
+            .header(header::ACCEPT, "application/vnd.github.raw+json")
+            .header(header::RANGE, range.to_header())
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub async fn upload(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
+        let sha = self.get_file_sha(path).await?;
+
+        let path = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/contents/{}",
+            self.owner,
+            self.repo,
+            percent_encode_path(&path)
+        );
+
+        let req = Request::put(url);
+
+        let req = req.extension(Operation::Write);
+
+        let req = self.sign(req)?;
+
+        let mut req_body = CreateOrUpdateContentsRequest {
+            message: format!("Write {} at {} via opendal", path, chrono::Local::now()),
+            content: base64::engine::general_purpose::STANDARD.encode(bs.to_bytes()),
+            sha: None,
+        };
+
+        if let Some(sha) = sha {
+            req_body.sha = Some(sha);
+        }
+
+        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
+
+        let req = req
+            .header("Accept", "application/vnd.github+json")
+            .body(Buffer::from(req_body))
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn delete(&self, path: &str) -> Result<()> {
+        // If path is a directory, we should delete path/.gitkeep
+        let formatted_path = format!("{path}.gitkeep");
+        let p = if path.ends_with('/') {
+            formatted_path.as_str()
+        } else {
+            path
+        };
+
+        let Some(sha) = self.get_file_sha(p).await? else {
+            return Ok(());
+        };
+
+        let path = build_abs_path(&self.root, p);
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/contents/{}",
+            self.owner,
+            self.repo,
+            percent_encode_path(&path)
+        );
+
+        let req = Request::delete(url);
+
+        let req = req.extension(Operation::Delete);
+
+        let req = self.sign(req)?;
+
+        let req_body = DeleteContentsRequest {
+            message: format!("Delete {} at {} via opendal", path, chrono::Local::now()),
+            sha,
+        };
+
+        let req_body = serde_json::to_vec(&req_body).map_err(new_json_serialize_error)?;
+
+        let req = req
+            .header("Accept", "application/vnd.github.object+json")
+            .body(Buffer::from(Bytes::from(req_body)))
+            .map_err(new_request_build_error)?;
+
+        let resp = self.send(req).await?;
+
+        match resp.status() {
+            StatusCode::OK => Ok(()),
+            StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn list(&self, path: &str) -> Result<ListResponse> {
+        let path = build_abs_path(&self.root, path);
+
+        let url = format!(
+            "https://api.github.com/repos/{}/{}/contents/{}",
+            self.owner,
+            self.repo,
+            percent_encode_path(&path)
+        );
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::List);
+
+        let req = self.sign(req)?;
+
+        let req = req
+            .header("Accept", "application/vnd.github.object+json")
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        let resp = self.send(req).await?;
+
+        match resp.status() {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let resp: ListResponse =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(resp)
+            }
+            StatusCode::NOT_FOUND => Ok(ListResponse::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    /// We use git_url to call github's Tree based API.
+    pub async fn list_with_recursive(&self, git_url: &str) -> Result<Vec<Tree>> {
+        let url = format!("{git_url}?recursive=true");
+
+        let req = Request::get(url);
+
+        let req = req.extension(Operation::List);
+
+        let req = self.sign(req)?;
+
+        let req = req
+            .header("Accept", "application/vnd.github.object+json")
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        let resp = self.send(req).await?;
+
+        match resp.status() {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let resp: ListTreeResponse =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(resp.tree)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+#[derive(Default, Debug, Clone, Serialize)]
+pub struct CreateOrUpdateContentsRequest {
+    pub message: String,
+    pub content: String,
+    pub sha: Option<String>,
+}
+
+#[derive(Default, Debug, Clone, Serialize)]
+pub struct DeleteContentsRequest {
+    pub message: String,
+    pub sha: String,
+}
+
+#[derive(Default, Debug, Clone, Deserialize)]
+pub struct ListTreeResponse {
+    pub tree: Vec<Tree>,
+}
+
+#[derive(Default, Debug, Clone, Deserialize)]
+pub struct Tree {
+    pub path: String,
+    #[serde(rename = "type")]
+    pub type_field: String,
+    pub size: Option<u64>,
+    pub sha: String,
+}
+
+#[derive(Default, Debug, Clone, Deserialize)]
+pub struct ListResponse {
+    pub git_url: String,
+    pub entries: Vec<Entry>,
+}
+
+#[derive(Default, Debug, Clone, Deserialize)]
+pub struct Entry {
+    pub path: String,
+    pub sha: String,
+    pub size: u64,
+    #[serde(rename = "type")]
+    pub type_field: String,
+}

data/core/src/services/github/delete.rs

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct GithubDeleter {
+    core: Arc<GithubCore>,
+}
+
+impl GithubDeleter {
+    pub fn new(core: Arc<GithubCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for GithubDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        match self.core.delete(&path).await {
+            Ok(_) => Ok(()),
+            Err(err) => Err(err),
+        }
+    }
+}

data/core/src/services/github/docs.md

@@ -0,0 +1,52 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [ ] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [x] list
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work directory for backend
+- `token`: Github access token
+- `owner`: Github owner
+- `repo`: Github repository
+
+You can refer to [`GithubBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Github;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create backend builder
+    let mut builder = Github::default()
+        // set the storage root for OpenDAL
+        .root("/")
+        // set the access token for Github API
+        .token("your_access_token")
+        // set the owner for Github
+        .owner("your_owner")
+        // set the repository for Github
+        .repo("your_repo");
+
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/github/error.rs

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+#[derive(Default, Debug, Deserialize)]
+#[allow(dead_code)]
+struct GithubError {
+    error: GithubSubError,
+}
+
+#[derive(Default, Debug, Deserialize)]
+#[allow(dead_code)]
+struct GithubSubError {
+    message: String,
+    documentation_url: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status.as_u16() {
+        401 | 403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // https://github.com/apache/opendal/issues/4146
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/423
+        // We should retry it when we get 423 error.
+        423 => (ErrorKind::RateLimited, true),
+        // Service like Upyun could return 499 error with a message like:
+        // Client Disconnect, we should retry it.
+        499 => (ErrorKind::Unexpected, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, _github_content_err) =
+        serde_json::from_reader::<_, GithubError>(bs.clone().reader())
+            .map(|github_content_err| (format!("{github_content_err:?}"), Some(github_content_err)))
+            .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod test {
+    use http::StatusCode;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![(
+            r#"{
+                "message": "Not Found",
+                "documentation_url": "https://docs.github.com/rest/repos/contents#get-repository-content"
+            }"#,
+            ErrorKind::NotFound,
+            StatusCode::NOT_FOUND,
+        )];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
+
+            let err = parse_error(resp);
+
+            assert_eq!(err.kind(), res.1);
+        }
+    }
+}

data/core/src/services/github/lister.rs

@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::GithubCore;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct GithubLister {
+    core: Arc<GithubCore>,
+
+    path: String,
+    recursive: bool,
+}
+
+impl GithubLister {
+    pub fn new(core: Arc<GithubCore>, path: &str, recursive: bool) -> Self {
+        Self {
+            core,
+
+            path: path.to_string(),
+            recursive,
+        }
+    }
+}
+
+impl oio::PageList for GithubLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self.core.list(&self.path).await?;
+
+        // Record whether there is a dir in the list so that we need recursive later.
+        let has_dir = resp.entries.iter().any(|e| e.type_field == "dir");
+
+        ctx.done = true;
+
+        if !self.recursive || !has_dir {
+            for entry in resp.entries {
+                let path = build_rel_path(&self.core.root, &entry.path);
+                let entry = if entry.type_field == "dir" {
+                    let path = format!("{path}/");
+                    Entry::new(&path, Metadata::new(EntryMode::DIR))
+                } else {
+                    if path.ends_with(".gitkeep") {
+                        continue;
+                    }
+                    let m = Metadata::new(EntryMode::FILE)
+                        .with_content_length(entry.size)
+                        .with_etag(entry.sha);
+                    Entry::new(&path, m)
+                };
+                ctx.entries.push_back(entry);
+            }
+            if !self.path.ends_with('/') {
+                ctx.entries.push_back(Entry::new(
+                    &format!("{}/", self.path),
+                    Metadata::new(EntryMode::DIR),
+                ));
+            }
+            return Ok(());
+        }
+
+        // if recursive is true and there is a dir in the list, we need to list it recursively.
+
+        let tree = self.core.list_with_recursive(&resp.git_url).await?;
+        for t in tree {
+            let path = if self.path == "/" {
+                t.path
+            } else {
+                format!("{}/{}", self.path, t.path)
+            };
+            let entry = if t.type_field == "tree" {
+                let path = format!("{path}/");
+                Entry::new(&path, Metadata::new(EntryMode::DIR))
+            } else {
+                if path.ends_with(".gitkeep") {
+                    continue;
+                }
+                let mut m = Metadata::new(EntryMode::FILE).with_etag(t.sha);
+
+                if let Some(size) = t.size {
+                    m = m.with_content_length(size);
+                }
+                Entry::new(&path, m)
+            };
+            ctx.entries.push_back(entry);
+        }
+        if !self.path.ends_with('/') {
+            ctx.entries.push_back(Entry::new(
+                &format!("{}/", self.path),
+                Metadata::new(EntryMode::DIR),
+            ));
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/github/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for github service.
+#[cfg(feature = "services-github")]
+pub(super) const DEFAULT_SCHEME: &str = "github";
+#[cfg(feature = "services-github")]
+mod core;
+#[cfg(feature = "services-github")]
+mod delete;
+#[cfg(feature = "services-github")]
+mod error;
+#[cfg(feature = "services-github")]
+mod lister;
+#[cfg(feature = "services-github")]
+mod writer;
+
+#[cfg(feature = "services-github")]
+mod backend;
+#[cfg(feature = "services-github")]
+pub use backend::GithubBuilder as Github;
+
+mod config;
+pub use config::GithubConfig;