opendal 0.1.6.pre.rc.1

data/core/src/services/b2/delete.rs

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct B2Deleter {
+    core: Arc<B2Core>,
+}
+
+impl B2Deleter {
+    pub fn new(core: Arc<B2Core>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for B2Deleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.hide_file(&path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(()),
+            _ => {
+                let err = parse_error(resp);
+                match err.kind() {
+                    ErrorKind::NotFound => Ok(()),
+                    // Representative deleted
+                    ErrorKind::AlreadyExists => Ok(()),
+                    _ => Err(err),
+                }
+            }
+        }
+    }
+}

data/core/src/services/b2/docs.md

@@ -0,0 +1,54 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [x] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work directory for backend
+- `key_id`: B2 application key keyID
+- `application_key` B2 application key applicationKey
+- `bucket` B2 bucket name
+- `bucket_id` B2 bucket_id
+
+You can refer to [`B2Builder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::B2;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create backend builder
+    let mut builder = B2::default()
+        // set the storage bucket for OpenDAL
+        .root("/")
+        // set the key_id for OpenDAL
+        .application_key_id("xxxxxxxxxx")
+        // set the key_id for OpenDAL
+        .application_key("xxxxxxxxxx")
+        // set the     bucket name for OpenDAL
+        .bucket("opendal")
+        // set the bucket_id for OpenDAL
+        .bucket_id("xxxxxxxxxxxxx");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/b2/error.rs

@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// the error response of b2
+#[derive(Default, Debug, Deserialize)]
+#[allow(dead_code)]
+struct B2Error {
+    status: u32,
+    code: String,
+    message: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // Service b2 could return 403, show the authorization error
+        401 => (ErrorKind::PermissionDenied, true),
+        429 => (ErrorKind::RateLimited, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, b2_err) = serde_json::from_reader::<_, B2Error>(bs.clone().reader())
+        .map(|b2_err| (format!("{b2_err:?}"), Some(b2_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(b2_err) = b2_err {
+        (kind, retryable) = parse_b2_error_code(b2_err.code.as_str()).unwrap_or((kind, retryable));
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+/// Returns the `Error kind` of this code and whether the error is retryable.
+pub(crate) fn parse_b2_error_code(code: &str) -> Option<(ErrorKind, bool)> {
+    match code {
+        "already_hidden" => Some((ErrorKind::AlreadyExists, false)),
+        "no_such_file" => Some((ErrorKind::NotFound, false)),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use http::StatusCode;
+
+    use super::*;
+
+    #[test]
+    fn test_parse_b2_error_code() {
+        let code = "already_hidden";
+        assert_eq!(
+            parse_b2_error_code(code),
+            Some((crate::ErrorKind::AlreadyExists, false))
+        );
+
+        let code = "no_such_file";
+        assert_eq!(
+            parse_b2_error_code(code),
+            Some((crate::ErrorKind::NotFound, false))
+        );
+
+        let code = "not_found";
+        assert_eq!(parse_b2_error_code(code), None);
+    }
+
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"status": 403, "code": "access_denied", "message":"The provided customer-managed encryption key is wrong."}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::FORBIDDEN,
+            ),
+            (
+                r#"{"status": 404, "code": "not_found", "message":"File is not in B2 Cloud Storage."}"#,
+                ErrorKind::NotFound,
+                StatusCode::NOT_FOUND,
+            ),
+            (
+                r#"{"status": 401, "code": "bad_auth_token", "message":"The auth token used is not valid. Call b2_authorize_account again to either get a new one, or an error message describing the problem."}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::UNAUTHORIZED,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
+
+            let err = parse_error(resp);
+
+            assert_eq!(err.kind(), res.1);
+        }
+    }
+}

data/core/src/services/b2/lister.rs

@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::parse_file_info;
+use super::core::B2Core;
+use super::core::ListFileNamesResponse;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct B2Lister {
+    core: Arc<B2Core>,
+
+    path: String,
+    delimiter: Option<&'static str>,
+    limit: Option<usize>,
+
+    /// B2 starts listing **after** this specified key
+    start_after: Option<String>,
+}
+
+impl B2Lister {
+    pub fn new(
+        core: Arc<B2Core>,
+        path: &str,
+        recursive: bool,
+        limit: Option<usize>,
+        start_after: Option<&str>,
+    ) -> Self {
+        let delimiter = if recursive { None } else { Some("/") };
+        Self {
+            core,
+
+            path: path.to_string(),
+            delimiter,
+            limit,
+            start_after: start_after.map(String::from),
+        }
+    }
+}
+
+impl oio::PageList for B2Lister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .list_file_names(
+                Some(&self.path),
+                self.delimiter,
+                self.limit,
+                if ctx.token.is_empty() {
+                    self.start_after
+                        .as_ref()
+                        .map(|v| build_abs_path(&self.core.root, v))
+                } else {
+                    Some(ctx.token.clone())
+                },
+            )
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let output: ListFileNamesResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        if let Some(token) = output.next_file_name {
+            ctx.token = token;
+        } else {
+            ctx.done = true;
+        }
+
+        for file in output.files {
+            if let Some(start_after) = self.start_after.clone() {
+                if build_abs_path(&self.core.root, &start_after) == file.file_name {
+                    continue;
+                }
+            }
+            let file_name = file.file_name.clone();
+            let metadata = parse_file_info(&file);
+
+            ctx.entries.push_back(oio::Entry::new(
+                &build_rel_path(&self.core.root, &file_name),
+                metadata,
+            ))
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/b2/mod.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for b2 service.
+#[cfg(feature = "services-b2")]
+pub(super) const DEFAULT_SCHEME: &str = "b2";
+#[cfg(feature = "services-b2")]
+mod core;
+#[cfg(feature = "services-b2")]
+mod delete;
+#[cfg(feature = "services-b2")]
+mod error;
+#[cfg(feature = "services-b2")]
+mod lister;
+#[cfg(feature = "services-b2")]
+mod writer;
+
+#[cfg(feature = "services-b2")]
+mod backend;
+#[cfg(feature = "services-b2")]
+pub use backend::B2Builder as B2;
+
+mod config;
+
+pub use config::B2Config;

data/core/src/services/b2/writer.rs

@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::B2Core;
+use super::core::StartLargeFileResponse;
+use super::core::UploadPartResponse;
+use super::core::UploadResponse;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub type B2Writers = oio::MultipartWriter<B2Writer>;
+
+pub struct B2Writer {
+    core: Arc<B2Core>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl B2Writer {
+    pub fn new(core: Arc<B2Core>, path: &str, op: OpWrite) -> Self {
+        B2Writer {
+            core,
+            path: path.to_string(),
+            op,
+        }
+    }
+
+    pub fn parse_body_into_meta(path: &str, resp: UploadResponse) -> Metadata {
+        let mut meta = Metadata::new(EntryMode::from_path(path));
+
+        if let Some(md5) = resp.content_md5 {
+            meta.set_content_md5(&md5);
+        }
+
+        if let Some(content_type) = resp.content_type {
+            meta.set_content_type(&content_type);
+        }
+
+        meta.set_content_length(resp.content_length);
+
+        meta
+    }
+}
+
+impl oio::MultipartWrite for B2Writer {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .upload_file(&self.path, Some(size), &self.op, body)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: UploadResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                let meta = Self::parse_body_into_meta(&self.path, result);
+
+                Ok(meta)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self.core.start_large_file(&self.path, &self.op).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: StartLargeFileResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(result.file_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<oio::MultipartPart> {
+        // B2 requires part number must between [1..=10000]
+        let part_number = part_number + 1;
+
+        let resp = self
+            .core
+            .upload_part(upload_id, part_number, size, body)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: UploadPartResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(oio::MultipartPart {
+                    etag: result.content_sha1,
+                    part_number,
+                    checksum: None,
+                })
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        parts: &[oio::MultipartPart],
+    ) -> Result<Metadata> {
+        let part_sha1_array = parts
+            .iter()
+            .map(|p| {
+                let binding = p.etag.clone();
+                let sha1 = binding.strip_prefix("unverified:");
+                let Some(sha1) = sha1 else {
+                    return "".to_string();
+                };
+                sha1.to_string()
+            })
+            .collect();
+
+        let resp = self
+            .core
+            .finish_large_file(upload_id, part_sha1_array)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: UploadResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                let meta = Self::parse_body_into_meta(&self.path, result);
+
+                Ok(meta)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self.core.cancel_large_file(upload_id).await?;
+        match resp.status() {
+            // b2 returns code 200 if abort succeeds.
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}