opendal 0.1.6.pre.rc.1

data/core/src/services/azblob/delete.rs

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::*;
+
+pub struct AzblobDeleter {
+    core: Arc<AzblobCore>,
+}
+
+impl AzblobDeleter {
+    pub fn new(core: Arc<AzblobCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::BatchDelete for AzblobDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.azblob_delete_blob(&path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::ACCEPTED | StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+        // TODO: Add remove version support.
+        let paths = batch.into_iter().map(|(p, _)| p).collect::<Vec<_>>();
+
+        // construct and complete batch request
+        let resp = self.core.azblob_batch_delete(&paths).await?;
+
+        // check response status
+        if resp.status() != StatusCode::ACCEPTED {
+            return Err(parse_error(resp));
+        }
+
+        // get boundary from response header
+        let boundary = parse_multipart_boundary(resp.headers())?
+            .ok_or_else(|| {
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "invalid response: no boundary provided in header",
+                )
+            })?
+            .to_string();
+
+        let bs = resp.into_body().to_bytes();
+        let multipart: Multipart<MixedPart> =
+            Multipart::new().with_boundary(&boundary).parse(bs)?;
+        let parts = multipart.into_parts();
+
+        if paths.len() != parts.len() {
+            return Err(Error::new(
+                ErrorKind::Unexpected,
+                "invalid batch response, paths and response parts don't match",
+            ));
+        }
+
+        let mut batched_result = BatchDeleteResult::default();
+
+        for (i, part) in parts.into_iter().enumerate() {
+            let resp = part.into_response();
+            let path = paths[i].clone();
+
+            // deleting not existing objects is ok
+            if resp.status() == StatusCode::ACCEPTED || resp.status() == StatusCode::NOT_FOUND {
+                batched_result.succeeded.push((path, OpDelete::default()));
+            } else {
+                batched_result
+                    .failed
+                    .push((path, OpDelete::default(), parse_error(resp)));
+            }
+        }
+
+        // If no object is deleted, return directly.
+        if batched_result.succeeded.is_empty() {
+            let err = batched_result.failed.remove(0).2;
+            return Err(err);
+        }
+
+        Ok(batched_result)
+    }
+}

data/core/src/services/azblob/docs.md

@@ -0,0 +1,77 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] append
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [x] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- `container`: Set the container name for backend.
+- `endpoint`: Set the endpoint for backend.
+- `account_name`: Set the account_name for backend.
+- `account_key`: Set the account_key for backend.
+
+Refer to public API docs for more information.
+
+## Examples
+
+This example works on [Azurite](https://github.com/Azure/Azurite) for local developments.
+
+### Start local blob service
+
+```shell
+docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite
+az storage container create --name test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+```
+
+### Init OpenDAL Operator
+
+### Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Azblob;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create azblob backend builder.
+    let mut builder = Azblob::default()
+        // Set the root for azblob, all operations will happen under this root.
+        //
+        // NOTE: the root must be absolute path.
+        .root("/path/to/dir")
+        // Set the container name, this is required.
+        .container("test")
+        // Set the endpoint, this is required.
+        //
+        // For examples:
+        // - "http://127.0.0.1:10000/devstoreaccount1"
+        // - "https://accountname.blob.core.windows.net"
+        .endpoint("http://127.0.0.1:10000/devstoreaccount1")
+        // Set the account_name and account_key.
+        //
+        // OpenDAL will try load credential from the env.
+        // If credential not set and no valid credential in env, OpenDAL will
+        // send request without signing like anonymous user.
+        .account_name("devstoreaccount1")
+        .account_key("Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");
+
+    // `Accessor` provides the low level APIs, we will use `Operator` normally.
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/azblob/error.rs

@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// AzblobError is the error returned by azure blob service.
+#[derive(Default, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct AzblobError {
+    code: String,
+    message: String,
+    query_parameter_name: String,
+    query_parameter_value: String,
+    reason: String,
+}
+
+impl Debug for AzblobError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("AzblobError");
+        de.field("code", &self.code);
+        // replace `\n` to ` ` for better reading.
+        de.field("message", &self.message.replace('\n', " "));
+
+        if !self.query_parameter_name.is_empty() {
+            de.field("query_parameter_name", &self.query_parameter_name);
+        }
+        if !self.query_parameter_value.is_empty() {
+            de.field("query_parameter_value", &self.query_parameter_value);
+        }
+        if !self.reason.is_empty() {
+            de.field("reason", &self.reason);
+        }
+
+        de.finish()
+    }
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED | StatusCode::CONFLICT => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let bs_content = bs.chunk();
+    let mut message = match de::from_reader::<_, AzblobError>(bs_content.reader()) {
+        Ok(azblob_err) => format!("{azblob_err:?}"),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    // If there is no body here, fill with error code.
+    if message.is_empty() {
+        if let Some(v) = parts.headers.get("x-ms-error-code") {
+            if let Ok(code) = v.to_str() {
+                message = format!(
+                    "{:?}",
+                    AzblobError {
+                        code: code.to_string(),
+                        ..Default::default()
+                    }
+                )
+            }
+        }
+    }
+
+    let mut err = Error::new(kind, &message);
+
+    err = with_azure_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_error() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" encoding="utf-8"?>
+<Error>
+  <Code>string-value</Code>
+  <Message>string-value</Message>
+</Error>
+"#,
+        );
+
+        let out: AzblobError = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code, "string-value");
+        assert_eq!(out.message, "string-value");
+    }
+
+    #[test]
+    fn test_parse_error_with_reason() {
+        let bs = bytes::Bytes::from(
+            r#"
+<?xml version="1.0" encoding="utf-8"?>
+<Error>
+  <Code>InvalidQueryParameterValue</Code>
+  <Message>Value for one of the query parameters specified in the request URI is invalid.</Message>
+  <QueryParameterName>popreceipt</QueryParameterName>
+  <QueryParameterValue>33537277-6a52-4a2b-b4eb-0f905051827b</QueryParameterValue>
+  <Reason>invalid receipt format</Reason>
+</Error>
+"#,
+        );
+
+        let out: AzblobError = de::from_reader(bs.reader()).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(out.code, "InvalidQueryParameterValue");
+        assert_eq!(
+            out.message,
+            "Value for one of the query parameters specified in the request URI is invalid."
+        );
+        assert_eq!(out.query_parameter_name, "popreceipt");
+        assert_eq!(
+            out.query_parameter_value,
+            "33537277-6a52-4a2b-b4eb-0f905051827b"
+        );
+        assert_eq!(out.reason, "invalid receipt format");
+    }
+}

data/core/src/services/azblob/lister.rs

@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use quick_xml::de;
+
+use super::core::AzblobCore;
+use super::core::ListBlobsOutput;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct AzblobLister {
+    core: Arc<AzblobCore>,
+
+    path: String,
+    delimiter: &'static str,
+    limit: Option<usize>,
+}
+
+impl AzblobLister {
+    pub fn new(core: Arc<AzblobCore>, path: String, recursive: bool, limit: Option<usize>) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+
+        Self {
+            core,
+            path,
+            delimiter,
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for AzblobLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .azblob_list_blobs(&self.path, &ctx.token, self.delimiter, self.limit)
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let output: ListBlobsOutput =
+            de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+
+        // Try our best to check whether this list is done.
+        //
+        // - Check `next_marker`
+        if let Some(next_marker) = output.next_marker.as_ref() {
+            ctx.done = next_marker.is_empty();
+        };
+        ctx.token = output.next_marker.clone().unwrap_or_default();
+
+        let prefixes = output.blobs.blob_prefix;
+
+        for prefix in prefixes {
+            let de = oio::Entry::new(
+                &build_rel_path(&self.core.root, &prefix.name),
+                Metadata::new(EntryMode::DIR),
+            );
+
+            ctx.entries.push_back(de)
+        }
+
+        for object in output.blobs.blob {
+            let mut path = build_rel_path(&self.core.root, &object.name);
+            if path.is_empty() {
+                path = "/".to_string();
+            }
+
+            let meta = Metadata::new(EntryMode::from_path(&path))
+                // Keep fit with ETag header.
+                .with_etag(format!("\"{}\"", object.properties.etag.as_str()))
+                .with_content_length(object.properties.content_length)
+                .with_content_md5(object.properties.content_md5)
+                .with_content_type(object.properties.content_type)
+                .with_last_modified(parse_datetime_from_rfc2822(
+                    object.properties.last_modified.as_str(),
+                )?);
+
+            let de = oio::Entry::with(path, meta);
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/azblob/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for azblob service.
+#[cfg(feature = "services-azblob")]
+pub(super) const DEFAULT_SCHEME: &str = "azblob";
+#[cfg(feature = "services-azblob")]
+pub(crate) mod core;
+#[cfg(feature = "services-azblob")]
+mod delete;
+#[cfg(feature = "services-azblob")]
+mod error;
+#[cfg(feature = "services-azblob")]
+mod lister;
+#[cfg(feature = "services-azblob")]
+pub(crate) mod writer;
+
+#[cfg(feature = "services-azblob")]
+mod backend;
+#[cfg(feature = "services-azblob")]
+pub use backend::AzblobBuilder as Azblob;
+
+mod config;
+pub use config::AzblobConfig;

data/core/src/services/azblob/writer.rs

@@ -0,0 +1,177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+use uuid::Uuid;
+
+use super::core::constants::X_MS_VERSION_ID;
+use super::core::AzblobCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+const X_MS_BLOB_TYPE: &str = "x-ms-blob-type";
+
+pub type AzblobWriters = TwoWays<oio::BlockWriter<AzblobWriter>, oio::AppendWriter<AzblobWriter>>;
+
+pub struct AzblobWriter {
+    core: Arc<AzblobCore>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl AzblobWriter {
+    pub fn new(core: Arc<AzblobCore>, op: OpWrite, path: String) -> Self {
+        AzblobWriter { core, op, path }
+    }
+
+    // skip extracting `content-md5` here, as it pertains to the content of the request rather than
+    // the content of the block itself for the `append` and `complete put block list` operations.
+    fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
+        let mut metadata = Metadata::default();
+
+        if let Some(last_modified) = parse_last_modified(headers)? {
+            metadata.set_last_modified(last_modified);
+        }
+        let etag = parse_etag(headers)?;
+        if let Some(etag) = etag {
+            metadata.set_etag(etag);
+        }
+        let version_id = parse_header_to_str(headers, X_MS_VERSION_ID)?;
+        if let Some(version_id) = version_id {
+            metadata.set_version(version_id);
+        }
+
+        Ok(metadata)
+    }
+}
+
+impl oio::AppendWrite for AzblobWriter {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self
+            .core
+            .azblob_get_blob_properties(&self.path, &OpStat::default())
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let headers = resp.headers();
+                let blob_type = headers.get(X_MS_BLOB_TYPE).and_then(|v| v.to_str().ok());
+                if blob_type != Some("AppendBlob") {
+                    return Err(Error::new(
+                        ErrorKind::ConditionNotMatch,
+                        "the blob is not an appendable blob.",
+                    ));
+                }
+
+                Ok(parse_content_length(headers)?.unwrap_or_default())
+            }
+            StatusCode::NOT_FOUND => {
+                let resp = self
+                    .core
+                    .azblob_init_appendable_blob(&self.path, &self.op)
+                    .await?;
+
+                let status = resp.status();
+                match status {
+                    StatusCode::CREATED => {
+                        // do nothing
+                    }
+                    _ => {
+                        return Err(parse_error(resp));
+                    }
+                }
+                Ok(0)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .azblob_append_blob(&self.path, offset, size, body)
+            .await?;
+
+        let meta = AzblobWriter::parse_metadata(resp.headers())?;
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+impl oio::BlockWrite for AzblobWriter {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .azblob_put_blob(&self.path, Some(size), &self.op, body)
+            .await?;
+
+        let status = resp.status();
+
+        let mut meta = AzblobWriter::parse_metadata(resp.headers())?;
+        let md5 = parse_content_md5(resp.headers())?;
+        if let Some(md5) = md5 {
+            meta.set_content_md5(md5);
+        }
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_block(&self, block_id: Uuid, size: u64, body: Buffer) -> Result<()> {
+        let resp = self
+            .core
+            .azblob_put_block(&self.path, block_id, Some(size), &self.op, body)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_block(&self, block_ids: Vec<Uuid>) -> Result<Metadata> {
+        let resp = self
+            .core
+            .azblob_complete_put_block_list(&self.path, block_ids, &self.op)
+            .await?;
+
+        let meta = AzblobWriter::parse_metadata(resp.headers())?;
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_block(&self, _block_ids: Vec<Uuid>) -> Result<()> {
+        // refer to https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list?tabs=microsoft-entra-id
+        // Any uncommitted blocks are garbage collected if there are no successful calls to Put Block or Put Block List on the blob within a week.
+        // If Put Blob is called on the blob, any uncommitted blocks are garbage collected.
+        Ok(())
+    }
+}