opendal 0.1.9 → 0.1.11

data/core/services/s3/src/deleter.rs

@@ -45,6 +45,8 @@ impl oio::BatchDelete for S3Deleter {
             return Ok(());
         }
 
+        let error_ctx = ErrorContext::new(ServiceOperation("DeleteObject"))
+            .with_caller_condition(args.is_conditional());
         let resp = self.core.s3_delete_object(&self.ctx, &path, &args).await?;
 
         let status = resp.status();
@@ -54,8 +56,12 @@ impl oio::BatchDelete for S3Deleter {
             // Allow 404 when deleting a non-existing object
             // This is not a standard behavior, only some s3 alike service like GCS XML API do this.
             // ref: <https://cloud.google.com/storage/docs/xml-api/delete-object>
+            StatusCode::NOT_FOUND if args.if_match().is_some() => Err(Error::new(
+                ErrorKind::ConditionNotMatch,
+                "delete precondition requires a live target",
+            )),
             StatusCode::NOT_FOUND => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(error_ctx, resp)),
         }
     }
 
@@ -64,7 +70,10 @@ impl oio::BatchDelete for S3Deleter {
 
         let status = resp.status();
         if status != StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("DeleteObjects")),
+                resp,
+            ));
         }
 
         let bs = resp.into_body();
@@ -88,9 +97,12 @@ impl oio::BatchDelete for S3Deleter {
                 .position(|e| e.key == abs_path && e.version_id.as_deref() == op.version())
             {
                 let error = errors.swap_remove(idx);
-                batched_result
-                    .failed
-                    .push((path, op, parse_delete_objects_result_error(error)));
+                let conditional = op.if_match().is_some();
+                batched_result.failed.push((
+                    path,
+                    op,
+                    parse_delete_objects_result_error(error, conditional),
+                ));
             } else {
                 batched_result.succeeded.push((path, op));
             }
@@ -100,9 +112,16 @@ impl oio::BatchDelete for S3Deleter {
     }
 }
 
-fn parse_delete_objects_result_error(err: DeleteObjectsResultError) -> Error {
+fn parse_delete_objects_result_error(err: DeleteObjectsResultError, conditional: bool) -> Error {
+    let error_ctx =
+        ErrorContext::new(ServiceOperation("DeleteObjects")).with_caller_condition(conditional);
     let (kind, retryable) =
-        parse_s3_error_code(err.code.as_str()).unwrap_or((ErrorKind::Unexpected, false));
+        parse_s3_error_code(error_ctx, err.code.as_str()).unwrap_or((ErrorKind::Unexpected, false));
+    let kind = if conditional && kind == ErrorKind::NotFound {
+        ErrorKind::ConditionNotMatch
+    } else {
+        kind
+    };
     let mut err: Error = Error::new(kind, format!("{err:?}"));
     if retryable {
         err = err.set_temporary();

data/core/services/s3/src/docs.md

@@ -9,18 +9,75 @@ Depending on its configuration and the backing system, this service can expose:
 - [x] delete
 - [x] list
 - [x] copy
+- [x] restore
 - [ ] rename
 - [x] presign
 
 Inspect the effective capability set with [`opendal_core::Operator::info`] and
 [`opendal_core::OperatorInfo::capability`] after building an operator.
 
+## Object restoration
+
+S3 restoration requires bucket versioning. [`opendal_core::Operator::restore`]
+removes the current delete marker. Calling it on an already-live object succeeds.
+It does not scan or remove older marker versions. If no live object or current
+delete marker exists, it returns [`opendal_core::ErrorKind::NotFound`].
+
+Use [`opendal_core::Operator::restore_with`] to promote a specific version:
+
+```rust,no_run
+# use opendal_core::Operator;
+# use opendal_core::Result;
+# async fn restore(op: Operator, version: &str) -> Result<()> {
+op.restore_with("path/to/file")
+    .version(version)
+    .await?;
+# Ok(())
+# }
+```
+
+Add `.if_not_exists(true)` when a selected version must not overwrite an
+object recreated by another writer. This condition requires an explicit
+version.
+
 ## Configuration
 
 Use [`crate::S3Config`] for serializable configuration and this builder's
 methods for direct construction. The field and method documentation defines
 accepted values, defaults, and environment interaction.
 
+## S3 Express One Zone
+
+OpenDAL recognizes valid AWS directory bucket names on AWS endpoints and uses
+the bucket's zonal endpoint automatically. It creates and refreshes `ReadWrite`
+S3 Express sessions through the configured AWS credential provider. No extra
+configuration is required:
+
+```rust,no_run
+use opendal_core::Operator;
+use opendal_core::Result;
+use opendal_service_s3::S3;
+
+fn build_operator() -> Result<Operator> {
+    Operator::new(
+        S3::default()
+            .bucket("example--usw2-az1--x-s3")
+            .region("us-west-2"),
+    )
+}
+```
+
+This automatic behavior applies only to AWS endpoints. S3-compatible services
+retain the existing IAM authentication behavior.
+
+An explicit AWS endpoint must match the bucket's derived Zone, Region, and
+partition. OpenDAL rejects unsupported endpoint variants, including dual-stack
+endpoints, with `ConfigInvalid` instead of falling back to general S3 behavior.
+
+OpenDAL uses the source IAM credentials for `CopyObject`, `UploadPartCopy`, and
+presigned requests because these operations do not accept S3 Express session
+credentials.
+
 ## Temporary security credentials
 
 OpenDAL now provides support for S3 temporary security credentials in IAM.

data/core/services/s3/src/lister.rs

@@ -22,9 +22,8 @@ use quick_xml::de;
 
 use crate::core::parse_error;
 use crate::core::*;
-use opendal_core::EntryMode;
 use opendal_core::Error;
-use opendal_core::Metadata;
+use opendal_core::MetadataBuilder;
 use opendal_core::OperationContext;
 use opendal_core::Result;
 use opendal_core::raw::oio::PageContext;
@@ -90,7 +89,10 @@ impl oio::PageList for S3ListerV1 {
             .await?;
 
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListObjects")),
+                resp,
+            ));
         }
         let bs = resp.into_body();
 
@@ -130,7 +132,7 @@ impl oio::PageList for S3ListerV1 {
         for prefix in output.common_prefixes {
             let de = oio::Entry::new(
                 &build_rel_path(&self.core.root, &prefix.prefix),
-                Metadata::new(EntryMode::DIR),
+                MetadataBuilder::dir().build(),
             );
 
             ctx.entries.push_back(de);
@@ -142,18 +144,20 @@ impl oio::PageList for S3ListerV1 {
                 path = "/".to_string();
             }
 
-            let mut meta = Metadata::new(EntryMode::from_path(&path));
-            meta.set_is_current(true);
+            let mut meta = if path.ends_with('/') {
+                MetadataBuilder::dir()
+            } else {
+                MetadataBuilder::file(object.size)
+            };
+            meta.is_current(Some(true));
             if let Some(etag) = &object.etag {
-                meta.set_etag(etag);
+                meta.etag(etag);
             }
-            meta.set_content_length(object.size);
-
             // object.last_modified provides more precise time that contains
             // nanosecond, let's trim them.
-            meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
+            meta.last_modified(object.last_modified.parse::<Timestamp>()?);
 
-            let de = oio::Entry::with(path, meta);
+            let de = oio::Entry::with(path, meta.build());
             ctx.entries.push_back(de);
         }
 
@@ -212,7 +216,10 @@ impl oio::PageList for S3ListerV2 {
             .await?;
 
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListObjectsV2")),
+                resp,
+            ));
         }
         let bs = resp.into_body();
 
@@ -242,7 +249,7 @@ impl oio::PageList for S3ListerV2 {
         for prefix in output.common_prefixes {
             let de = oio::Entry::new(
                 &build_rel_path(&self.core.root, &prefix.prefix),
-                Metadata::new(EntryMode::DIR),
+                MetadataBuilder::dir().build(),
             );
 
             ctx.entries.push_back(de);
@@ -254,18 +261,20 @@ impl oio::PageList for S3ListerV2 {
                 path = "/".to_string();
             }
 
-            let mut meta = Metadata::new(EntryMode::from_path(&path));
-            meta.set_is_current(true);
+            let mut meta = if path.ends_with('/') {
+                MetadataBuilder::dir()
+            } else {
+                MetadataBuilder::file(object.size)
+            };
+            meta.is_current(Some(true));
             if let Some(etag) = &object.etag {
-                meta.set_etag(etag);
+                meta.etag(etag);
             }
-            meta.set_content_length(object.size);
-
             // object.last_modified provides more precise time that contains
             // nanosecond, let's trim them.
-            meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
+            meta.last_modified(object.last_modified.parse::<Timestamp>()?);
 
-            let de = oio::Entry::with(path, meta);
+            let de = oio::Entry::with(path, meta.build());
             ctx.entries.push_back(de);
         }
 
@@ -326,7 +335,10 @@ impl oio::PageList for S3ObjectVersionsLister {
             )
             .await?;
         if resp.status() != http::StatusCode::OK {
-            return Err(parse_error(resp));
+            return Err(parse_error(
+                ErrorContext::new(ServiceOperation("ListObjectVersions")),
+                resp,
+            ));
         }
 
         let body = resp.into_body();
@@ -353,7 +365,7 @@ impl oio::PageList for S3ObjectVersionsLister {
         for prefix in output.common_prefixes {
             let de = oio::Entry::new(
                 &build_rel_path(&self.core.root, &prefix.prefix),
-                Metadata::new(EntryMode::DIR),
+                MetadataBuilder::dir().build(),
             );
             ctx.entries.push_back(de);
         }
@@ -372,17 +384,20 @@ impl oio::PageList for S3ObjectVersionsLister {
                 path = "/".to_owned();
             }
 
-            let mut meta = Metadata::new(EntryMode::from_path(&path));
-            meta.set_version(&version_object.version_id);
-            meta.set_is_current(version_object.is_latest);
-            meta.set_content_length(version_object.size);
-            meta.set_last_modified(version_object.last_modified.parse::<Timestamp>()?);
+            let mut meta = if path.ends_with('/') {
+                MetadataBuilder::dir()
+            } else {
+                MetadataBuilder::file(version_object.size)
+            };
+            meta.version(&version_object.version_id)
+                .is_current(Some(version_object.is_latest))
+                .last_modified(version_object.last_modified.parse::<Timestamp>()?);
 
             if let Some(etag) = version_object.etag {
-                meta.set_etag(&etag);
+                meta.etag(&etag);
             }
 
-            let entry = oio::Entry::new(&path, meta);
+            let entry = oio::Entry::new(&path, meta.build());
             ctx.entries.push_back(entry);
         }
 
@@ -393,13 +408,17 @@ impl oio::PageList for S3ObjectVersionsLister {
                     path = "/".to_owned();
                 }
 
-                let mut meta = Metadata::new(EntryMode::from_path(&path));
-                meta.set_version(&delete_marker.version_id);
-                meta.set_is_deleted(true);
-                meta.set_is_current(delete_marker.is_latest);
-                meta.set_last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
-
-                let entry = oio::Entry::new(&path, meta);
+                let mut meta = if path.ends_with('/') {
+                    MetadataBuilder::dir()
+                } else {
+                    MetadataBuilder::file(0)
+                };
+                meta.version(&delete_marker.version_id)
+                    .is_deleted(true)
+                    .is_current(Some(delete_marker.is_latest))
+                    .last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
+
+                let entry = oio::Entry::new(&path, meta.build());
                 ctx.entries.push_back(entry);
             }
         }

data/core/services/s3/src/reader.rs

@@ -16,7 +16,11 @@
 // under the License.
 
 use crate::backend::*;
+use crate::core::ErrorContext;
+use crate::core::constants::X_AMZ_META_PREFIX;
+use crate::core::constants::X_AMZ_VERSION_ID;
 use crate::core::parse_error;
+use http::HeaderMap;
 use http::Response;
 use http::StatusCode;
 use opendal_core::raw::*;
@@ -48,6 +52,8 @@ impl oio::StreamRead for S3Reader {
         let backend = &self.backend;
         let path = self.path.as_str();
         let args = self.args.clone();
+        let error_ctx = ErrorContext::new(ServiceOperation("GetObject"))
+            .with_caller_condition(args.is_conditional());
         let resp = backend
             .core
             .s3_get_object(&self.ctx, path, range, &args)
@@ -56,16 +62,30 @@ impl oio::StreamRead for S3Reader {
         let status = resp.status();
         let (rp, stream) = match status {
             StatusCode::OK | StatusCode::PARTIAL_CONTENT => (
-                RpRead::new(parse_into_metadata(path, resp.headers())?),
+                RpRead::new(parse_into_s3_metadata(path, resp.headers())?),
                 resp.into_body(),
             ),
             _ => {
                 let (part, mut body) = resp.into_parts();
                 let buf = body.to_buffer().await?;
-                return Err(parse_error(Response::from_parts(part, buf)));
+                return Err(parse_error(error_ctx, Response::from_parts(part, buf)));
             }
         };
 
         Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
     }
 }
+
+pub(super) fn parse_into_s3_metadata(path: &str, headers: &HeaderMap) -> Result<Metadata> {
+    let mut meta = parse_into_metadata(path, headers)?.into_builder();
+
+    let user_meta = parse_prefixed_headers(headers, X_AMZ_META_PREFIX);
+    if !user_meta.is_empty() {
+        meta.user_metadata(user_meta);
+    }
+
+    if let Some(v) = parse_header_to_str(headers, X_AMZ_VERSION_ID)? {
+        meta.version(v);
+    }
+    Ok(meta.build())
+}

data/core/services/s3/src/writer.rs

@@ -20,10 +20,9 @@ use std::sync::Arc;
 use bytes::Buf;
 use constants::X_AMZ_OBJECT_SIZE;
 use constants::X_AMZ_VERSION_ID;
+use http::Response;
 use http::StatusCode;
 
-use crate::core::S3Error;
-use crate::core::from_s3_error;
 use crate::core::parse_error;
 use crate::core::*;
 use opendal_core::raw::*;
@@ -50,19 +49,28 @@ impl S3Writer {
     }
 
     fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> {
-        let mut meta = Metadata::new(EntryMode::from_path(path));
+        let mut meta = if path.ends_with('/') {
+            MetadataBuilder::dir()
+        } else {
+            MetadataBuilder::unknown()
+        };
         if let Some(etag) = parse_etag(headers)? {
-            meta.set_etag(etag);
+            meta.etag(etag);
         }
         if let Some(version) = parse_header_to_str(headers, X_AMZ_VERSION_ID)? {
-            meta.set_version(version);
+            meta.version(version);
         }
-        if let Some(value) =
-            parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
+        if !path.ends_with('/')
+            && let Some(value) =
+                parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
         {
-            meta.set_content_length(value);
+            meta.set_file(value);
         }
-        Ok(meta)
+        Ok(meta.build())
+    }
+
+    fn error_context(&self, service_operation: ServiceOperation) -> ErrorContext {
+        ErrorContext::new(service_operation).with_caller_condition(self.op.is_conditional())
     }
 }
 
@@ -72,7 +80,10 @@ impl oio::MultipartWrite for S3Writer {
             .core
             .s3_put_object_request(&self.path, Some(size), &self.op, body)?;
 
-        let resp = self.core.send(&self.ctx, req).await?;
+        let resp = self
+            .core
+            .send(&self.ctx, req, self.core.signers.default())
+            .await?;
 
         let status = resp.status();
 
@@ -80,7 +91,17 @@ impl oio::MultipartWrite for S3Writer {
 
         match status {
             StatusCode::CREATED | StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => {
+                let err = parse_error(self.error_context(ServiceOperation("PutObject")), resp);
+                if self.op.if_match().is_some() && err.kind() == ErrorKind::NotFound {
+                    Err(Error::new(
+                        ErrorKind::ConditionNotMatch,
+                        "write precondition requires a live target",
+                    ))
+                } else {
+                    Err(err)
+                }
+            }
         }
     }
 
@@ -101,7 +122,10 @@ impl oio::MultipartWrite for S3Writer {
 
                 Ok(result.upload_id)
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("CreateMultipartUpload")),
+                resp,
+            )),
         }
     }
 
@@ -126,7 +150,10 @@ impl oio::MultipartWrite for S3Writer {
             checksum.clone(),
         )?;
 
-        let resp = self.core.send(&self.ctx, req).await?;
+        let resp = self
+            .core
+            .send(&self.ctx, req, self.core.signers.default())
+            .await?;
 
         let status = resp.status();
 
@@ -148,7 +175,70 @@ impl oio::MultipartWrite for S3Writer {
                     size: None,
                 })
             }
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("UploadPart")),
+                resp,
+            )),
+        }
+    }
+
+    async fn copy_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        path: &str,
+        args: OpRead,
+        range: BytesRange,
+    ) -> Result<oio::MultipartPart> {
+        let size = range
+            .size()
+            .expect("multipart writer copy range must be bounded");
+        let part_number = part_number + 1;
+        let error_context = ErrorContext::new(ServiceOperation("UploadPartCopy"))
+            .with_caller_condition(args.is_conditional());
+        let req = self
+            .core
+            .s3_upload_part_copy_request(S3UploadPartCopyRequest {
+                from: path,
+                to: &self.path,
+                source_version: args.version(),
+                if_match: args.if_match(),
+                if_none_match: args.if_none_match(),
+                if_modified_since: args.if_modified_since(),
+                if_unmodified_since: args.if_unmodified_since(),
+                upload_id,
+                part_number,
+                range,
+                operation: Operation::Write,
+            })?;
+
+        let resp = self
+            .core
+            .send(&self.ctx, req, self.core.signers.iam())
+            .await?;
+        match resp.status() {
+            StatusCode::OK => {
+                let (parts, body) = resp.into_parts();
+                let bs = body.to_bytes();
+                let result: CopyObjectResult =
+                    quick_xml::de::from_reader(bs.as_ref()).map_err(new_xml_deserialize_error)?;
+
+                // S3 may return 200 OK with an <Error> body for UploadPartCopy.
+                if result.etag.is_empty() {
+                    return Err(parse_error(
+                        error_context,
+                        Response::from_parts(parts, Buffer::from(bs)),
+                    ));
+                }
+
+                Ok(oio::MultipartPart {
+                    part_number,
+                    etag: result.etag,
+                    checksum: None,
+                    size: Some(size),
+                })
+            }
+            _ => Err(parse_error(error_context, resp)),
         }
     }
 
@@ -187,32 +277,50 @@ impl oio::MultipartWrite for S3Writer {
 
         let status = resp.status();
 
-        let mut meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
+        let meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
 
         match status {
             StatusCode::OK => {
                 // still check if there is any error because S3 might return error for status code 200
                 // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_Example_4
                 let (parts, body) = resp.into_parts();
+                let bs = body.to_bytes();
 
                 let ret: CompleteMultipartUploadResult =
-                    quick_xml::de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
+                    quick_xml::de::from_reader(bs.as_ref()).map_err(new_xml_deserialize_error)?;
                 if !ret.code.is_empty() {
-                    return Err(from_s3_error(
-                        S3Error {
-                            code: ret.code,
-                            message: ret.message,
-                            resource: "".to_string(),
-                            request_id: ret.request_id,
-                        },
-                        parts,
-                    ));
+                    let err = parse_error(
+                        self.error_context(ServiceOperation("CompleteMultipartUpload")),
+                        Response::from_parts(parts, Buffer::from(bs)),
+                    );
+                    return if self.op.if_match().is_some() && err.kind() == ErrorKind::NotFound {
+                        Err(Error::new(
+                            ErrorKind::ConditionNotMatch,
+                            "write precondition requires a live target",
+                        ))
+                    } else {
+                        Err(err)
+                    };
                 }
-                meta.set_etag(&ret.etag);
+                let mut meta = meta.into_builder();
+                meta.etag(&ret.etag);
 
-                Ok(meta)
+                Ok(meta.build())
+            }
+            _ => {
+                let err = parse_error(
+                    self.error_context(ServiceOperation("CompleteMultipartUpload")),
+                    resp,
+                );
+                if self.op.if_match().is_some() && err.kind() == ErrorKind::NotFound {
+                    Err(Error::new(
+                        ErrorKind::ConditionNotMatch,
+                        "write precondition requires a live target",
+                    ))
+                } else {
+                    Err(err)
+                }
             }
-            _ => Err(parse_error(resp)),
         }
     }
 
@@ -224,7 +332,10 @@ impl oio::MultipartWrite for S3Writer {
         match resp.status() {
             // s3 returns code 204 if abort succeeds.
             StatusCode::NO_CONTENT => Ok(()),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("AbortMultipartUpload")),
+                resp,
+            )),
         }
     }
 }
@@ -241,16 +352,27 @@ impl oio::AppendWrite for S3Writer {
         match status {
             StatusCode::OK => Ok(parse_content_length(resp.headers())?.unwrap_or_default()),
             StatusCode::NOT_FOUND => Ok(0),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(
+                ErrorContext::new(ServiceOperation("HeadObject")),
+                resp,
+            )),
         }
     }
 
     async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let error_ctx = if offset == 0 {
+            self.error_context(ServiceOperation("PutObject"))
+        } else {
+            ErrorContext::new(ServiceOperation("PutObject"))
+        };
         let req = self
             .core
             .s3_append_object_request(&self.path, offset, size, &self.op, body)?;
 
-        let resp = self.core.send(&self.ctx, req).await?;
+        let resp = self
+            .core
+            .send(&self.ctx, req, self.core.signers.default())
+            .await?;
 
         let status = resp.status();
 
@@ -258,7 +380,7 @@ impl oio::AppendWrite for S3Writer {
 
         match status {
             StatusCode::CREATED | StatusCode::OK => Ok(meta),
-            _ => Err(parse_error(resp)),
+            _ => Err(parse_error(error_ctx, resp)),
         }
     }
 }

data/core/services/seafile/Cargo.toml

@@ -32,12 +32,12 @@ version = { workspace = true }
 all-features = true
 
 [dependencies]
-opendal-core = { path = "../../core", version = "0.58.1", default-features = false }
+opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
 
+asyncband = { workspace = true }
 bytes = { workspace = true }
 http = { workspace = true }
 log = { workspace = true }
-mea = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }