opendal 0.1.6.pre.rc.1

data/core/src/types/builder.rs

@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+
+use http::Uri;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// Builder is used to set up underlying services.
+///
+/// This trait allows the developer to define a builder struct that can:
+///
+/// - build a service via builder style API.
+/// - configure in-memory options like `http_client` or `customized_credential_load`.
+///
+/// Usually, users don't need to use or import this trait directly, they can use `Operator` API instead.
+///
+/// For example:
+///
+/// ```
+/// # use anyhow::Result;
+/// use opendal::services::Fs;
+/// use opendal::Operator;
+/// async fn test() -> Result<()> {
+///     // Create fs backend builder.
+///     let mut builder = Fs::default().root("/tmp");
+///
+///     // Build an `Operator` to start operating the storage.
+///     let op: Operator = Operator::new(builder)?.finish();
+///
+///     Ok(())
+/// }
+/// ```
+pub trait Builder: Default + 'static {
+    /// Associated configuration for this builder.
+    type Config: Configurator;
+
+    /// Consume the accessor builder to build a service.
+    fn build(self) -> Result<impl Access>;
+}
+
+/// Dummy implementation of builder
+impl Builder for () {
+    type Config = ();
+
+    fn build(self) -> Result<impl Access> {
+        Ok(())
+    }
+}
+
+/// Configurator is used to configure the underlying service.
+///
+/// This trait allows the developer to define a configuration struct that can:
+///
+/// - deserialize from an iterator like hashmap or vector.
+/// - convert into a service builder and finally build the underlying services.
+///
+/// Usually, users don't need to use or import this trait directly, they can use `Operator` API instead.
+///
+/// For example:
+///
+/// ```
+/// # use anyhow::Result;
+/// use std::collections::HashMap;
+///
+/// use opendal::services::MemoryConfig;
+/// use opendal::Operator;
+/// async fn test() -> Result<()> {
+///     let mut cfg = MemoryConfig::default();
+///     cfg.root = Some("/".to_string());
+///
+///     // Build an `Operator` to start operating the storage.
+///     let op: Operator = Operator::from_config(cfg)?.finish();
+///
+///     Ok(())
+/// }
+/// ```
+///
+/// Some service builder might contain in memory options like `http_client` . Users can call
+/// `into_builder` to convert the configuration into a builder instead.
+///
+/// ```
+/// # use anyhow::Result;
+/// use std::collections::HashMap;
+///
+/// use opendal::raw::HttpClient;
+/// use opendal::services::S3Config;
+/// use opendal::Configurator;
+/// use opendal::Operator;
+///
+/// async fn test() -> Result<()> {
+///     let mut cfg = S3Config::default();
+///     cfg.root = Some("/".to_string());
+///     cfg.bucket = "test".to_string();
+///
+///     let builder = cfg.into_builder();
+///     let builder = builder.http_client(HttpClient::new()?);
+///
+///     // Build an `Operator` to start operating the storage.
+///     let op: Operator = Operator::new(builder)?.finish();
+///
+///     Ok(())
+/// }
+/// ```
+pub trait Configurator: Serialize + DeserializeOwned + Debug + 'static {
+    /// Associated builder for this configuration.
+    type Builder: Builder;
+
+    /// Build configuration from a URI plus merged options.
+    fn from_uri(_uri: &Uri, _options: &HashMap<String, String>) -> Result<Self> {
+        Err(Error::new(ErrorKind::Unsupported, "uri is not supported"))
+    }
+
+    /// Deserialize from an iterator.
+    ///
+    /// This API is provided by opendal, developer should not implement it.
+    fn from_iter(iter: impl IntoIterator<Item = (String, String)>) -> Result<Self> {
+        let cfg = ConfigDeserializer::new(iter.into_iter().collect());
+
+        Self::deserialize(cfg).map_err(|err| {
+            Error::new(ErrorKind::ConfigInvalid, "failed to deserialize config").set_source(err)
+        })
+    }
+
+    /// Convert this configuration into a service builder.
+    fn into_builder(self) -> Self::Builder;
+}
+
+impl Configurator for () {
+    type Builder = ();
+
+    fn into_builder(self) -> Self::Builder {}
+}

data/core/src/types/capability.rs

@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+/// Capability defines the supported operations and their constraints for a storage Operator.
+///
+/// # Overview
+///
+/// This structure provides a comprehensive description of an Operator's capabilities,
+/// including:
+///
+/// - Basic operations support (read, write, delete, etc.)
+/// - Advanced operation variants (conditional operations, metadata handling)
+/// - Operational constraints (size limits, batch limitations)
+///
+/// # Capability Types
+///
+/// Every operator maintains two capability sets:
+///
+/// 1. [`OperatorInfo::native_capability`][crate::OperatorInfo::native_capability]:
+///    Represents operations natively supported by the storage backend.
+///
+/// 2. [`OperatorInfo::full_capability`][crate::OperatorInfo::full_capability]:
+///    Represents all available operations, including those implemented through
+///    alternative mechanisms.
+///
+/// # Implementation Details
+///
+/// Some operations might be available even when not natively supported by the
+/// backend. For example:
+///
+/// - Blocking operations are provided through the BlockingLayer
+///
+/// Developers should:
+/// - Use `full_capability` to determine available operations
+/// - Use `native_capability` to identify optimized operations
+///
+/// # Field Naming Conventions
+///
+/// Fields follow these naming patterns:
+///
+/// - Basic operations: Simple lowercase (e.g., `read`, `write`)
+/// - Compound operations: Underscore-separated (e.g., `presign_read`)
+/// - Variants: Capability description (e.g., `write_can_empty`)
+/// - Parameterized operations: With-style (e.g., `read_with_if_match`)
+/// - Limitations: Constraint description (e.g., `write_multi_max_size`)
+/// - Metadata Results: Returning metadata capabilities (e.g., `stat_has_content_length`)
+///
+/// All capability fields are public and can be accessed directly.
+#[derive(Copy, Clone, Default)]
+pub struct Capability {
+    /// Indicates if the operator supports metadata retrieval operations.
+    pub stat: bool,
+    /// Indicates if conditional stat operations using If-Match are supported.
+    pub stat_with_if_match: bool,
+    /// Indicates if conditional stat operations using If-None-Match are supported.
+    pub stat_with_if_none_match: bool,
+    /// Indicates if conditional stat operations using If-Modified-Since are supported.
+    pub stat_with_if_modified_since: bool,
+    /// Indicates if conditional stat operations using If-Unmodified-Since are supported.
+    pub stat_with_if_unmodified_since: bool,
+    /// Indicates if Cache-Control header override is supported during stat operations.
+    pub stat_with_override_cache_control: bool,
+    /// Indicates if Content-Disposition header override is supported during stat operations.
+    pub stat_with_override_content_disposition: bool,
+    /// Indicates if Content-Type header override is supported during stat operations.
+    pub stat_with_override_content_type: bool,
+    /// Indicates if versions stat operations are supported.
+    pub stat_with_version: bool,
+
+    /// Indicates if the operator supports read operations.
+    pub read: bool,
+    /// Indicates if conditional read operations using If-Match are supported.
+    pub read_with_if_match: bool,
+    /// Indicates if conditional read operations using If-None-Match are supported.
+    pub read_with_if_none_match: bool,
+    /// Indicates if conditional read operations using If-Modified-Since are supported.
+    pub read_with_if_modified_since: bool,
+    /// Indicates if conditional read operations using If-Unmodified-Since are supported.
+    pub read_with_if_unmodified_since: bool,
+    /// Indicates if Cache-Control header override is supported during read operations.
+    pub read_with_override_cache_control: bool,
+    /// Indicates if Content-Disposition header override is supported during read operations.
+    pub read_with_override_content_disposition: bool,
+    /// Indicates if Content-Type header override is supported during read operations.
+    pub read_with_override_content_type: bool,
+    /// Indicates if versions read operations are supported.
+    pub read_with_version: bool,
+
+    /// Indicates if the operator supports write operations.
+    pub write: bool,
+    /// Indicates if multiple write operations can be performed on the same object.
+    pub write_can_multi: bool,
+    /// Indicates if writing empty content is supported.
+    pub write_can_empty: bool,
+    /// Indicates if append operations are supported.
+    pub write_can_append: bool,
+    /// Indicates if Content-Type can be specified during write operations.
+    pub write_with_content_type: bool,
+    /// Indicates if Content-Disposition can be specified during write operations.
+    pub write_with_content_disposition: bool,
+    /// Indicates if Content-Encoding can be specified during write operations.
+    pub write_with_content_encoding: bool,
+    /// Indicates if Cache-Control can be specified during write operations.
+    pub write_with_cache_control: bool,
+    /// Indicates if conditional write operations using If-Match are supported.
+    pub write_with_if_match: bool,
+    /// Indicates if conditional write operations using If-None-Match are supported.
+    pub write_with_if_none_match: bool,
+    /// Indicates if write operations can be conditional on object non-existence.
+    pub write_with_if_not_exists: bool,
+    /// Indicates if custom user metadata can be attached during write operations.
+    pub write_with_user_metadata: bool,
+    /// Maximum size supported for multipart uploads.
+    /// For example, AWS S3 supports up to 5GiB per part in multipart uploads.
+    pub write_multi_max_size: Option<usize>,
+    /// Minimum size required for multipart uploads (except for the last part).
+    /// For example, AWS S3 requires at least 5MiB per part.
+    pub write_multi_min_size: Option<usize>,
+    /// Maximum total size supported for write operations.
+    /// For example, Cloudflare D1 has a 1MB total size limit.
+    pub write_total_max_size: Option<usize>,
+
+    /// Indicates if directory creation is supported.
+    pub create_dir: bool,
+
+    /// Indicates if delete operations are supported.
+    pub delete: bool,
+    /// Indicates if versions delete operations are supported.
+    pub delete_with_version: bool,
+    /// Maximum size supported for single delete operations.
+    pub delete_max_size: Option<usize>,
+
+    /// Indicates if copy operations are supported.
+    pub copy: bool,
+    /// Indicates if conditional copy operations with if-not-exists are supported.
+    pub copy_with_if_not_exists: bool,
+
+    /// Indicates if rename operations are supported.
+    pub rename: bool,
+
+    /// Indicates if list operations are supported.
+    pub list: bool,
+    /// Indicates if list operations support result limiting.
+    pub list_with_limit: bool,
+    /// Indicates if list operations support continuation from a specific point.
+    pub list_with_start_after: bool,
+    /// Indicates if recursive listing is supported.
+    pub list_with_recursive: bool,
+    /// Indicates if versions listing is supported.
+    #[deprecated(since = "0.51.1", note = "use with_versions instead")]
+    pub list_with_version: bool,
+    /// Indicates if listing with versions included is supported.
+    pub list_with_versions: bool,
+    /// Indicates if listing with deleted files included is supported.
+    pub list_with_deleted: bool,
+
+    /// Indicates if presigned URL generation is supported.
+    pub presign: bool,
+    /// Indicates if presigned URLs for read operations are supported.
+    pub presign_read: bool,
+    /// Indicates if presigned URLs for stat operations are supported.
+    pub presign_stat: bool,
+    /// Indicates if presigned URLs for write operations are supported.
+    pub presign_write: bool,
+    /// Indicates if presigned URLs for delete operations are supported.
+    pub presign_delete: bool,
+
+    /// Indicate if the operator supports shared access.
+    pub shared: bool,
+}
+
+impl Debug for Capability {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // NOTE: All services in opendal are readable.
+        if self.read {
+            f.write_str("Read")?;
+        }
+        if self.write {
+            f.write_str("| Write")?;
+        }
+        if self.list {
+            f.write_str("| List")?;
+        }
+        if self.presign {
+            f.write_str("| Presign")?;
+        }
+        if self.shared {
+            f.write_str("| Shared")?;
+        }
+
+        Ok(())
+    }
+}

data/core/src/types/context/mod.rs

@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod read;
+pub use read::*;
+
+mod write;
+pub use write::*;

data/core/src/types/context/read.rs

@@ -0,0 +1,231 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::ops::Bound;
+use std::ops::Range;
+use std::ops::RangeBounds;
+use std::sync::Arc;
+
+use crate::raw::*;
+use crate::*;
+
+/// ReadContext holds the immutable context for give read operation.
+pub struct ReadContext {
+    /// The accessor to the storage services.
+    acc: Accessor,
+    /// Path to the file.
+    path: String,
+    /// Arguments for the read operation.
+    args: OpRead,
+    /// Options for the reader.
+    options: OpReader,
+}
+
+impl ReadContext {
+    /// Create a new ReadContext.
+    #[inline]
+    pub fn new(acc: Accessor, path: String, args: OpRead, options: OpReader) -> Self {
+        Self {
+            acc,
+            path,
+            args,
+            options,
+        }
+    }
+
+    /// Get the accessor.
+    #[inline]
+    pub fn accessor(&self) -> &Accessor {
+        &self.acc
+    }
+
+    /// Get the path.
+    #[inline]
+    pub fn path(&self) -> &str {
+        &self.path
+    }
+
+    /// Get the arguments.
+    #[inline]
+    pub fn args(&self) -> &OpRead {
+        &self.args
+    }
+
+    /// Get the options.
+    #[inline]
+    pub fn options(&self) -> &OpReader {
+        &self.options
+    }
+
+    /// Parse the range bounds into a range.
+    pub(crate) async fn parse_into_range(
+        &self,
+        range: impl RangeBounds<u64>,
+    ) -> Result<Range<u64>> {
+        let start = match range.start_bound() {
+            Bound::Included(v) => *v,
+            Bound::Excluded(v) => v + 1,
+            Bound::Unbounded => 0,
+        };
+
+        let end = match range.end_bound() {
+            Bound::Included(v) => v + 1,
+            Bound::Excluded(v) => *v,
+            Bound::Unbounded => {
+                let mut op_stat = OpStat::new();
+
+                if let Some(v) = self.args().version() {
+                    op_stat = op_stat.with_version(v);
+                }
+
+                self.accessor()
+                    .stat(self.path(), op_stat)
+                    .await?
+                    .into_metadata()
+                    .content_length()
+            }
+        };
+
+        Ok(start..end)
+    }
+}
+
+/// ReadGenerator is used to generate new readers.
+///
+/// If chunk is None, ReaderGenerator will only return one reader.
+/// Otherwise, ReaderGenerator will return multiple readers, each with size
+/// of chunk.
+///
+/// It's design that we didn't implement the generator as a stream, because
+/// we don't expose the generator to the user. Instead, we use the async method
+/// directly to keep it simple and easy to understand.
+pub struct ReadGenerator {
+    ctx: Arc<ReadContext>,
+
+    offset: u64,
+    size: Option<u64>,
+}
+
+impl ReadGenerator {
+    /// Create a new ReadGenerator.
+    #[inline]
+    pub fn new(ctx: Arc<ReadContext>, offset: u64, size: Option<u64>) -> Self {
+        Self { ctx, offset, size }
+    }
+
+    /// Generate next range to read.
+    fn next_range(&mut self) -> Option<BytesRange> {
+        if self.size == Some(0) {
+            return None;
+        }
+
+        let next_offset = self.offset;
+        let next_size = match self.size {
+            // Given size is None, read all data.
+            None => {
+                // Update size to Some(0) to indicate that there is no more data to read.
+                self.size = Some(0);
+                None
+            }
+            Some(remaining) => {
+                // If chunk is set, read data in chunks.
+                let read_size = self
+                    .ctx
+                    .options
+                    .chunk()
+                    .map_or(remaining, |chunk| remaining.min(chunk as u64));
+                // Update (offset, size) before building future.
+                self.offset += read_size;
+                self.size = Some(remaining - read_size);
+                Some(read_size)
+            }
+        };
+
+        Some(BytesRange::new(next_offset, next_size))
+    }
+
+    /// Generate next reader.
+    pub async fn next_reader(&mut self) -> Result<Option<oio::Reader>> {
+        let Some(range) = self.next_range() else {
+            return Ok(None);
+        };
+
+        let args = self.ctx.args.clone().with_range(range);
+        let (_, r) = self.ctx.acc.read(&self.ctx.path, args).await?;
+        Ok(Some(r))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use bytes::Bytes;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_next_reader() -> Result<()> {
+        let op = Operator::via_iter(Scheme::Memory, [])?;
+        op.write(
+            "test",
+            Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+        )
+        .await?;
+
+        let acc = op.into_inner();
+        let ctx = Arc::new(ReadContext::new(
+            acc,
+            "test".to_string(),
+            OpRead::new(),
+            OpReader::new().with_chunk(3),
+        ));
+        let mut generator = ReadGenerator::new(ctx, 0, Some(10));
+        let mut readers = vec![];
+        while let Some(r) = generator.next_reader().await? {
+            readers.push(r);
+        }
+
+        pretty_assertions::assert_eq!(readers.len(), 4);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_next_reader_without_size() -> Result<()> {
+        let op = Operator::via_iter(Scheme::Memory, [])?;
+        op.write(
+            "test",
+            Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+        )
+        .await?;
+
+        let acc = op.into_inner();
+        let ctx = Arc::new(ReadContext::new(
+            acc,
+            "test".to_string(),
+            OpRead::new(),
+            OpReader::new().with_chunk(3),
+        ));
+        let mut generator = ReadGenerator::new(ctx, 0, None);
+        let mut readers = vec![];
+        while let Some(r) = generator.next_reader().await? {
+            readers.push(r);
+        }
+
+        pretty_assertions::assert_eq!(readers.len(), 1);
+        Ok(())
+    }
+}