opendal 0.1.9 → 0.1.11

data/core/core/src/docs/internals/layer.rs

@@ -15,80 +15,64 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! The internal implementation details of [`Layer`].
+//! Implementing a layer.
 //!
-//! OpenDAL has one layer composition surface:
+//! A [`Layer`] intercepts an operator's composed service, operation context,
+//! or both. Use a layer for behavior that applies across services, such as
+//! retry, timeout, tracing, metrics, or runtime resource replacement.
 //!
-//! - [`Layer`] receives an already erased [`ServiceDyn`] stack and can wrap the
-//!   service or its [`OperationContext`].
-//! - Layer wrappers still implement typed [`Service`], so their own reader,
-//!   writer, lister, deleter, and copier bodies stay concrete.
-//! - The composition boundary is [`Servicer`]. Boxing operation
-//!   bodies remains centralized in the blanket [`ServiceDyn`] implementation
-//!   for typed [`Service`] values.
+//! # Two composition hooks
 //!
-//! [`Layer`] itself is the runtime hook surface. Every hook returns `inner` by
-//! default, so a layer only implements the plane it needs:
+//! [`Layer`] exposes two hooks:
 //!
-//! ```ignore
-//! pub trait Layer: Send + Sync + Debug + Unpin + 'static {
-//!     fn apply_service(&self, srv: Servicer) -> Servicer;
-//!     fn apply_context(&self, srv: Servicer, inner: OperationContext) -> OperationContext;
-//! }
+//! ```text
+//! fn apply_service(&self, service: Servicer) -> Servicer;
+//! fn apply_context(
+//!     &self,
+//!     service: Servicer,
+//!     context: OperationContext,
+//! ) -> OperationContext;
 //! ```
 //!
-//! [`Operator`] replays the same ordered layer list over the base service and
-//! base [`OperationContext`]. It first composes the operation service stack.
-//! Then context hooks receive the final [`Servicer`] and compose runtime
-//! resources such as HTTP transport and executor. The composed context is passed
-//! to service operations.
+//! `apply_service` wraps storage operations. `apply_context` wraps or replaces
+//! runtime resources such as the HTTP transport and executor. Each hook
+//! returns its input unchanged by default, so a layer implements only the
+//! plane it owns.
 //!
-//! An operation layer normally has two parts:
+//! The operator first applies every service hook in insertion order. It then
+//! applies every context hook in the same order, passing the final service
+//! stack to each context hook. Adding a layer or replacing the base context
+//! replays the complete layer list, producing a service stack and context from
+//! the same ordering.
 //!
-//! - `XxxLayer` implements [`Layer`] and returns a typed service wrapper from
-//!   `apply_service`.
-//! - `XxxService` stores the inner [`Servicer`] and implements [`Service`].
+//! # Operation layers
 //!
-//! ```ignore
-//! pub struct XxxLayer;
+//! An operation layer normally contains:
 //!
-//! impl Layer for XxxLayer {
-//!     fn apply_service(&self, inner: Servicer) -> Servicer {
-//!         Arc::new(XxxService { inner })
-//!     }
-//! }
-//! ```
-//!
-//! Most operation layers only override the operations they need and forward the
-//! rest to `inner`. This works because [`Servicer`] implements [`Service`] by
-//! forwarding calls to [`ServiceDyn`], while the wrapper keeps concrete
-//! operation body types until it is returned as a [`Servicer`].
+//! - An `XxxLayer` that implements [`Layer::apply_service`].
+//! - An `XxxService` that stores the inner [`Servicer`] and implements
+//!   [`Service`].
 //!
-//! Resource-only layers can implement only `apply_context`. If they do not need
-//! the final service stack, they should name that parameter `_srv`.
+//! The wrapper overrides only the operations it owns and forwards the rest to
+//! the inner service. It must also return capabilities that describe the
+//! behavior of the wrapped stack. The wrapper keeps its own operation body
+//! types concrete until OpenDAL erases it back into a [`Servicer`].
 //!
-//! ```ignore
-//! pub struct TransportLayer {
-//!     transport: HttpTransporter,
-//! }
+//! # Resource layers
 //!
-//! impl Layer for TransportLayer {
-//!     fn apply_context(&self, _srv: Servicer, inner: OperationContext) -> OperationContext {
-//!         inner.with_http_transport(self.transport.clone())
-//!     }
-//! }
-//! ```
+//! A resource-only layer implements [`Layer::apply_context`]. It should
+//! preserve the previous resource when lower layers must remain effective. A
+//! layer that wraps an HTTP transport or executor must decide explicitly
+//! whether requests continue through the previous value or replace it
+//! entirely.
 //!
-//! Layers that need consistent policy across planes can implement both hooks,
-//! for example operation and I/O timeout handling. Resource wrappers that
-//! replace HTTP transport or executor must forward to the previous value when
-//! they want lower layers to remain effective.
+//! Layers that coordinate policy across operation and I/O phases can implement
+//! both hooks. Shared mutable state requires interior mutability and must
+//! remain `Send` and `Sync`, because cloned operators can run operations
+//! concurrently.
 //!
 //! [`Layer`]: crate::raw::Layer
+//! [`Layer::apply_service`]: crate::raw::Layer::apply_service
+//! [`Layer::apply_context`]: crate::raw::Layer::apply_context
 //! [`Service`]: crate::raw::Service
-//! [`ServiceDyn`]: crate::raw::ServiceDyn
 //! [`Servicer`]: crate::raw::Servicer
-//! [`HttpTransporter`]: crate::HttpTransporter
-//! [`Executor`]: crate::Executor
-//! [`OperationContext`]: crate::raw::OperationContext
-//! [`Operator`]: crate::Operator

data/core/core/src/docs/internals/mod.rs

@@ -17,37 +17,35 @@
 
 //! The internal implementation details of OpenDAL.
 //!
-//! OpenDAL provides unified abstraction via two-level API sets:
+//! This guide explains the extension boundaries used to implement OpenDAL
+//! storage services and layers. Application code should use the public
+//! [`Operator`] API; the raw APIs described here can change between minor
+//! releases.
 //!
-//! - Public API like [`Operator`] provides user-level API.
-//! - Raw API like [`Service`], [`Layer`] provides developer-level API.
-//!
-//! OpenDAL tries its best to keep the public API stable. But raw APIs
-//! may change between minor releases from time to time. So most users
-//! should only use the public API. Developers use raw APIs to implement
-//! storage services or their own [`Layer`].
-//!
-//! In this section, we will talk about the following components:
-//!
-//! - [`Service`][accessor]: to connect underlying storage services.
-//! - [`Layer`][layer]: middleware/interceptor between storage services.
-//!
-//! The relation between [`Service`], [`Layer`], [`Servicer`], [`OperationContext`],
-//! and [`Operator`] looks like the following:
+//! OpenDAL keeps service implementations typed until it composes them into an
+//! operator:
 //!
 //! ```text
-//! ┌──────────────┐     ┌──────────────────────────────┐     ┌──────────────┐
-//! │ Service      │     │ Servicer                     │     │ Operator     │
-//! │ builders     ├────►│ + Layer::apply_* replay      ├────►│ srv + ctx    │
-//! └──────────────┘     │ + OperationContext resources │     └──────────────┘
-//!                      └──────────────────────────────┘
+//! Builder -> Service -> ServiceDyn -> Servicer -> Layer stack -> Operator
+//!              ^                          |
+//!              +------ type erasure ------+
 //! ```
 //!
-//! [`Builder`]: crate::Builder
+//! - A [`Service`] implements storage operations and reports the capabilities
+//!   it provides.
+//! - [`ServiceDyn`] is the object-safe boundary that erases a typed service
+//!   and its operation bodies.
+//! - A [`Servicer`] is the shared, type-erased service handle stored by
+//!   operators and layers.
+//! - A [`Layer`] wraps the service stack, the operation context, or both.
+//!
+//! Continue with [implementing a service][accessor] or
+//! [implementing a layer][layer].
+//!
 //! [`Operator`]: crate::Operator
 //! [`Service`]: crate::raw::Service
+//! [`ServiceDyn`]: crate::raw::ServiceDyn
 //! [`Servicer`]: crate::raw::Servicer
-//! [`OperationContext`]: crate::raw::OperationContext
 //! [`Layer`]: crate::raw::Layer
 
 pub mod accessor;

data/core/core/src/docs/mod.rs

@@ -15,29 +15,33 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! This module holds documentation for OpenDAL.
+//! OpenDAL contracts, Rust core guides, and historical project documents.
 //!
-//! It's highly recommended that you start by reading [`concepts`] first.
+//! Start with [`concepts`] for the Rust core API model. [`specs`] defines the
+//! current portable contracts implemented by OpenDAL. Service and layer authors
+//! can continue with [`internals`], while [`performance`] covers application
+//! tuning. Accepted RFCs in [`rfcs`] preserve the decisions and context that
+//! led to current designs.
 
 #![allow(rustdoc::bare_urls)]
 
-pub mod comparisons;
-
 pub mod concepts;
-
 pub mod internals;
-
 pub mod performance;
+pub mod specs;
 
 /// Changes log for all OpenDAL released versions.
 #[doc = include_str!("../../../CHANGELOG.md")]
 #[cfg(not(doctest))]
+#[allow(rustdoc::broken_intra_doc_links, rustdoc::invalid_rust_codeblocks)]
 pub mod changelog {}
 
 #[cfg(not(doctest))]
+#[allow(rustdoc::broken_intra_doc_links, rustdoc::invalid_rust_codeblocks)]
 pub mod rfcs;
 
 /// Upgrade and migrate procedures while OpenDAL meets breaking changes.
 #[doc = include_str!("upgrade.md")]
 #[cfg(not(doctest))]
+#[allow(rustdoc::broken_intra_doc_links, rustdoc::invalid_rust_codeblocks)]
 pub mod upgrade {}

data/core/core/src/docs/performance/concurrent_write.md

@@ -1,101 +1,138 @@
-# Concurrent Write
+# Concurrent Writes
 
-OpenDAL writes data sequentially by default.
+OpenDAL writes parts sequentially by default. This keeps memory and request
+pressure low, but one in-flight request often cannot fill the available
+bandwidth on a high-latency path.
 
 ```rust
-# use opendal::Operator;
-# async fn test() {
-let w = op.writer("test.txt").await?;
-w.write(data1).await?;
-w.write(data2).await?;
-w.close().await?;
+# use opendal_core::{Buffer, Operator, Result};
+# async fn write_sequentially(
+#     op: &Operator,
+#     first: Buffer,
+#     second: Buffer,
+# ) -> Result<()> {
+let mut writer = op.writer("large-object.bin").await?;
+writer.write(first).await?;
+writer.write(second).await?;
+writer.close().await?;
+# Ok(())
 # }
 ```
 
-Most of the time, this can't maximize write performance due to limitations on a single connection. We can perform concurrent writes to improve performance.
+Set `concurrent` above `1` to let services with multipart or block-based write
+support upload independent parts in parallel:
 
 ```rust
-# use opendal::Operator;
-# async fn test() {
-let w = op
-    .writer_with("test.txt")
+# use opendal_core::{Buffer, Operator, Result};
+# async fn write_concurrently(
+#     op: &Operator,
+#     first: Buffer,
+#     second: Buffer,
+# ) -> Result<()> {
+let mut writer = op
+    .writer_with("large-object.bin")
     .concurrent(8)
     .await?;
 
-w.write(data1).await?;
-w.write(data2).await?;
-w.close().await?;
+writer.write(first).await?;
+writer.write(second).await?;
+writer.close().await?;
+# Ok(())
 # }
 ```
 
-After setting `concurrent`, OpenDAL will attempt to write the specified file concurrently. The maximum level of concurrency is determined by the `concurrent` parameter. By default, it is set to 1, indicating a sequential write operation.
+For multipart and block writers, OpenDAL schedules part uploads through its
+[`Executor`][crate::Executor] and tracks their results in a task queue. A write
+call can return while earlier parts are still in flight when the queue has
+capacity. [`Writer::close`][crate::Writer::close] drains the queue and performs
+the final commit, so applications must always check the result from `close`.
 
-Under the hood, OpenDAL maintains a task queue to manage concurrent writes. It spawns asynchronous tasks in the background using the [`Executor`][crate::Executor] and tracks the status of each task. The task queue is flushed when the writer is closed, allowing data to be written concurrently without blocking the main thread.
-
-Take our example here, the write of `data1` will not block the write of `data2`. The two writes will be executed concurrently.
-
-The underlying implementation of concurrent writes may vary depending on the backend. For instance, the `s3` backend leverages the S3 Multipart Uploads API to handle concurrent writes, while the `azblob` backend utilizes the Block API for the same purpose.
+The storage service determines the native mechanism. S3 uses multipart upload,
+while Azure Blob uses block upload. Services without a concurrent write path
+continue to execute sequentially even when the option is present. Check
+[`Capability::write_can_multi`][crate::Capability::write_can_multi] before
+tuning multipart or block writes.
 
 ## Tuning
 
-There are two parameters that can be tuned to optimize concurrent writes:
+Concurrent writes have two controls:
 
-- `concurrent`: This parameter controls the maximum number of concurrent writes. The default value is 1.
-- `chunk`: This parameter specifies the size of each chunk of data to be written. The default value is vary for different storage services.
+- `concurrent` is the maximum number of part uploads OpenDAL attempts to keep
+  in flight. The default is `1`, and values below `1` are normalized to `1`.
+- `chunk` is the target size of each part. Its default and valid range depend on
+  the service.
 
 ### `concurrent`
 
-The most important thing to understand is that `concurrent` is not a strict limit. It represents the maximum number of concurrent writes that OpenDAL will attempt to perform. The actual number of concurrent writes may be lower, depending on the input data throughput.
-
-For example, if you set `concurrent` to 8, OpenDAL will attempt to perform up to 8 concurrent writes. However, if the input data throughput is low, it might only carry out 2 or 3 concurrent writes at a time, as there isn't enough data to keep all 8 writes active.
-
-The best value for `concurrent` depends on the specific use case and the underlying storage service. In general, a higher value can lead to better performance, but it highly depends on the storage service and the network conditions. For example, if the storage service is robust and bandwidth is sufficient, you may observe a linear increase in performance with higher `concurrent` values. However, if the storage service has request limits or the network is nearly saturated, increasing `concurrent` may not lead to any performance improvement—and could even degrade performance due to infinite retries on errors.
+`concurrent` is an upper bound, not a guaranteed level. Setting it to `8` does
+not create eight active requests unless the input arrives quickly enough and
+the object contains enough parts. Object size, request latency, and service
+support can all reduce the observed concurrency.
 
-It's recommended to start with a lower value like `2` or `4` and gradually increase it while monitoring performance and resource usage.
+Start with `2` or `4`, then try `8` while measuring the real workload. Higher
+concurrency can improve bandwidth utilization, but it also increases in-flight
+requests, buffered data, and pressure on service request limits. Once the
+network or service is saturated, increasing it further can reduce throughput by
+causing throttling and retries.
 
 ### `chunk`
 
-The `chunk` parameter specifies the size of each chunk of data to be written. A larger chunk size can improve performance, but it may also increase memory usage. The default value is vary for different storage services. 
+OpenDAL buffers small writes until it can form a part of the configured size.
+This avoids issuing one storage request for every small input buffer. The final
+part may be smaller.
 
-For example, s3 is using `5MiB` as the default chunk size. It's also the minimum chunk size for s3. If you set a smaller chunk size, OpenDAL will automatically adjust it to `5MiB`.
+Larger chunks reduce request count and per-request overhead, but use more memory
+per in-flight part and require more data to be retried after a failed upload.
+Smaller chunks begin parallel work sooner, but increase request rate and may be
+invalid for the service. For example, S3 multipart upload has a 5 MiB minimum
+part size; OpenDAL raises smaller configured values to the service minimum.
 
-The best value for `chunk` depends on the specific use case and the underlying storage service. For most object storage services, a chunk size of `8MiB` or larger is recommended. However, if you're working with smaller files or have limited memory resources, you may want to use a smaller chunk size.
+Inspect [`Capability::write_multi_min_size`][crate::Capability::write_multi_min_size]
+and [`Capability::write_multi_max_size`][crate::Capability::write_multi_max_size]
+before overriding the service default. For large object-storage uploads, 8 MiB
+is a useful starting point, followed by measurements at the expected object
+sizes and concurrency.
 
-Please note that if you input small chunks of data, OpenDAL will attempt to merge them into a larger chunk before writing. This helps avoid the overhead of writing numerous small chunks, which can negatively affect performance.
+## One-shot and adapter APIs
 
-## Usage
-
-To upload a large in-memory chunk concurrently:
+`write_with` can split one in-memory buffer into concurrent parts:
 
 ```rust
-# use opendal::Operator;
-# async fn test() {
-let data = vec![0; 10 * 1024 * 1024]; // 10MiB
-let _ = op.write_with("test.txt", data).concurrent(4).await?;
+# use opendal_core::{Buffer, Operator, Result};
+# async fn upload(op: &Operator, data: Buffer) -> Result<()> {
+let _metadata = op
+    .write_with("large-object.bin", data)
+    .chunk(8 * 1024 * 1024)
+    .concurrent(4)
+    .await?;
+# Ok(())
 # }
 ```
 
-`concurrent` and `chunk` also works in [`into_sink`][crate::Writer::into_sink], [`into_bytes_sink`][crate::Writer::into_bytes_sink] and [`into_futures_async_write`][crate::Writer::into_futures_async_write]:
+A configured writer keeps the same chunking and concurrency behavior after it
+is converted with [`Writer::into_sink`][crate::Writer::into_sink],
+[`Writer::into_bytes_sink`][crate::Writer::into_bytes_sink], or
+[`Writer::into_futures_async_write`][crate::Writer::into_futures_async_write].
 
 ```rust
 use std::io;
 
-use bytes::Bytes;
 use futures::SinkExt;
-use opendal::{Buffer, Operator};
-use opendal::Result;
-
-async fn test(op: Operator) -> io::Result<()> {
-    let mut w = op
-        .writer_with("hello.txt")
-        .concurrent(8)
-        .chunk(256)
+use opendal_core::{Buffer, Operator};
+
+async fn upload_with_sink(op: Operator, data: Buffer) -> io::Result<()> {
+    let mut sink = op
+        .writer_with("large-object.bin")
+        .chunk(8 * 1024 * 1024)
+        .concurrent(4)
         .await?
         .into_sink();
-    let bs = "Hello, World!".as_bytes();
-    w.send(Buffer::from(bs)).await?;
-    w.close().await?;
 
+    sink.send(data).await?;
+    sink.close().await?;
     Ok(())
 }
 ```
+
+Measure end-to-end throughput and completion latency together with peak memory,
+request count, retries, throttling, and the result from the final close.