opendal 0.1.6.pre.rc.1

data/core/src/layers/tracing.rs

@@ -0,0 +1,349 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::pin::Pin;
+use std::task::Context;
+use std::task::Poll;
+
+use futures::Stream;
+use futures::StreamExt;
+use tracing::span;
+use tracing::Level;
+use tracing::Span;
+
+use crate::raw::*;
+use crate::*;
+
+/// Add [tracing](https://docs.rs/tracing/) for every operation.
+///
+/// # Examples
+///
+/// ## Basic Setup
+///
+/// ```no_run
+/// # use opendal::layers::TracingLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+///
+/// # fn main() -> Result<()> {
+/// let _ = Operator::new(services::Memory::default())?
+///     .layer(TracingLayer)
+///     .finish();
+/// Ok(())
+/// # }
+/// ```
+///
+/// ## Real usage
+///
+/// ```no_run
+/// # use anyhow::Result;
+/// # use opendal::layers::TracingLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opentelemetry::KeyValue;
+/// # use opentelemetry_sdk::trace;
+/// # use opentelemetry_sdk::Resource;
+/// # use tracing_subscriber::prelude::*;
+/// # use tracing_subscriber::EnvFilter;
+///
+/// # fn main() -> Result<()> {
+/// use opentelemetry::trace::TracerProvider;
+/// let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+///     .with_simple_exporter(
+///         opentelemetry_otlp::SpanExporter::builder()
+///             .with_tonic()
+///             .build()?,
+///     )
+///     .with_resource(
+///         Resource::builder()
+///             .with_attributes(vec![KeyValue::new("service.name", "opendal_example")])
+///             .build(),
+///     )
+///     .build();
+/// let tracer = tracer_provider.tracer("opendal_tracer");
+/// let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+///
+/// tracing_subscriber::registry()
+///     .with(EnvFilter::from_default_env())
+///     .with(opentelemetry)
+///     .try_init()?;
+///
+/// {
+///     let runtime = tokio::runtime::Runtime::new()?;
+///     runtime.block_on(async {
+///         let root = tracing::span!(tracing::Level::INFO, "app_start", work_units = 2);
+///         let _enter = root.enter();
+///
+///         let _ = dotenvy::dotenv();
+///         let op = Operator::new(services::Memory::default())?
+///             .layer(TracingLayer)
+///             .finish();
+///
+///         op.write("test", "0".repeat(16 * 1024 * 1024).into_bytes())
+///             .await?;
+///         op.stat("test").await?;
+///         op.read("test").await?;
+///         Ok::<(), opendal::Error>(())
+///     })?;
+/// }
+///
+/// // Shut down the current tracer provider.
+/// // This will invoke the shutdown method on all span processors.
+/// // span processors should export remaining spans before return.
+/// tracer_provider.shutdown()?;
+///
+/// Ok(())
+/// # }
+/// ```
+///
+/// # Output
+///
+/// OpenDAL is using [`tracing`](https://docs.rs/tracing/latest/tracing/) for tracing internally.
+///
+/// To enable tracing output, please init one of the subscribers that `tracing` supports.
+///
+/// For example:
+///
+/// ```no_run
+/// # use tracing::dispatcher;
+/// # use tracing::Event;
+/// # use tracing::Metadata;
+/// # use tracing::span::Attributes;
+/// # use tracing::span::Id;
+/// # use tracing::span::Record;
+/// # use tracing::subscriber::Subscriber;
+///
+/// # pub struct FooSubscriber;
+/// # impl Subscriber for FooSubscriber {
+/// #   fn enabled(&self, _: &Metadata) -> bool { false }
+/// #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }
+/// #   fn record(&self, _: &Id, _: &Record) {}
+/// #   fn record_follows_from(&self, _: &Id, _: &Id) {}
+/// #   fn event(&self, _: &Event) {}
+/// #   fn enter(&self, _: &Id) {}
+/// #   fn exit(&self, _: &Id) {}
+/// # }
+/// # impl FooSubscriber { fn new() -> Self { FooSubscriber } }
+///
+/// let my_subscriber = FooSubscriber::new();
+/// tracing::subscriber::set_global_default(my_subscriber).expect("setting tracing default failed");
+/// ```
+///
+/// For real-world usage, please take a look at [`tracing-opentelemetry`](https://crates.io/crates/tracing-opentelemetry).
+pub struct TracingLayer;
+
+impl<A: Access> Layer<A> for TracingLayer {
+    type LayeredAccess = TracingAccessor<A>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccess {
+        let info = inner.info();
+
+        // Update http client with metrics http fetcher.
+        info.update_http_client(|client| {
+            HttpClient::with(TracingHttpFetcher {
+                inner: client.into_inner(),
+            })
+        });
+
+        TracingAccessor { inner }
+    }
+}
+
+pub struct TracingHttpFetcher {
+    inner: HttpFetcher,
+}
+
+impl HttpFetch for TracingHttpFetcher {
+    async fn fetch(&self, req: http::Request<Buffer>) -> Result<http::Response<HttpBody>> {
+        let span = span!(Level::DEBUG, "http::fetch", ?req);
+
+        let resp = {
+            let _enter = span.enter();
+            self.inner.fetch(req).await?
+        };
+
+        let (parts, body) = resp.into_parts();
+        let body = body.map_inner(|s| Box::new(TracingStream { inner: s, span }));
+        Ok(http::Response::from_parts(parts, body))
+    }
+}
+
+pub struct TracingStream<S> {
+    inner: S,
+    span: Span,
+}
+
+impl<S> Stream for TracingStream<S>
+where
+    S: Stream<Item = Result<Buffer>> + Unpin + 'static,
+{
+    type Item = Result<Buffer>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let _enter = self.span.clone().entered();
+        self.inner.poll_next_unpin(cx)
+    }
+}
+
+#[derive(Debug)]
+pub struct TracingAccessor<A> {
+    inner: A,
+}
+
+impl<A: Access> LayeredAccess for TracingAccessor<A> {
+    type Inner = A;
+    type Reader = TracingWrapper<A::Reader>;
+    type Writer = TracingWrapper<A::Writer>;
+    type Lister = TracingWrapper<A::Lister>;
+    type Deleter = TracingWrapper<A::Deleter>;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
+        self.inner.create_dir(path, args).await
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let span = span!(Level::DEBUG, "read", path, ?args);
+
+        let (rp, r) = {
+            let _enter = span.enter();
+            self.inner.read(path, args).await?
+        };
+
+        Ok((rp, TracingWrapper::new(span, r)))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let span = span!(Level::DEBUG, "write", path, ?args);
+
+        let (rp, r) = {
+            let _enter = span.enter();
+            self.inner.write(path, args).await?
+        };
+
+        Ok((rp, TracingWrapper::new(span, r)))
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+        self.inner().copy(from, to, args).await
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+        self.inner().rename(from, to, args).await
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        self.inner.stat(path, args).await
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        let span = span!(Level::DEBUG, "delete");
+
+        let (rp, r) = {
+            let _enter = span.enter();
+            self.inner.delete().await?
+        };
+
+        Ok((rp, TracingWrapper::new(span, r)))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let span = span!(Level::DEBUG, "list", path, ?args);
+
+        let (rp, r) = {
+            let _enter = span.enter();
+            self.inner.list(path, args).await?
+        };
+
+        Ok((rp, TracingWrapper::new(span, r)))
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+        self.inner.presign(path, args).await
+    }
+}
+
+pub struct TracingWrapper<R> {
+    span: Span,
+    inner: R,
+}
+
+impl<R> TracingWrapper<R> {
+    fn new(span: Span, inner: R) -> Self {
+        Self { span, inner }
+    }
+}
+
+impl<R: oio::Read> oio::Read for TracingWrapper<R> {
+    async fn read(&mut self) -> Result<Buffer> {
+        let _enter = self.span.enter();
+
+        self.inner.read().await
+    }
+}
+
+impl<R: oio::Write> oio::Write for TracingWrapper<R> {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        let _enter = self.span.enter();
+
+        self.inner.write(bs).await
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        let _enter = self.span.enter();
+
+        self.inner.abort().await
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let _enter = self.span.enter();
+
+        self.inner.close().await
+    }
+}
+
+impl<R: oio::List> oio::List for TracingWrapper<R> {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        let _enter = self.span.enter();
+
+        self.inner.next().await
+    }
+}
+
+impl<R: oio::Delete> oio::Delete for TracingWrapper<R> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        let _enter = self.span.enter();
+
+        self.inner.delete(path, args)
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        let _enter = self.span.enter();
+
+        self.inner.flush().await
+    }
+}

data/core/src/layers/type_eraser.rs

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use crate::raw::*;
+use crate::*;
+
+/// TypeEraseLayer will erase the types on internal accessor.
+///
+/// For example, we will erase `Self::Reader` to `oio::Reader` (`Box<dyn oio::Read>`).
+///
+/// # Notes
+///
+/// TypeEraseLayer is not a public accessible layer that can be used by
+/// external users. We use this layer to erase any generic types.
+pub struct TypeEraseLayer;
+
+impl<A: Access> Layer<A> for TypeEraseLayer {
+    type LayeredAccess = TypeEraseAccessor<A>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccess {
+        TypeEraseAccessor { inner }
+    }
+}
+
+/// Provide reader wrapper for backend.
+pub struct TypeEraseAccessor<A: Access> {
+    inner: A,
+}
+
+impl<A: Access> Debug for TypeEraseAccessor<A> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        self.inner.fmt(f)
+    }
+}
+
+impl<A: Access> LayeredAccess for TypeEraseAccessor<A> {
+    type Inner = A;
+    type Reader = oio::Reader;
+    type Writer = oio::Writer;
+    type Lister = oio::Lister;
+    type Deleter = oio::Deleter;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        self.inner
+            .read(path, args)
+            .await
+            .map(|(rp, r)| (rp, Box::new(r) as oio::Reader))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        self.inner
+            .write(path, args)
+            .await
+            .map(|(rp, w)| (rp, Box::new(w) as oio::Writer))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        self.inner
+            .delete()
+            .await
+            .map(|(rp, p)| (rp, Box::new(p) as oio::Deleter))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        self.inner
+            .list(path, args)
+            .await
+            .map(|(rp, p)| (rp, Box::new(p) as oio::Lister))
+    }
+}

data/core/src/lib.rs

@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![doc(
+    html_logo_url = "https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"
+)]
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+
+//! Apache OpenDAL™ is an Open Data Access Layer that enables seamless interaction with diverse storage services.
+//!
+//! OpenDAL's development is guided by its vision of **One Layer, All Storage** and its core principles: **Open Community**, **Solid Foundation**, **Fast Access**, **Object Storage First**, and **Extensible Architecture**. Read the explained vision at [OpenDAL Vision](https://opendal.apache.org/vision).
+//!
+//! # Quick Start
+//!
+//! OpenDAL's API entry points are [`Operator`] and [`blocking::Operator`]. All
+//! public APIs are accessible through the operator. To utilize OpenDAL, you
+//! need to:
+//!
+//! - [Init a service](#init-a-service)
+//! - [Compose layers](#compose-layers)
+//! - [Use operator](#use-operator)
+//!
+//! ## Init a service
+//!
+//! The first step is to pick a service and init it with a builder. All supported
+//! services could be found at [`services`].
+//!
+//! Let's take [`services::S3`] as an example:
+//!
+//! ```no_run
+//! use opendal::services;
+//! use opendal::Operator;
+//! use opendal::Result;
+//!
+//! fn main() -> Result<()> {
+//!     // Pick a builder and configure it.
+//!     let mut builder = services::S3::default().bucket("test");
+//!
+//!     // Init an operator
+//!     let op = Operator::new(builder)?.finish();
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Compose layers
+//!
+//! The next setup is to compose layers. Layers are modules that provide extra
+//! features for every operation. All builtin layers could be found at [`layers`].
+//!
+//! Let's use [`layers::LoggingLayer`] as an example; this layer adds logging to
+//! every operation that OpenDAL performs.
+//!
+//! ```no_run
+//! use opendal::layers::LoggingLayer;
+//! use opendal::services;
+//! use opendal::Operator;
+//! use opendal::Result;
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<()> {
+//!     // Pick a builder and configure it.
+//!     let mut builder = services::S3::default().bucket("test");
+//!
+//!     // Init an operator
+//!     let op = Operator::new(builder)?
+//!         // Init with logging layer enabled.
+//!         .layer(LoggingLayer::default())
+//!         .finish();
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! ## Use operator
+//!
+//! The final step is to use the operator. OpenDAL supports both async [`Operator`]
+//! and blocking [`blocking::Operator`]. Please pick the one that fits your use case.
+//!
+//! Every Operator API follows a consistent pattern. For example, consider the `read` operation:
+//!
+//! - [`Operator::read`]: Executes a read operation.
+//! - [`Operator::read_with`]: Executes a read operation with additional options using the builder pattern.
+//! - [`Operator::read_options`]: Executes a read operation with extra options provided via a [`options::ReadOptions`] struct.
+//! - [`Operator::reader`]: Creates a reader for streaming data, allowing for flexible access.
+//! - [`Operator::reader_with`]: Creates a reader with advanced options using the builder pattern.
+//! - [`Operator::reader_options`]: Creates a reader with extra options provided via a [`options::ReadOptions`] struct.
+//!
+//! The [`Reader`] created by [`Operator`] supports custom read control methods and can be converted
+//! into [`futures::AsyncRead`] or [`futures::Stream`] for broader ecosystem compatibility.
+//!
+//! ```no_run
+//! use opendal::layers::LoggingLayer;
+//! use opendal::options;
+//! use opendal::services;
+//! use opendal::Operator;
+//! use opendal::Result;
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<()> {
+//!     // Pick a builder and configure it.
+//!     let mut builder = services::S3::default().bucket("test");
+//!
+//!     // Init an operator
+//!     let op = Operator::new(builder)?
+//!         // Init with logging layer enabled.
+//!         .layer(LoggingLayer::default())
+//!         .finish();
+//!
+//!     // Fetch this file's metadata
+//!     let meta = op.stat("hello.txt").await?;
+//!     let length = meta.content_length();
+//!
+//!     // Read data from `hello.txt` with options.
+//!     let bs = op
+//!         .read_with("hello.txt")
+//!         .range(0..8 * 1024 * 1024)
+//!         .chunk(1024 * 1024)
+//!         .concurrent(4)
+//!         .await?;
+//!
+//!     // The same to:
+//!     let bs = op
+//!         .read_options("hello.txt", options::ReadOptions {
+//!             range: (0..8 * 1024 * 1024).into(),
+//!             chunk: Some(1024 * 1024),
+//!             concurrent: 4,
+//!             ..Default::default()
+//!         })
+//!         .await?;
+//!
+//!     Ok(())
+//! }
+//! ```
+//!
+//! # Useful Links
+//!
+//! - [Concept][crate::docs::concepts]
+//! - [Internals][crate::docs::internals]
+//! - [Performance Guide][crate::docs::performance]
+
+// Make sure all our public APIs have docs.
+#![deny(missing_docs)]
+
+// Private module with public types, they will be accessed via `opendal::Xxxx`
+mod types;
+pub use types::*;
+
+// Public modules, they will be accessed like `opendal::layers::Xxxx`
+#[cfg(feature = "blocking")]
+pub mod blocking;
+#[cfg(docsrs)]
+pub mod docs;
+pub mod layers;
+pub mod raw;
+pub mod services;
+
+#[cfg(test)]
+mod tests {
+    use std::mem::size_of;
+
+    use super::*;
+    /// This is not a real test case.
+    ///
+    /// We assert our public structs here to make sure we don't introduce
+    /// unexpected struct/enum size change.
+    #[cfg(target_pointer_width = "64")]
+    #[test]
+    fn assert_size() {
+        assert_eq!(16, size_of::<Operator>());
+        assert_eq!(320, size_of::<Entry>());
+        assert_eq!(296, size_of::<Metadata>());
+        assert_eq!(1, size_of::<EntryMode>());
+        assert_eq!(24, size_of::<Scheme>());
+    }
+
+    trait AssertSendSync: Send + Sync {}
+    impl AssertSendSync for Entry {}
+    impl AssertSendSync for Capability {}
+    impl AssertSendSync for Error {}
+    impl AssertSendSync for Reader {}
+    impl AssertSendSync for Writer {}
+    impl AssertSendSync for Lister {}
+    impl AssertSendSync for Operator {}
+
+    /// This is used to make sure our public API implement Send + Sync
+    #[test]
+    fn test_trait() {
+        let _: Box<dyn AssertSendSync> = Box::new(Capability::default());
+    }
+}