opendal 0.1.6.pre.rc.1

data/core/src/layers/async_backtrace.rs

@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+
+/// Add Efficient, logical 'stack' traces of async functions for the underlying services.
+///
+/// # Async Backtrace
+///
+/// async-backtrace allows developers to get a stack trace of the async functions.
+/// Read more about [async-backtrace](https://docs.rs/async-backtrace/latest/async_backtrace/)
+///
+/// # Examples
+///
+/// ```no_run
+/// # use opendal::layers::AsyncBacktraceLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+/// # use opendal::Scheme;
+///
+/// # fn main() -> Result<()> {
+/// let _ = Operator::new(services::Memory::default())?
+///     .layer(AsyncBacktraceLayer::default())
+///     .finish();
+/// Ok(())
+/// # }
+/// ```
+#[derive(Clone, Default)]
+pub struct AsyncBacktraceLayer;
+
+impl<A: Access> Layer<A> for AsyncBacktraceLayer {
+    type LayeredAccess = AsyncBacktraceAccessor<A>;
+
+    fn layer(&self, accessor: A) -> Self::LayeredAccess {
+        AsyncBacktraceAccessor { inner: accessor }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AsyncBacktraceAccessor<A: Access> {
+    inner: A,
+}
+
+impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
+    type Inner = A;
+    type Reader = AsyncBacktraceWrapper<A::Reader>;
+    type Writer = AsyncBacktraceWrapper<A::Writer>;
+    type Lister = AsyncBacktraceWrapper<A::Lister>;
+    type Deleter = AsyncBacktraceWrapper<A::Deleter>;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    #[async_backtrace::framed]
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        self.inner
+            .read(path, args)
+            .await
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+    }
+
+    #[async_backtrace::framed]
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        self.inner
+            .write(path, args)
+            .await
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+    }
+
+    #[async_backtrace::framed]
+    async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+        self.inner.copy(from, to, args).await
+    }
+
+    #[async_backtrace::framed]
+    async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+        self.inner.rename(from, to, args).await
+    }
+
+    #[async_backtrace::framed]
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        self.inner.stat(path, args).await
+    }
+
+    #[async_backtrace::framed]
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        self.inner
+            .delete()
+            .await
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+    }
+
+    #[async_backtrace::framed]
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        self.inner
+            .list(path, args)
+            .await
+            .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+    }
+
+    #[async_backtrace::framed]
+    async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+        self.inner.presign(path, args).await
+    }
+}
+
+pub struct AsyncBacktraceWrapper<R> {
+    inner: R,
+}
+
+impl<R> AsyncBacktraceWrapper<R> {
+    fn new(inner: R) -> Self {
+        Self { inner }
+    }
+}
+
+impl<R: oio::Read> oio::Read for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn read(&mut self) -> Result<Buffer> {
+        self.inner.read().await
+    }
+}
+
+impl<R: oio::Write> oio::Write for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.inner.write(bs).await
+    }
+
+    #[async_backtrace::framed]
+    async fn close(&mut self) -> Result<Metadata> {
+        self.inner.close().await
+    }
+
+    #[async_backtrace::framed]
+    async fn abort(&mut self) -> Result<()> {
+        self.inner.abort().await
+    }
+}
+
+impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
+    #[async_backtrace::framed]
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        self.inner.next().await
+    }
+}
+
+impl<R: oio::Delete> oio::Delete for AsyncBacktraceWrapper<R> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        self.inner.delete(path, args)
+    }
+
+    #[async_backtrace::framed]
+    async fn flush(&mut self) -> Result<usize> {
+        self.inner.flush().await
+    }
+}

data/core/src/layers/await_tree.rs

@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use await_tree::InstrumentAwait;
+use futures::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// Add an Instrument await-tree for actor-based applications to the underlying services.
+///
+/// # AwaitTree
+///
+/// await-tree allows developers to dump this execution tree at runtime,
+/// with the span of each Future annotated by instrument_await.
+/// Read more about [await-tree](https://docs.rs/await-tree/latest/await_tree/)
+///
+/// # Examples
+///
+/// ```no_run
+/// # use opendal::layers::AwaitTreeLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+/// # use opendal::Scheme;
+///
+/// # fn main() -> Result<()> {
+/// let _ = Operator::new(services::Memory::default())?
+///     .layer(AwaitTreeLayer::new())
+///     .finish();
+/// Ok(())
+/// # }
+/// ```
+#[derive(Clone, Default)]
+pub struct AwaitTreeLayer {}
+
+impl AwaitTreeLayer {
+    /// Create a new `AwaitTreeLayer`.
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl<A: Access> Layer<A> for AwaitTreeLayer {
+    type LayeredAccess = AwaitTreeAccessor<A>;
+
+    fn layer(&self, accessor: A) -> Self::LayeredAccess {
+        AwaitTreeAccessor { inner: accessor }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct AwaitTreeAccessor<A: Access> {
+    inner: A,
+}
+
+impl<A: Access> LayeredAccess for AwaitTreeAccessor<A> {
+    type Inner = A;
+    type Reader = AwaitTreeWrapper<A::Reader>;
+    type Writer = AwaitTreeWrapper<A::Writer>;
+    type Lister = AwaitTreeWrapper<A::Lister>;
+    type Deleter = AwaitTreeWrapper<A::Deleter>;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        self.inner
+            .read(path, args)
+            .instrument_await(format!("opendal::{}", Operation::Read))
+            .await
+            .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        self.inner
+            .write(path, args)
+            .instrument_await(format!("opendal::{}", Operation::Write))
+            .await
+            .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+    }
+
+    async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+        self.inner()
+            .copy(from, to, args)
+            .instrument_await(format!("opendal::{}", Operation::Copy))
+            .await
+    }
+
+    async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+        self.inner()
+            .rename(from, to, args)
+            .instrument_await(format!("opendal::{}", Operation::Rename))
+            .await
+    }
+
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        self.inner
+            .stat(path, args)
+            .instrument_await(format!("opendal::{}", Operation::Stat))
+            .await
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        self.inner
+            .delete()
+            .instrument_await(format!("opendal::{}", Operation::Delete))
+            .await
+            .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        self.inner
+            .list(path, args)
+            .instrument_await(format!("opendal::{}", Operation::List))
+            .await
+            .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+    }
+
+    async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+        self.inner
+            .presign(path, args)
+            .instrument_await(format!("opendal::{}", Operation::Presign))
+            .await
+    }
+}
+
+pub struct AwaitTreeWrapper<R> {
+    inner: R,
+}
+
+impl<R> AwaitTreeWrapper<R> {
+    fn new(inner: R) -> Self {
+        Self { inner }
+    }
+}
+
+impl<R: oio::Read> oio::Read for AwaitTreeWrapper<R> {
+    async fn read(&mut self) -> Result<Buffer> {
+        self.inner
+            .read()
+            .instrument_await(format!("opendal::{}", Operation::Read))
+            .await
+    }
+}
+
+impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
+    fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend {
+        self.inner
+            .write(bs)
+            .instrument_await(format!("opendal::{}", Operation::Write.into_static()))
+    }
+
+    fn abort(&mut self) -> impl Future<Output = Result<()>> + MaybeSend {
+        self.inner
+            .abort()
+            .instrument_await(format!("opendal::{}", Operation::Write.into_static()))
+    }
+
+    fn close(&mut self) -> impl Future<Output = Result<Metadata>> + MaybeSend {
+        self.inner
+            .close()
+            .instrument_await(format!("opendal::{}", Operation::Write.into_static()))
+    }
+}
+
+impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        self.inner
+            .next()
+            .instrument_await(format!("opendal::{}", Operation::List))
+            .await
+    }
+}
+
+impl<R: oio::Delete> oio::Delete for AwaitTreeWrapper<R> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        self.inner.delete(path, args)
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        self.inner
+            .flush()
+            .instrument_await(format!("opendal::{}", Operation::Delete))
+            .await
+    }
+}

data/core/src/layers/capability_check.rs

@@ -0,0 +1,239 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use crate::layers::correctness_check::new_unsupported_error;
+use crate::raw::*;
+
+/// Add an extra capability check layer for every operation
+///
+/// Similar to `CorrectnessChecker`, Before performing any operations, this layer will first verify
+/// its arguments against the capability of the underlying service. If the arguments is not supported,
+/// an error will be returned directly.
+///
+/// Notes
+///
+/// There are two main differences between this checker with the `CorrectnessChecker`:
+/// 1. This checker provides additional checks for capabilities like write_with_content_type and
+///    list_with_versions, among others. These capabilities do not affect data integrity, even if
+///    the underlying storage services do not support them.
+///
+/// 2. OpenDAL doesn't apply this checker by default. Users can enable this layer if they want to
+///    enforce stricter requirements.
+///
+/// # examples
+///
+/// ```no_run
+/// # use opendal::layers::CapabilityCheckLayer;
+/// # use opendal::services;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+/// # use opendal::Scheme;
+///
+/// # fn main() -> Result<()> {
+/// use opendal::layers::CapabilityCheckLayer;
+/// let _ = Operator::new(services::Memory::default())?
+///     .layer(CapabilityCheckLayer)
+///     .finish();
+/// Ok(())
+/// # }
+/// ```
+#[derive(Default)]
+pub struct CapabilityCheckLayer;
+
+impl<A: Access> Layer<A> for CapabilityCheckLayer {
+    type LayeredAccess = CapabilityAccessor<A>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccess {
+        let info = inner.info();
+
+        CapabilityAccessor { info, inner }
+    }
+}
+pub struct CapabilityAccessor<A: Access> {
+    info: Arc<AccessorInfo>,
+    inner: A,
+}
+
+impl<A: Access> Debug for CapabilityAccessor<A> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("CapabilityCheckAccessor")
+            .field("inner", &self.inner)
+            .finish_non_exhaustive()
+    }
+}
+
+impl<A: Access> LayeredAccess for CapabilityAccessor<A> {
+    type Inner = A;
+    type Reader = A::Reader;
+    type Writer = A::Writer;
+    type Lister = A::Lister;
+    type Deleter = A::Deleter;
+
+    fn inner(&self) -> &Self::Inner {
+        &self.inner
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> crate::Result<(RpRead, Self::Reader)> {
+        self.inner.read(path, args).await
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> crate::Result<(RpWrite, Self::Writer)> {
+        let capability = self.info.full_capability();
+        if !capability.write_with_content_type && args.content_type().is_some() {
+            return Err(new_unsupported_error(
+                self.info.as_ref(),
+                Operation::Write,
+                "content_type",
+            ));
+        }
+        if !capability.write_with_cache_control && args.cache_control().is_some() {
+            return Err(new_unsupported_error(
+                self.info.as_ref(),
+                Operation::Write,
+                "cache_control",
+            ));
+        }
+        if !capability.write_with_content_disposition && args.content_disposition().is_some() {
+            return Err(new_unsupported_error(
+                self.info.as_ref(),
+                Operation::Write,
+                "content_disposition",
+            ));
+        }
+
+        self.inner.write(path, args).await
+    }
+
+    async fn delete(&self) -> crate::Result<(RpDelete, Self::Deleter)> {
+        self.inner.delete().await
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> crate::Result<(RpList, Self::Lister)> {
+        let capability = self.info.full_capability();
+        if !capability.list_with_versions && args.versions() {
+            return Err(new_unsupported_error(
+                self.info.as_ref(),
+                Operation::List,
+                "version",
+            ));
+        }
+
+        self.inner.list(path, args).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Capability;
+    use crate::ErrorKind;
+    use crate::Operator;
+
+    #[derive(Debug)]
+    struct MockService {
+        capability: Capability,
+    }
+
+    impl Access for MockService {
+        type Reader = oio::Reader;
+        type Writer = oio::Writer;
+        type Lister = oio::Lister;
+        type Deleter = oio::Deleter;
+
+        fn info(&self) -> Arc<AccessorInfo> {
+            let info = AccessorInfo::default();
+            info.set_native_capability(self.capability);
+
+            info.into()
+        }
+
+        async fn write(&self, _: &str, _: OpWrite) -> crate::Result<(RpWrite, Self::Writer)> {
+            Ok((RpWrite::new(), Box::new(())))
+        }
+
+        async fn list(&self, _: &str, _: OpList) -> crate::Result<(RpList, Self::Lister)> {
+            Ok((RpList {}, Box::new(())))
+        }
+    }
+
+    fn new_test_operator(capability: Capability) -> Operator {
+        let srv = MockService { capability };
+
+        Operator::from_inner(Arc::new(srv)).layer(CapabilityCheckLayer)
+    }
+
+    #[tokio::test]
+    async fn test_writer_with() {
+        let op = new_test_operator(Capability {
+            write: true,
+            ..Default::default()
+        });
+        let res = op.writer_with("path").content_type("type").await;
+        assert!(res.is_err());
+
+        let res = op.writer_with("path").cache_control("cache").await;
+        assert!(res.is_err());
+
+        let res = op
+            .writer_with("path")
+            .content_disposition("disposition")
+            .await;
+        assert!(res.is_err());
+
+        let op = new_test_operator(Capability {
+            write: true,
+            write_with_content_type: true,
+            write_with_cache_control: true,
+            write_with_content_disposition: true,
+            ..Default::default()
+        });
+        let res = op.writer_with("path").content_type("type").await;
+        assert!(res.is_ok());
+
+        let res = op.writer_with("path").cache_control("cache").await;
+        assert!(res.is_ok());
+
+        let res = op
+            .writer_with("path")
+            .content_disposition("disposition")
+            .await;
+        assert!(res.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_list_with() {
+        let op = new_test_operator(Capability {
+            list: true,
+            ..Default::default()
+        });
+        let res = op.list_with("path/").versions(true).await;
+        assert!(res.is_err());
+        assert_eq!(res.unwrap_err().kind(), ErrorKind::Unsupported);
+
+        let op = new_test_operator(Capability {
+            list: true,
+            list_with_versions: true,
+            ..Default::default()
+        });
+        let res = op.lister_with("path/").versions(true).await;
+        assert!(res.is_ok())
+    }
+}