opendal 0.1.6.pre.rc.1

data/core/src/types/execute/executor.rs

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::future::Future;
+use std::sync::Arc;
+
+use futures::FutureExt;
+
+use super::*;
+use crate::raw::BoxedStaticFuture;
+use crate::raw::MaybeSend;
+
+/// Executor that runs futures in background.
+///
+/// Executor is created by users and used by opendal. So it's by design that Executor only
+/// expose constructor methods.
+///
+/// Executor will run futures in background and return a `Task` as handle to the future. Users
+/// can call `task.await` to wait for the future to complete or drop the `Task` to cancel it.
+#[derive(Clone)]
+pub struct Executor {
+    executor: Arc<dyn Execute>,
+}
+
+impl Debug for Executor {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Executor")
+    }
+}
+
+impl Default for Executor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Executor {
+    /// Create a default executor.
+    ///
+    /// The default executor is enabled by feature flags. If no feature flags enabled, the default
+    /// executor will always return error if users try to perform concurrent tasks.
+    pub fn new() -> Self {
+        #[cfg(feature = "executors-tokio")]
+        {
+            Self::with(executors::TokioExecutor::default())
+        }
+        #[cfg(not(feature = "executors-tokio"))]
+        {
+            Self::with(())
+        }
+    }
+
+    /// Create a new executor with given execute impl.
+    pub fn with(exec: impl Execute) -> Self {
+        Self {
+            executor: Arc::new(exec),
+        }
+    }
+
+    /// Return the inner executor.
+    pub(crate) fn into_inner(self) -> Arc<dyn Execute> {
+        self.executor
+    }
+
+    /// Return a future that will be resolved after the given timeout.
+    pub(crate) fn timeout(&self) -> Option<BoxedStaticFuture<()>> {
+        self.executor.timeout()
+    }
+
+    /// Run given future in background immediately.
+    pub(crate) fn execute<F>(&self, f: F) -> Task<F::Output>
+    where
+        F: Future + MaybeSend + 'static,
+        F::Output: MaybeSend + 'static,
+    {
+        let (fut, handle) = f.remote_handle();
+        self.executor.execute(Box::pin(fut));
+        Task::new(handle)
+    }
+}

data/core/src/types/execute/executors/mod.rs

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! executors module provides implementations for the [`Execute`](crate::Execute) trait for widely used runtimes.
+//!
+//! Every executor will be hide behind the feature like `executors-xxx`. Users can switch or enable
+//! the executors they want by enabling the corresponding feature. Also, users can provide their
+//! own executor by implementing the [`Execute`](crate::Execute) trait directly.
+
+#[cfg(feature = "executors-tokio")]
+mod tokio_executor;
+#[cfg(feature = "executors-tokio")]
+pub use tokio_executor::TokioExecutor;

data/core/src/types/execute/executors/tokio_executor.rs

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::BoxedStaticFuture;
+use crate::*;
+
+/// Executor that uses the [`tokio::task::spawn`] to execute futures.
+#[derive(Default)]
+pub struct TokioExecutor {}
+
+impl Execute for TokioExecutor {
+    /// Tokio's JoinHandle has its own `abort` support, so dropping handle won't cancel the task.
+    fn execute(&self, f: BoxedStaticFuture<()>) {
+        let _handle = tokio::task::spawn(f);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::AtomicBool;
+    use std::sync::atomic::Ordering;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use tokio::time::sleep;
+
+    use super::*;
+    use crate::Executor;
+
+    #[tokio::test]
+    async fn test_tokio_executor() {
+        let executor = Executor::with(TokioExecutor::default());
+
+        let finished = Arc::new(AtomicBool::new(false));
+
+        let finished_clone = finished.clone();
+        let _task = executor.execute(async move {
+            sleep(Duration::from_secs(1)).await;
+            finished_clone.store(true, Ordering::Relaxed);
+        });
+
+        sleep(Duration::from_secs(2)).await;
+        // Task must have been finished even without await task.
+        assert!(finished.load(Ordering::Relaxed))
+    }
+}

data/core/src/types/execute/mod.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod api;
+pub use api::Execute;
+pub(crate) use api::Task;
+
+mod executor;
+pub use executor::Executor;
+
+pub mod executors;

data/core/src/types/list.rs

@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::pin::Pin;
+use std::task::ready;
+use std::task::Context;
+use std::task::Poll;
+
+use futures::Stream;
+
+use crate::raw::*;
+use crate::*;
+
+/// Lister is designed to list entries at given path in an asynchronous
+/// manner.
+///
+/// - Lister implements `Stream<Item = Result<Entry>>`.
+/// - Lister will return `None` if there is no more entries or error has been returned.
+pub struct Lister {
+    lister: Option<oio::Lister>,
+
+    fut: Option<BoxedStaticFuture<(oio::Lister, Result<Option<oio::Entry>>)>>,
+    errored: bool,
+}
+
+/// # Safety
+///
+/// Lister will only be accessed by `&mut Self`
+unsafe impl Sync for Lister {}
+
+impl Lister {
+    /// Create a new lister.
+    pub(crate) async fn create(acc: Accessor, path: &str, args: OpList) -> Result<Self> {
+        let (_, lister) = acc.list(path, args).await?;
+
+        Ok(Self {
+            lister: Some(lister),
+
+            fut: None,
+            errored: false,
+        })
+    }
+}
+
+impl Stream for Lister {
+    type Item = Result<Entry>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        // Returns `None` if we have errored.
+        if self.errored {
+            return Poll::Ready(None);
+        }
+
+        if let Some(mut lister) = self.lister.take() {
+            let fut = async move {
+                let res = lister.next_dyn().await;
+                (lister, res)
+            };
+            self.fut = Some(Box::pin(fut));
+        }
+
+        if let Some(fut) = self.fut.as_mut() {
+            let (lister, entry) = ready!(fut.as_mut().poll(cx));
+            self.lister = Some(lister);
+            self.fut = None;
+
+            return match entry {
+                Ok(Some(oe)) => Poll::Ready(Some(Ok(oe.into_entry()))),
+                Ok(None) => {
+                    self.lister = None;
+                    Poll::Ready(None)
+                }
+                Err(err) => {
+                    self.errored = true;
+                    Poll::Ready(Some(Err(err)))
+                }
+            };
+        }
+
+        Poll::Ready(None)
+    }
+}
+
+#[cfg(test)]
+#[cfg(feature = "services-azblob")]
+mod tests {
+    use futures::future;
+    use futures::StreamExt;
+
+    use super::*;
+    use crate::services::Azblob;
+
+    /// Inspired by <https://gist.github.com/kyle-mccarthy/1e6ae89cc34495d731b91ebf5eb5a3d9>
+    ///
+    /// Invalid lister should not panic nor endless loop.
+    #[tokio::test]
+    async fn test_invalid_lister() -> Result<()> {
+        let _ = tracing_subscriber::fmt().try_init();
+
+        let builder = Azblob::default()
+            .container("container")
+            .account_name("account_name")
+            .account_key("YWNjb3VudF9rZXk=") // Valid base64 encoding of "account_key"
+            .endpoint("https://account_name.blob.core.windows.net");
+
+        let operator = Operator::new(builder)?.finish();
+
+        let lister = operator.lister("/").await?;
+
+        lister
+            .filter_map(|entry| {
+                dbg!(&entry);
+                future::ready(entry.ok())
+            })
+            .for_each(|entry| {
+                println!("{entry:?}");
+                future::ready(())
+            })
+            .await;
+
+        Ok(())
+    }
+}