opendal 0.1.6.pre.rc.1

data/core/src/blocking/read/buffer_iterator.rs

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use futures::StreamExt;
+
+use crate::Buffer;
+use crate::*;
+
+/// BufferIterator is an iterator of buffers.
+///
+/// # TODO
+///
+/// We can support chunked reader for concurrent read in the future.
+pub struct BufferIterator {
+    handle: tokio::runtime::Handle,
+    inner: Option<BufferStream>,
+}
+
+impl BufferIterator {
+    /// Create a new buffer iterator.
+    #[inline]
+    pub(crate) fn new(handle: tokio::runtime::Handle, inner: BufferStream) -> Self {
+        Self {
+            handle,
+            inner: Some(inner),
+        }
+    }
+}
+
+impl Iterator for BufferIterator {
+    type Item = Result<Buffer>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let Some(inner) = self.inner.as_mut() else {
+            return Some(Err(Error::new(
+                ErrorKind::Unexpected,
+                "reader has been dropped",
+            )));
+        };
+
+        self.handle.block_on(inner.next())
+    }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for BufferIterator {
+    fn drop(&mut self) {
+        if let Some(v) = self.inner.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/read/mod.rs

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod reader;
+pub use reader::Reader;
+
+mod buffer_iterator;
+pub use buffer_iterator::BufferIterator;
+
+mod std_bytes_iterator;
+pub use std_bytes_iterator::StdBytesIterator;
+mod std_reader;
+pub use std_reader::StdReader;

data/core/src/blocking/read/reader.rs

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::ops::RangeBounds;
+
+use bytes::BufMut;
+
+use super::BufferIterator;
+use super::StdBytesIterator;
+use super::StdReader;
+use crate::Reader as AsyncReader;
+use crate::*;
+
+/// BlockingReader is designed to read data from given path in an blocking
+/// manner.
+#[derive(Clone)]
+pub struct Reader {
+    handle: tokio::runtime::Handle,
+    inner: Option<AsyncReader>,
+}
+
+impl Reader {
+    /// Create a new blocking reader.
+    ///
+    /// We don't want to expose those details to users so keep this function
+    /// in crate only.
+    pub(crate) fn new(handle: tokio::runtime::Handle, inner: AsyncReader) -> Self {
+        Reader {
+            handle,
+            inner: Some(inner),
+        }
+    }
+
+    /// Read give range from reader into [`Buffer`].
+    ///
+    /// This operation is zero-copy, which means it keeps the [`bytes::Bytes`] returned by underlying
+    /// storage services without any extra copy or intensive memory allocations.
+    ///
+    /// # Notes
+    ///
+    /// - Buffer length smaller than range means we have reached the end of file.
+    pub fn read(&self, range: impl RangeBounds<u64>) -> Result<Buffer> {
+        let inner = self
+            .inner
+            .as_ref()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+        self.handle.block_on(inner.read(range))
+    }
+
+    ///
+    /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while
+    /// [`BufMut`] doesn't have enough space.
+    ///
+    /// # Notes
+    ///
+    /// - Returning length smaller than range means we have reached the end of file.
+    pub fn read_into(&self, buf: &mut impl BufMut, range: impl RangeBounds<u64>) -> Result<usize> {
+        let inner = self
+            .inner
+            .as_ref()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+        self.handle.block_on(inner.read_into(buf, range))
+    }
+
+    /// Create a buffer iterator to read specific range from given reader.
+    pub fn into_iterator(mut self, range: impl RangeBounds<u64>) -> Result<BufferIterator> {
+        let inner = self
+            .inner
+            .take()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+        let iter = self.handle.block_on(inner.into_stream(range))?;
+
+        Ok(BufferIterator::new(self.handle.clone(), iter))
+    }
+
+    /// Convert reader into [`StdReader`] which implements [`futures::AsyncRead`],
+    /// [`futures::AsyncSeek`] and [`futures::AsyncBufRead`].
+    #[inline]
+    pub fn into_std_read(mut self, range: impl RangeBounds<u64>) -> Result<StdReader> {
+        let inner = self
+            .inner
+            .take()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+
+        let r = self.handle.block_on(inner.into_futures_async_read(range))?;
+
+        Ok(StdReader::new(self.handle.clone(), r))
+    }
+
+    /// Convert reader into [`StdBytesIterator`] which implements [`Iterator`].
+    #[inline]
+    pub fn into_bytes_iterator(mut self, range: impl RangeBounds<u64>) -> Result<StdBytesIterator> {
+        let inner = self
+            .inner
+            .take()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+
+        let iter = self.handle.block_on(inner.into_bytes_stream(range))?;
+        Ok(StdBytesIterator::new(self.handle.clone(), iter))
+    }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for Reader {
+    fn drop(&mut self) {
+        if let Some(v) = self.inner.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/read/std_bytes_iterator.rs

@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+
+use bytes::Bytes;
+use futures::StreamExt;
+
+use crate::*;
+
+/// StdIterator is the adapter of [`Iterator`] for [`BlockingReader`][crate::BlockingReader].
+///
+/// Users can use this adapter in cases where they need to use [`Iterator`] trait.
+///
+/// StdIterator also implements [`Send`] and [`Sync`].
+pub struct StdBytesIterator {
+    handle: tokio::runtime::Handle,
+    inner: Option<FuturesBytesStream>,
+}
+
+impl StdBytesIterator {
+    /// NOTE: don't allow users to create StdIterator directly.
+    #[inline]
+    pub(crate) fn new(handle: tokio::runtime::Handle, inner: FuturesBytesStream) -> Self {
+        StdBytesIterator {
+            handle,
+            inner: Some(inner),
+        }
+    }
+}
+
+impl Iterator for StdBytesIterator {
+    type Item = io::Result<Bytes>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let Some(inner) = self.inner.as_mut() else {
+            return Some(Err(Error::new(
+                ErrorKind::Unexpected,
+                "reader has been dropped",
+            )
+            .into()));
+        };
+
+        self.handle.block_on(inner.next())
+    }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for StdBytesIterator {
+    fn drop(&mut self) {
+        if let Some(v) = self.inner.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/read/std_reader.rs

@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::io::BufRead;
+use std::io::Read;
+use std::io::Seek;
+use std::io::SeekFrom;
+
+use futures::AsyncBufReadExt;
+use futures::AsyncReadExt;
+use futures::AsyncSeekExt;
+
+use crate::*;
+
+/// StdReader is the adapter of [`Read`], [`Seek`] and [`BufRead`] for [`BlockingReader`][crate::BlockingReader].
+///
+/// Users can use this adapter in cases where they need to use [`Read`] or [`BufRead`] trait.
+///
+/// StdReader also implements [`Send`] and [`Sync`].
+pub struct StdReader {
+    handle: tokio::runtime::Handle,
+    r: Option<FuturesAsyncReader>,
+}
+
+impl StdReader {
+    /// NOTE: don't allow users to create StdReader directly.
+    #[inline]
+    pub(super) fn new(handle: tokio::runtime::Handle, r: FuturesAsyncReader) -> Self {
+        Self { handle, r: Some(r) }
+    }
+}
+
+impl BufRead for StdReader {
+    fn fill_buf(&mut self) -> io::Result<&[u8]> {
+        let Some(r) = self.r.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+        };
+
+        self.handle.block_on(r.fill_buf())
+    }
+
+    fn consume(&mut self, amt: usize) {
+        let Some(r) = self.r.as_mut() else {
+            return;
+        };
+
+        r.consume_unpin(amt);
+    }
+}
+
+impl Read for StdReader {
+    #[inline]
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        let Some(r) = self.r.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+        };
+
+        self.handle.block_on(r.read(buf))
+    }
+}
+
+impl Seek for StdReader {
+    #[inline]
+    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+        let Some(r) = self.r.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+        };
+
+        self.handle.block_on(r.seek(pos))
+    }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for StdReader {
+    fn drop(&mut self) {
+        if let Some(v) = self.r.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/write/mod.rs

@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod writer;
+pub use writer::Writer;
+
+mod std_writer;
+pub use std_writer::StdWriter;

data/core/src/blocking/write/std_writer.rs

@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Write;
+
+use futures::AsyncWriteExt;
+
+use crate::*;
+
+/// StdWriter is the adapter of [`std::io::Write`] for [`BlockingWriter`].
+///
+/// Users can use this adapter in cases where they need to use [`std::io::Write`] related trait.
+///
+/// # Notes
+///
+/// Files are automatically closed when they go out of scope. Errors detected on closing are ignored
+/// by the implementation of Drop. Use the method `close` if these errors must be manually handled.
+pub struct StdWriter {
+    handle: tokio::runtime::Handle,
+    w: Option<FuturesAsyncWriter>,
+}
+
+impl StdWriter {
+    /// NOTE: don't allow users to create directly.
+    #[inline]
+    pub(crate) fn new(handle: tokio::runtime::Handle, w: Writer) -> Self {
+        StdWriter {
+            handle,
+            w: Some(w.into_futures_async_write()),
+        }
+    }
+
+    /// Close the internal writer and make sure all data have been stored.
+    pub fn close(&mut self) -> std::io::Result<()> {
+        let Some(w) = self.w.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
+        };
+
+        self.handle.block_on(w.close())
+    }
+}
+
+impl Write for StdWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        let Some(w) = self.w.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
+        };
+
+        self.handle.block_on(w.write(buf))
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        let Some(w) = self.w.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
+        };
+
+        self.handle.block_on(w.flush())
+    }
+}
+
+/// Make sure the inner writer is dropped in async context.
+impl Drop for StdWriter {
+    fn drop(&mut self) {
+        if let Some(v) = self.w.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/write/writer.rs

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::std_writer::StdWriter;
+use crate::Writer as AsyncWriter;
+use crate::*;
+
+/// BlockingWriter is designed to write data into given path in an blocking
+/// manner.
+pub struct Writer {
+    handle: tokio::runtime::Handle,
+    inner: Option<AsyncWriter>,
+}
+
+impl Writer {
+    /// Create a new writer.
+    ///
+    /// Create will use internal information to decide the most suitable
+    /// implementation for users.
+    ///
+    /// We don't want to expose those details to users so keep this function
+    /// in crate only.
+    pub(crate) fn new(handle: tokio::runtime::Handle, inner: AsyncWriter) -> Self {
+        Self {
+            handle,
+            inner: Some(inner),
+        }
+    }
+
+    /// Write [`Buffer`] into writer.
+    ///
+    /// This operation will write all data in given buffer into writer.
+    ///
+    /// ## Examples
+    ///
+    /// ```
+    /// use bytes::Bytes;
+    /// use opendal::blocking;
+    /// use opendal::blocking::Operator;
+    /// use opendal::Result;
+    ///
+    /// fn test(op: blocking::Operator) -> Result<()> {
+    ///     let mut w = op.writer("hello.txt")?;
+    ///     // Buffer can be created from continues bytes.
+    ///     w.write("hello, world")?;
+    ///     // Buffer can also be created from non-continues bytes.
+    ///     w.write(vec![Bytes::from("hello,"), Bytes::from("world!")])?;
+    ///
+    ///     // Make sure file has been written completely.
+    ///     w.close()?;
+    ///     Ok(())
+    /// }
+    /// ```
+    pub fn write(&mut self, bs: impl Into<Buffer>) -> Result<()> {
+        let Some(inner) = self.inner.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped"));
+        };
+
+        self.handle.block_on(inner.write(bs))
+    }
+
+    /// Close the writer and make sure all data have been committed.
+    ///
+    /// ## Notes
+    ///
+    /// Close should only be called when the writer is not closed or
+    /// aborted, otherwise an unexpected error could be returned.
+    pub fn close(&mut self) -> Result<Metadata> {
+        let Some(inner) = self.inner.as_mut() else {
+            return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped"));
+        };
+
+        self.handle.block_on(inner.close())
+    }
+
+    /// Convert writer into [`StdWriter`] which implements [`std::io::Write`],
+    pub fn into_std_write(mut self) -> StdWriter {
+        let inner = self
+            .inner
+            .take()
+            .ok_or_else(|| Error::new(ErrorKind::Unexpected, "writer has been dropped"))
+            .expect("writer has been dropped");
+
+        StdWriter::new(self.handle.clone(), inner)
+    }
+}
+
+/// Make sure the inner writer is dropped in async context.
+impl Drop for Writer {
+    fn drop(&mut self) {
+        if let Some(v) = self.inner.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/docs/comparisons/mod.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Compare opendal with other projects to find out the differences and areas that opendal can improve.
+//!
+//! All documents listed should be treated as highly biased. Because:
+//!
+//! - OpenDAL's maintainers and contributors write them.
+//! - Writers may not be familiar with the compared projects (at least not as familiar with OpenDAL)
+//!
+//! Let's see OpenDAL:
+//!
+//! - [vs `object_store`][`vs_object_store`]
+
+#[doc = include_str!("vs_object_store.md")]
+pub mod vs_object_store {}