opendal 0.1.6.pre.rc.1

data/core/src/raw/oio/read/api.rs

@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::mem;
+use std::ops::DerefMut;
+
+use bytes::Bytes;
+use futures::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// Reader is a type erased [`Read`].
+pub type Reader = Box<dyn ReadDyn>;
+
+/// Read is the internal trait used by OpenDAL to read data from storage.
+///
+/// Users should not use or import this trait unless they are implementing an `Accessor`.
+///
+/// # Notes
+///
+/// ## Object Safety
+///
+/// `Read` uses `async in trait`, making it not object safe, preventing the use of `Box<dyn Read>`.
+/// To address this, we've introduced [`ReadDyn`] and its compatible type `Box<dyn ReadDyn>`.
+///
+/// `ReadDyn` uses `Box::pin()` to transform the returned future into a [`BoxedFuture`], introducing
+/// an additional layer of indirection and an extra allocation. Ideally, `ReadDyn` should occur only
+/// once, at the outermost level of our API.
+pub trait Read: Unpin + Send + Sync {
+    /// Read at the given offset with the given size.
+    fn read(&mut self) -> impl Future<Output = Result<Buffer>> + MaybeSend;
+
+    /// Read all data from the reader.
+    fn read_all(&mut self) -> impl Future<Output = Result<Buffer>> + MaybeSend {
+        async {
+            let mut bufs = vec![];
+            loop {
+                match self.read().await {
+                    Ok(buf) if buf.is_empty() => break,
+                    Ok(buf) => bufs.push(buf),
+                    Err(err) => return Err(err),
+                }
+            }
+            Ok(bufs.into_iter().flatten().collect())
+        }
+    }
+}
+
+impl Read for () {
+    async fn read(&mut self) -> Result<Buffer> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "output reader doesn't support read",
+        ))
+    }
+}
+
+impl Read for Bytes {
+    async fn read(&mut self) -> Result<Buffer> {
+        Ok(Buffer::from(self.split_off(0)))
+    }
+}
+
+impl Read for Buffer {
+    async fn read(&mut self) -> Result<Buffer> {
+        Ok(mem::take(self))
+    }
+}
+
+/// ReadDyn is the dyn version of [`Read`] make it possible to use as
+/// `Box<dyn ReadDyn>`.
+pub trait ReadDyn: Unpin + Send + Sync {
+    /// The dyn version of [`Read::read`].
+    ///
+    /// This function returns a boxed future to make it object safe.
+    fn read_dyn(&mut self) -> BoxedFuture<'_, Result<Buffer>>;
+
+    /// The dyn version of [`Read::read_all`]
+    fn read_all_dyn(&mut self) -> BoxedFuture<'_, Result<Buffer>>;
+}
+
+impl<T: Read + ?Sized> ReadDyn for T {
+    fn read_dyn(&mut self) -> BoxedFuture<'_, Result<Buffer>> {
+        Box::pin(self.read())
+    }
+
+    fn read_all_dyn(&mut self) -> BoxedFuture<'_, Result<Buffer>> {
+        Box::pin(self.read_all())
+    }
+}
+
+/// # NOTE
+///
+/// Take care about the `deref_mut()` here. This makes sure that we are calling functions
+/// upon `&mut T` instead of `&mut Box<T>`. The later could result in infinite recursion.
+impl<T: ReadDyn + ?Sized> Read for Box<T> {
+    async fn read(&mut self) -> Result<Buffer> {
+        self.deref_mut().read_dyn().await
+    }
+
+    async fn read_all(&mut self) -> Result<Buffer> {
+        self.deref_mut().read_all_dyn().await
+    }
+}

data/core/src/raw/oio/read/mod.rs

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod api;
+pub use api::Read;
+pub use api::ReadDyn;
+pub use api::Reader;

data/core/src/raw/oio/write/api.rs

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+use std::ops::DerefMut;
+
+use crate::raw::*;
+use crate::*;
+
+/// Writer is a type erased [`Write`]
+pub type Writer = Box<dyn WriteDyn>;
+
+/// Write is the trait that OpenDAL returns to callers.
+pub trait Write: Unpin + Send + Sync {
+    /// Write given bytes into writer.
+    ///
+    /// # Behavior
+    ///
+    /// - `Ok(())` means all bytes has been written successfully.
+    /// - `Err(err)` means error happens and no bytes has been written.
+    fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend;
+
+    /// Close the writer and make sure all data has been flushed.
+    fn close(&mut self) -> impl Future<Output = Result<Metadata>> + MaybeSend;
+
+    /// Abort the pending writer.
+    fn abort(&mut self) -> impl Future<Output = Result<()>> + MaybeSend;
+}
+
+impl Write for () {
+    async fn write(&mut self, _: Buffer) -> Result<()> {
+        unimplemented!("write is required to be implemented for oio::Write")
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "output writer doesn't support close",
+        ))
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "output writer doesn't support abort",
+        ))
+    }
+}
+
+/// WriteDyn is the dyn version of [`Write`] make it possible to use as
+/// `Box<dyn WriteDyn>`.
+pub trait WriteDyn: Unpin + Send + Sync {
+    /// The dyn version of [`Write::write`].
+    fn write_dyn(&mut self, bs: Buffer) -> BoxedFuture<'_, Result<()>>;
+
+    /// The dyn version of [`Write::close`].
+    fn close_dyn(&mut self) -> BoxedFuture<'_, Result<Metadata>>;
+
+    /// The dyn version of [`Write::abort`].
+    fn abort_dyn(&mut self) -> BoxedFuture<'_, Result<()>>;
+}
+
+impl<T: Write + ?Sized> WriteDyn for T {
+    fn write_dyn(&mut self, bs: Buffer) -> BoxedFuture<'_, Result<()>> {
+        Box::pin(self.write(bs))
+    }
+
+    fn close_dyn(&mut self) -> BoxedFuture<'_, Result<Metadata>> {
+        Box::pin(self.close())
+    }
+
+    fn abort_dyn(&mut self) -> BoxedFuture<'_, Result<()>> {
+        Box::pin(self.abort())
+    }
+}
+
+impl<T: WriteDyn + ?Sized> Write for Box<T> {
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        self.deref_mut().write_dyn(bs).await
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        self.deref_mut().close_dyn().await
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        self.deref_mut().abort_dyn().await
+    }
+}

data/core/src/raw/oio/write/append_write.rs

@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// AppendWrite is used to implement [`oio::Write`] based on append
+/// object. By implementing AppendWrite, services don't need to
+/// care about the details of buffering and uploading parts.
+///
+/// The layout after adopting [`AppendWrite`]:
+///
+/// - Services impl `AppendWrite`
+/// - `AppendWriter` impl `Write`
+/// - Expose `AppendWriter` as `Accessor::Writer`
+///
+/// ## Requirements
+///
+/// Services that implement `AppendWrite` must fulfill the following requirements:
+///
+/// - Must be a http service that could accept `AsyncBody`.
+/// - Provide a way to get the current offset of the append object.
+pub trait AppendWrite: Send + Sync + Unpin + 'static {
+    /// Get the current offset of the append object.
+    ///
+    /// Returns `0` if the object is not exist.
+    fn offset(&self) -> impl Future<Output = Result<u64>> + MaybeSend;
+
+    /// Append the data to the end of this object.
+    fn append(
+        &self,
+        offset: u64,
+        size: u64,
+        body: Buffer,
+    ) -> impl Future<Output = Result<Metadata>> + MaybeSend;
+}
+
+/// AppendWriter will implements [`oio::Write`] based on append object.
+///
+/// ## TODO
+///
+/// - Allow users to switch to un-buffered mode if users write 16MiB every time.
+pub struct AppendWriter<W: AppendWrite> {
+    inner: W,
+
+    offset: Option<u64>,
+
+    meta: Metadata,
+}
+
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this state.
+impl<W: AppendWrite> AppendWriter<W> {
+    /// Create a new AppendWriter.
+    pub fn new(inner: W) -> Self {
+        Self {
+            inner,
+            offset: None,
+            meta: Metadata::default(),
+        }
+    }
+}
+
+impl<W> oio::Write for AppendWriter<W>
+where
+    W: AppendWrite,
+{
+    async fn write(&mut self, bs: Buffer) -> Result<()> {
+        let offset = match self.offset {
+            Some(offset) => offset,
+            None => {
+                let offset = self.inner.offset().await?;
+                self.offset = Some(offset);
+                offset
+            }
+        };
+
+        let size = bs.len();
+        self.meta = self.inner.append(offset, size as u64, bs).await?;
+        // Update offset after succeed.
+        self.offset = Some(offset + size as u64);
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        self.meta
+            .set_content_length(self.offset.unwrap_or_default());
+        Ok(self.meta.clone())
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Ok(())
+    }
+}