opendal 0.1.6.pre.rc.1

data/core/src/raw/oio/buf/mod.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod flex_buf;
+pub use flex_buf::FlexBuf;
+
+mod queue_buf;
+pub use queue_buf::QueueBuf;
+
+mod pooled_buf;
+pub use pooled_buf::PooledBuf;

data/core/src/raw/oio/buf/pooled_buf.rs

@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::VecDeque;
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::fmt::{self};
+use std::sync::Mutex;
+
+use bytes::BytesMut;
+
+/// PooledBuf is a buffer pool that designed for reusing already allocated bufs.
+///
+/// It works as best-effort that tries to reuse the buffer if possible. It
+/// won't block the thread if the pool is locked, just returning a new buffer
+/// or dropping existing buffer.
+pub struct PooledBuf {
+    pool: Mutex<VecDeque<BytesMut>>,
+    size: usize,
+    initial_capacity: usize,
+}
+
+impl Debug for PooledBuf {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PooledBuf")
+            .field("size", &self.size)
+            .field("initial_capacity", &self.initial_capacity)
+            .finish_non_exhaustive()
+    }
+}
+
+impl PooledBuf {
+    /// Create a new buffer pool with a given size.
+    pub fn new(size: usize) -> Self {
+        Self {
+            pool: Mutex::new(VecDeque::with_capacity(size)),
+            size,
+            initial_capacity: 0,
+        }
+    }
+
+    /// Set the initial capacity of the buffer.
+    ///
+    /// The default value is 0.
+    pub fn with_initial_capacity(mut self, initial_capacity: usize) -> Self {
+        self.initial_capacity = initial_capacity;
+        self
+    }
+
+    /// Get a [`BytesMut`] from the pool.
+    ///
+    /// It's guaranteed that the buffer is empty.
+    pub fn get(&self) -> BytesMut {
+        // We don't want to block the thread if the pool is locked.
+        //
+        // Just returning a new buffer in this case.
+        let Ok(mut pool) = self.pool.try_lock() else {
+            return BytesMut::with_capacity(self.initial_capacity);
+        };
+
+        if let Some(buf) = pool.pop_front() {
+            buf
+        } else {
+            BytesMut::with_capacity(self.initial_capacity)
+        }
+    }
+
+    /// Put a [`BytesMut`] back to the pool.
+    pub fn put(&self, mut buf: BytesMut) {
+        // We don't want to block the thread if the pool is locked.
+        //
+        // Just dropping the buffer in this case.
+        let Ok(mut pool) = self.pool.try_lock() else {
+            return;
+        };
+
+        if pool.len() < self.size {
+            // Clean the buffer before putting it back to the pool.
+            buf.clear();
+            pool.push_back(buf);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::BufMut;
+
+    use super::*;
+
+    #[test]
+    fn test_pooled_buf() {
+        let pool = PooledBuf::new(2);
+
+        let mut buf1 = pool.get();
+        buf1.put_slice(b"hello, world!");
+
+        let mut buf2 = pool.get();
+        buf2.reserve(1024);
+
+        pool.put(buf1);
+        pool.put(buf2);
+
+        let buf3 = pool.get();
+        assert_eq!(buf3.len(), 0);
+        assert_eq!(buf3.capacity(), 13);
+
+        let buf4 = pool.get();
+        assert_eq!(buf4.len(), 0);
+        assert_eq!(buf4.capacity(), 1024);
+    }
+}

data/core/src/raw/oio/buf/queue_buf.rs

@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::VecDeque;
+use std::mem;
+
+use bytes::Buf;
+
+use crate::*;
+
+/// QueueBuf is a queue of [`Buffer`].
+///
+/// It's designed to allow storing multiple buffers without copying underlying bytes and consume them
+/// in order.
+///
+/// QueueBuf mainly provides the following operations:
+///
+/// - `push`: Push a new buffer in the queue.
+/// - `collect`: Collect all buffer in the queue as a new [`Buffer`]
+/// - `advance`: Advance the queue by `cnt` bytes.
+#[derive(Clone, Default)]
+pub struct QueueBuf(VecDeque<Buffer>);
+
+impl QueueBuf {
+    /// Create a new buffer queue.
+    #[inline]
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Push new [`Buffer`] into the queue.
+    #[inline]
+    pub fn push(&mut self, buf: Buffer) {
+        if buf.is_empty() {
+            return;
+        }
+
+        self.0.push_back(buf);
+    }
+
+    /// Total bytes size inside the buffer queue.
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.0.iter().map(|b| b.len()).sum()
+    }
+
+    /// Is the buffer queue empty.
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Take the entire buffer queue and leave `self` in empty states.
+    #[inline]
+    pub fn take(&mut self) -> QueueBuf {
+        mem::take(self)
+    }
+
+    /// Build a new [`Buffer`] from the queue.
+    ///
+    /// If the queue is empty, it will return an empty buffer. Otherwise, it will iterate over all
+    /// buffers and collect them into a new buffer.
+    ///
+    /// # Notes
+    ///
+    /// There are allocation overheads when collecting multiple buffers into a new buffer. But
+    /// most of them should be acceptable since we can expect the item length of buffers are slower
+    /// than 4k.
+    #[inline]
+    pub fn collect(mut self) -> Buffer {
+        if self.0.is_empty() {
+            Buffer::new()
+        } else if self.0.len() == 1 {
+            self.0.pop_front().unwrap()
+        } else {
+            self.0.into_iter().flatten().collect()
+        }
+    }
+
+    /// Advance the buffer queue by `cnt` bytes.
+    #[inline]
+    pub fn advance(&mut self, cnt: usize) {
+        assert!(cnt <= self.len(), "cannot advance past {cnt} bytes");
+
+        let mut new_cnt = cnt;
+        while new_cnt > 0 {
+            let buf = self.0.front_mut().expect("buffer must be valid");
+            if new_cnt < buf.remaining() {
+                buf.advance(new_cnt);
+                break;
+            } else {
+                new_cnt -= buf.remaining();
+                self.0.pop_front();
+            }
+        }
+    }
+
+    /// Clear the buffer queue.
+    #[inline]
+    pub fn clear(&mut self) {
+        self.0.clear()
+    }
+}

data/core/src/raw/oio/delete/api.rs

@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+use std::ops::DerefMut;
+
+use crate::raw::BoxedFuture;
+use crate::raw::MaybeSend;
+use crate::raw::OpDelete;
+use crate::*;
+
+/// Deleter is a type erased [`Delete`]
+pub type Deleter = Box<dyn DeleteDyn>;
+
+/// The Delete trait defines interfaces for performing deletion operations.
+pub trait Delete: Unpin + Send + Sync {
+    /// Requests deletion of a resource at the specified path with optional arguments
+    ///
+    /// # Parameters
+    /// - `path`: The path of the resource to delete
+    /// - `args`: Additional arguments for the delete operation
+    ///
+    /// # Returns
+    /// - `Ok(())`: The deletion request has been successfully queued (does not guarantee actual deletion)
+    /// - `Err(err)`: An error occurred and the deletion request was not queued
+    ///
+    /// # Notes
+    /// This method just queue the delete request. The actual deletion will be
+    /// performed when `flush` is called.
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()>;
+
+    /// Flushes the deletion queue to ensure queued deletions are executed
+    ///
+    /// # Returns
+    /// - `Ok(0)`: All queued deletions have been processed or the queue is empty.
+    /// - `Ok(count)`: The number of resources successfully deleted. Implementations should
+    ///   return an error if the queue is non-empty but no resources were deleted
+    /// - `Err(err)`: An error occurred while performing the deletions
+    ///
+    /// # Notes
+    /// - This method is asynchronous and will wait for queued deletions to complete
+    fn flush(&mut self) -> impl Future<Output = Result<usize>> + MaybeSend;
+}
+
+impl Delete for () {
+    fn delete(&mut self, _: &str, _: OpDelete) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "output deleter doesn't support delete",
+        ))
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "output deleter doesn't support flush",
+        ))
+    }
+}
+
+/// The dyn version of [`Delete`]
+pub trait DeleteDyn: Unpin + Send + Sync {
+    /// The dyn version of [`Delete::delete`]
+    fn delete_dyn(&mut self, path: &str, args: OpDelete) -> Result<()>;
+
+    /// The dyn version of [`Delete::flush`]
+    fn flush_dyn(&mut self) -> BoxedFuture<'_, Result<usize>>;
+}
+
+impl<T: Delete + ?Sized> DeleteDyn for T {
+    fn delete_dyn(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        Delete::delete(self, path, args)
+    }
+
+    fn flush_dyn(&mut self) -> BoxedFuture<'_, Result<usize>> {
+        Box::pin(self.flush())
+    }
+}
+
+impl<T: DeleteDyn + ?Sized> Delete for Box<T> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        self.deref_mut().delete_dyn(path, args)
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        self.deref_mut().flush_dyn().await
+    }
+}

data/core/src/raw/oio/delete/batch_delete.rs

@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashSet;
+use std::future::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// BatchDelete is used to implement [`oio::Delete`] based on batch delete operation.
+///
+/// OneShotDeleter will perform delete operation while calling `flush`.
+pub trait BatchDelete: Send + Sync + Unpin + 'static {
+    /// delete_once delete one path at once.
+    ///
+    /// Implementations should make sure that the data is deleted correctly at once.
+    ///
+    /// BatchDeleter may call this method while there are only one path to delete.
+    fn delete_once(
+        &self,
+        path: String,
+        args: OpDelete,
+    ) -> impl Future<Output = Result<()>> + MaybeSend;
+
+    /// delete_batch delete multiple paths at once.
+    ///
+    /// - Implementations should make sure that the length of `batch` equals to the return result's length.
+    /// - Implementations should return error no path is deleted.
+    fn delete_batch(
+        &self,
+        batch: Vec<(String, OpDelete)>,
+    ) -> impl Future<Output = Result<BatchDeleteResult>> + MaybeSend;
+}
+
+/// BatchDeleteResult is the result of batch delete operation.
+#[derive(Default)]
+pub struct BatchDeleteResult {
+    /// Collection of successful deletions, containing tuples of (path, args)
+    pub succeeded: Vec<(String, OpDelete)>,
+    /// Collection of failed deletions, containing tuples of (path, args, error)
+    pub failed: Vec<(String, OpDelete, Error)>,
+}
+
+/// BatchDeleter is used to implement [`oio::Delete`] based on batch delete.
+pub struct BatchDeleter<D: BatchDelete> {
+    inner: D,
+    buffer: HashSet<(String, OpDelete)>,
+}
+
+impl<D: BatchDelete> BatchDeleter<D> {
+    /// Create a new batch deleter.
+    pub fn new(inner: D) -> Self {
+        Self {
+            inner,
+            buffer: HashSet::default(),
+        }
+    }
+}
+
+impl<D: BatchDelete> oio::Delete for BatchDeleter<D> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        self.buffer.insert((path.to_string(), args));
+        Ok(())
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        if self.buffer.is_empty() {
+            return Ok(0);
+        }
+        if self.buffer.len() == 1 {
+            let (path, args) = self
+                .buffer
+                .iter()
+                .next()
+                .expect("the delete buffer size must be 1")
+                .clone();
+            self.inner.delete_once(path, args).await?;
+            self.buffer.clear();
+            return Ok(1);
+        }
+
+        let batch = self.buffer.iter().cloned().collect();
+        let result = self.inner.delete_batch(batch).await?;
+        debug_assert!(
+            !result.succeeded.is_empty(),
+            "the number of succeeded operations must be greater than 0"
+        );
+        debug_assert_eq!(
+            result.succeeded.len() + result.failed.len(),
+            self.buffer.len(),
+            "the number of succeeded and failed operations must be equal to the input batch size"
+        );
+
+        // Remove all succeeded operations from the buffer.
+        let deleted = result.succeeded.len();
+        for i in result.succeeded {
+            self.buffer.remove(&i);
+        }
+
+        // Return directly if there are non-temporary errors.
+        for (path, op, err) in result.failed {
+            if !err.is_temporary() {
+                return Err(err
+                    .with_context("path", path)
+                    .with_context("version", op.version().unwrap_or("<latest>")));
+            }
+        }
+
+        // Return the number of succeeded operations to allow users to decide whether
+        // to retry or push more delete operations.
+        Ok(deleted)
+    }
+}

data/core/src/raw/oio/delete/mod.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod api;
+pub use api::Delete;
+pub use api::DeleteDyn;
+pub use api::Deleter;
+
+mod batch_delete;
+pub use batch_delete::BatchDelete;
+pub use batch_delete::BatchDeleteResult;
+pub use batch_delete::BatchDeleter;
+
+mod one_shot_delete;
+pub use one_shot_delete::OneShotDelete;
+pub use one_shot_delete::OneShotDeleter;

data/core/src/raw/oio/delete/one_shot_delete.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// OneShotDelete is used to implement [`oio::Delete`] based on one shot operation.
+///
+/// OneShotDeleter will perform delete operation while calling `flush`.
+pub trait OneShotDelete: Send + Sync + Unpin + 'static {
+    /// delete_once delete one path at once.
+    ///
+    /// Implementations should make sure that the data is deleted correctly at once.
+    fn delete_once(
+        &self,
+        path: String,
+        args: OpDelete,
+    ) -> impl Future<Output = Result<()>> + MaybeSend;
+}
+
+/// OneShotDelete is used to implement [`oio::Delete`] based on one shot.
+pub struct OneShotDeleter<D> {
+    inner: D,
+    delete: Option<(String, OpDelete)>,
+}
+
+impl<D> OneShotDeleter<D> {
+    /// Create a new one shot deleter.
+    pub fn new(inner: D) -> Self {
+        Self {
+            inner,
+            delete: None,
+        }
+    }
+
+    fn delete_inner(&mut self, path: String, args: OpDelete) -> Result<()> {
+        if self.delete.is_some() {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "OneShotDeleter doesn't support batch delete",
+            ));
+        }
+
+        self.delete = Some((path, args));
+        Ok(())
+    }
+}
+
+impl<D: OneShotDelete> oio::Delete for OneShotDeleter<D> {
+    fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+        self.delete_inner(path.to_string(), args)
+    }
+
+    async fn flush(&mut self) -> Result<usize> {
+        let Some((path, args)) = self.delete.clone() else {
+            return Ok(0);
+        };
+
+        self.inner.delete_once(path, args).await?;
+        self.delete = None;
+        Ok(1)
+    }
+}