opendal 0.1.6.pre.rc.1

data/core/examples/multipart-upload/src/main.rs

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::Operator;
+use opendal::Result;
+
+async fn example(op: Operator) -> Result<()> {
+    let mut w = op.writer("test.txt").await?;
+
+    // Write a 10MiB chunk.
+    w.write(vec![0; 10 * 1024 * 1024]).await?;
+    println!("vec![0; 10 * 1024 * 1024] uploaded");
+    // Write another 10MiB chunk.
+    w.write(vec![1; 10 * 1024 * 1024]).await?;
+    println!("vec![1; 10 * 1024 * 1024] uploaded");
+    // Finish the upload.
+    w.close().await?;
+    println!("upload finished");
+
+    let bs = op.read("test.txt").await?;
+    println!("read: {} bytes", bs.len());
+    assert_eq!(
+        bs.to_vec(),
+        vec![0; 10 * 1024 * 1024]
+            .into_iter()
+            .chain(vec![1; 10 * 1024 * 1024])
+            .collect::<Vec<_>>()
+    );
+
+    Ok(())
+}
+
+// --- The following data is not part of this example ---
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    use opendal::raw::tests::init_test_service;
+    let op = init_test_service()?.expect("OPENDAL_TEST must be set");
+
+    println!("service {:?} has been initialized", op.info());
+    example(op).await
+}

data/core/fuzz/.gitignore

@@ -0,0 +1,5 @@
+target
+corpus
+artifacts
+coverage
+tmp

data/core/fuzz/Cargo.toml

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+edition = "2021"
+license = "Apache-2.0"
+name = "opendal-fuzz"
+publish = false
+rust-version = "1.82"
+version = "0.0.0"
+
+[package.metadata]
+cargo-fuzz = true
+
+[features]
+services-alluxio = ["opendal/services-alluxio"]
+services-azblob = ["opendal/services-azblob"]
+services-azdls = ["opendal/services-azdls"]
+services-azfile = ["opendal/services-azfile"]
+services-b2 = ["opendal/services-b2"]
+services-cacache = ["opendal/services-cacache"]
+services-cos = ["opendal/services-cos"]
+services-dashmap = ["opendal/services-dashmap"]
+services-dropbox = ["opendal/services-dropbox"]
+services-etcd = ["opendal/services-etcd"]
+services-foundationdb = ["opendal/services-foundationdb"]
+services-fs = ["opendal/services-fs"]
+services-ftp = ["opendal/services-ftp"]
+services-gcs = ["opendal/services-gcs"]
+services-gdrive = ["opendal/services-gdrive"]
+services-ghac = ["opendal/services-ghac"]
+services-gridfs = ["opendal/services-gridfs"]
+services-hdfs = ["opendal/services-hdfs"]
+services-http = ["opendal/services-http"]
+services-huggingface = ["opendal/services-huggingface"]
+services-ipfs = ["opendal/services-ipfs"]
+services-ipmfs = ["opendal/services-ipmfs"]
+services-memcached = ["opendal/services-memcached"]
+services-memory = ["opendal/services-memory"]
+services-mini-moka = ["opendal/services-mini-moka"]
+services-moka = ["opendal/services-moka"]
+services-mongodb = ["opendal/services-mongodb"]
+services-mysql = ["opendal/services-mysql"]
+services-obs = ["opendal/services-obs"]
+services-onedrive = ["opendal/services-onedrive"]
+services-oss = ["opendal/services-oss"]
+services-persy = ["opendal/services-persy"]
+services-postgresql = ["opendal/services-postgresql"]
+services-redb = ["opendal/services-redb"]
+services-redis = ["opendal/services-redis"]
+services-rocksdb = ["opendal/services-rocksdb"]
+services-s3 = ["opendal/services-s3"]
+services-seafile = ["opendal/services-seafile"]
+services-sftp = ["opendal/services-sftp"]
+services-sled = ["opendal/services-sled"]
+services-sqlite = ["opendal/services-sqlite"]
+services-swift = ["opendal/services-swift"]
+services-tikv = ["opendal/services-tikv"]
+services-upyun = ["opendal/services-upyun"]
+services-vercel-artifacts = ["opendal/services-vercel-artifacts"]
+services-webdav = ["opendal/services-webdav"]
+services-webhdfs = ["opendal/services-webhdfs"]
+
+[dependencies]
+arbitrary = { version = "1.3.0", features = ["derive"] }
+libfuzzer-sys = "0.4"
+log = { version = "0.4.22" }
+logforth = { version = "0.23.1", default-features = false }
+opendal = { path = "..", features = ["tests"] }
+uuid = { version = "1", features = ["v4"] }
+
+[[bin]]
+name = "fuzz_reader"
+path = "fuzz_reader.rs"
+
+[[bin]]
+name = "fuzz_writer"
+path = "fuzz_writer.rs"

data/core/fuzz/README.md

@@ -0,0 +1,68 @@
+# Fuzz Test for OpenDAL
+
+fuzz test are used to test the robustness of the code. 
+
+## Setup
+
+
+
+To run the fuzz tests, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
+
+Take `fs` for example, we need to change to enable fuzz test on `fs` on `/tmp`.
+
+```dotenv
+OPENDAL_TEST=fs
+OPENDAL_FS_ROOT=/path/to/dir/
+```
+
+into
+
+```dotenv
+OPENDAL_TEST=fs
+OPENDAL_FS_ROOT=/tmp/
+```
+
+
+## Run
+
+List all fuzz targets.
+
+```bash
+cargo +nightly fuzz list
+```
+
+Build fuzz targets.
+
+```bash
+cargo +nightly fuzz build
+```
+
+Run a fuzz target(such as `reader`).
+
+```bash
+cargo +nightly fuzz run fuzz_reader
+```
+
+## Crash Reproduction
+
+If you want to reproduce a crash, you first need to obtain the Base64 encoded code, which usually appears at the end of a crash report, and store it in a file.
+
+Alternatively, if you already have the crash file, you can skip this step.
+
+```bash
+echo "Base64" > .crash
+```
+
+Print the `std::fmt::Debug` output for an input.
+
+```bash
+cargo +nightly fuzz fmt fuzz_target .crash
+```
+
+Rerun the fuzz test with the input.
+
+```bash
+cargo +nightly fuzz run fuzz_target .crash
+```
+
+For more details, please visit [cargo fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/tutorial.html) or run the command cargo fuzz --help.

data/core/fuzz/fuzz_reader.rs

@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![no_main]
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use libfuzzer_sys::arbitrary::Arbitrary;
+use libfuzzer_sys::arbitrary::Unstructured;
+use libfuzzer_sys::fuzz_target;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::ReadAction;
+use opendal::raw::tests::ReadChecker;
+use opendal::raw::tests::TEST_RUNTIME;
+use opendal::Operator;
+use opendal::Result;
+
+const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
+
+#[derive(Clone)]
+struct FuzzInput {
+    path: String,
+    size: usize,
+    actions: Vec<ReadAction>,
+}
+
+impl Debug for FuzzInput {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut actions = self.actions.clone();
+        // Remove all Read(0) entry.
+        let empty = ReadAction::Read(0, 0);
+        actions.retain(|e| e != &empty);
+
+        f.debug_struct("FuzzInput")
+            .field("path", &self.path)
+            .field("size", &self.size)
+            .field("actions", &actions)
+            .finish()
+    }
+}
+
+impl Arbitrary<'_> for FuzzInput {
+    fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+        let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?;
+
+        let count = u.int_in_range(1..=1024)?;
+        let mut actions = vec![];
+
+        for _ in 0..count {
+            let offset = u.int_in_range(0..=total_size)?;
+            let size = u.int_in_range(0..=total_size - offset)?;
+
+            actions.push(ReadAction::Read(offset, size));
+        }
+
+        Ok(FuzzInput {
+            path: uuid::Uuid::new_v4().to_string(),
+            size: total_size,
+            actions,
+        })
+    }
+}
+
+async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
+    let mut checker = ReadChecker::new(input.size);
+    op.write(&input.path, checker.data()).await?;
+
+    let r = op.reader(&input.path).await?;
+
+    checker.check(r, &input.actions).await;
+
+    op.delete(&input.path).await?;
+    Ok(())
+}
+
+fuzz_target!(|input: FuzzInput| {
+    let _ = logforth::stderr().try_apply();
+
+    let op = init_test_service().expect("operator init must succeed");
+    if let Some(op) = op {
+        TEST_RUNTIME.block_on(async {
+            fuzz_reader(op, input.clone())
+                .await
+                .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
+        })
+    }
+});

data/core/fuzz/fuzz_writer.rs

@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![no_main]
+
+use libfuzzer_sys::arbitrary::Arbitrary;
+use libfuzzer_sys::arbitrary::Unstructured;
+use libfuzzer_sys::fuzz_target;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::WriteAction;
+use opendal::raw::tests::WriteChecker;
+use opendal::raw::tests::TEST_RUNTIME;
+use opendal::Operator;
+use opendal::Result;
+
+const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
+
+#[derive(Debug, Clone)]
+struct FuzzInput {
+    actions: Vec<WriteAction>,
+    buffer: Option<usize>,
+    concurrent: Option<usize>,
+}
+
+impl Arbitrary<'_> for FuzzInput {
+    fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+        let mut actions = vec![];
+        let buffer = if u.int_in_range(0..=1)? == 1 {
+            Some(u.int_in_range(1..=8 * 1024 * 1024)?)
+        } else {
+            None
+        };
+        let concurrent = if u.int_in_range(0..=1)? == 1 {
+            Some(u.int_in_range(0..=16)?)
+        } else {
+            None
+        };
+
+        let count = u.int_in_range(1..=1024)?;
+
+        for _ in 0..count {
+            let size = u.int_in_range(1..=MAX_DATA_SIZE)?;
+            actions.push(WriteAction::Write(size));
+        }
+
+        Ok(FuzzInput {
+            actions,
+            buffer,
+            concurrent,
+        })
+    }
+}
+
+async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
+    let path = uuid::Uuid::new_v4().to_string();
+
+    let total_size = input
+        .actions
+        .iter()
+        .map(|a| match a {
+            WriteAction::Write(size) => *size,
+        })
+        .collect();
+
+    let checker = WriteChecker::new(total_size);
+
+    let mut writer = op.writer_with(&path);
+    if let Some(buffer) = input.buffer {
+        writer = writer.chunk(buffer);
+    } else if let Some(min_size) = op.info().full_capability().write_multi_min_size {
+        writer = writer.chunk(min_size);
+    }
+    if let Some(concurrent) = input.concurrent {
+        writer = writer.concurrent(concurrent);
+    }
+
+    let mut writer = writer.await?;
+
+    for chunk in checker.chunks() {
+        writer.write(chunk.clone()).await?;
+    }
+
+    writer.close().await?;
+
+    let result = op.read(&path).await?.to_bytes();
+
+    checker.check(&result);
+
+    op.delete(&path).await?;
+    Ok(())
+}
+
+fuzz_target!(|input: FuzzInput| {
+    let _ = logforth::stderr().try_apply();
+
+    let op = init_test_service().expect("operator init must succeed");
+    if let Some(op) = op {
+        if !op.info().full_capability().write_can_multi {
+            log::warn!("service doesn't support write multi, skip fuzzing");
+            return;
+        }
+
+        TEST_RUNTIME.block_on(async {
+            fuzz_writer(op, input.clone())
+                .await
+                .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
+        })
+    }
+});

data/core/src/blocking/delete.rs

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::Deleter as AsyncDeleter;
+use crate::*;
+
+/// BlockingDeleter is designed to continuously remove content from storage.
+///
+/// It leverages batch deletion capabilities provided by storage services for efficient removal.
+pub struct Deleter {
+    handle: tokio::runtime::Handle,
+    inner: AsyncDeleter,
+}
+
+impl Deleter {
+    pub(crate) fn create(handle: tokio::runtime::Handle, inner: AsyncDeleter) -> Result<Self> {
+        Ok(Self { handle, inner })
+    }
+
+    /// Delete a path.
+    pub fn delete(&mut self, input: impl IntoDeleteInput) -> Result<()> {
+        self.handle.block_on(self.inner.delete(input))
+    }
+
+    /// Delete an infallible iterator of paths.
+    ///
+    /// Also see:
+    ///
+    /// - [`BlockingDeleter::delete_try_iter`]: delete an fallible iterator of paths.
+    pub fn delete_iter<I, D>(&mut self, iter: I) -> Result<()>
+    where
+        I: IntoIterator<Item = D>,
+        D: IntoDeleteInput,
+    {
+        self.handle.block_on(self.inner.delete_iter(iter))
+    }
+
+    /// Delete an fallible iterator of paths.
+    ///
+    /// Also see:
+    ///
+    /// - [`BlockingDeleter::delete_iter`]: delete an infallible iterator of paths.
+    pub fn delete_try_iter<I, D>(&mut self, try_iter: I) -> Result<()>
+    where
+        I: IntoIterator<Item = Result<D>>,
+        D: IntoDeleteInput,
+    {
+        self.handle.block_on(self.inner.delete_try_iter(try_iter))
+    }
+
+    /// Flush the deleter, returns the number of deleted paths.
+    pub fn flush(&mut self) -> Result<usize> {
+        self.handle.block_on(self.inner.flush())
+    }
+
+    /// Close the deleter, this will flush the deleter and wait until all paths are deleted.
+    pub fn close(&mut self) -> Result<()> {
+        self.handle.block_on(self.inner.close())
+    }
+}

data/core/src/blocking/list.rs

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use futures::StreamExt;
+
+use crate::Lister as AsyncLister;
+use crate::*;
+
+/// BlockingLister is designed to list entries at given path in a blocking
+/// manner.
+///
+/// Users can construct Lister by [`blocking::Operator::lister`] or [`blocking::Operator::lister_with`].
+///
+/// - Lister implements `Iterator<Item = Result<Entry>>`.
+/// - Lister will return `None` if there is no more entries or error has been returned.
+pub struct Lister {
+    handle: tokio::runtime::Handle,
+    lister: Option<AsyncLister>,
+}
+
+/// # Safety
+///
+/// BlockingLister will only be accessed by `&mut Self`
+unsafe impl Sync for Lister {}
+
+impl Lister {
+    /// Create a new lister.
+    pub(crate) fn new(handle: tokio::runtime::Handle, lister: AsyncLister) -> Self {
+        Self {
+            handle,
+            lister: Some(lister),
+        }
+    }
+}
+
+impl Iterator for Lister {
+    type Item = Result<Entry>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let Some(lister) = self.lister.as_mut() else {
+            return Some(Err(Error::new(
+                ErrorKind::Unexpected,
+                "lister has been dropped",
+            )));
+        };
+
+        self.handle.block_on(lister.next())
+    }
+}
+
+impl Drop for Lister {
+    fn drop(&mut self) {
+        if let Some(v) = self.lister.take() {
+            self.handle.block_on(async move { drop(v) });
+        }
+    }
+}

data/core/src/blocking/mod.rs

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! blocking module provides blocking APIs for OpenDAL.
+
+mod delete;
+pub use delete::*;
+
+mod list;
+pub use list::*;
+
+mod read;
+pub use read::*;
+
+mod write;
+pub use write::*;
+
+mod operator;
+pub use operator::*;