opendal 0.1.6.pre.rc.1

data/core/src/raw/std_io_util.rs

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+
+use crate::*;
+
+/// Parse std io error into opendal::Error.
+///
+/// # TODO
+///
+/// Add `NotADirectory` and `IsADirectory` once they are stable.
+///
+/// ref: <https://github.com/rust-lang/rust/issues/86442>
+pub fn new_std_io_error(err: std::io::Error) -> Error {
+    use std::io::ErrorKind::*;
+
+    let (kind, retryable) = match err.kind() {
+        NotFound => (ErrorKind::NotFound, false),
+        PermissionDenied => (ErrorKind::PermissionDenied, false),
+        AlreadyExists => (ErrorKind::AlreadyExists, false),
+        Unsupported => (ErrorKind::Unsupported, false),
+
+        Interrupted | UnexpectedEof | TimedOut | WouldBlock => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, true),
+    };
+
+    let mut err = Error::new(kind, err.kind().to_string()).set_source(err);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+/// helper functions to format `Error` into `io::Error`.
+///
+/// This function is added privately by design and only valid in current
+/// context (i.e. `raw` mod). We don't want to expose this function to
+/// users.
+#[inline]
+pub(crate) fn format_std_io_error(err: Error) -> io::Error {
+    let kind = match err.kind() {
+        ErrorKind::NotFound => io::ErrorKind::NotFound,
+        ErrorKind::PermissionDenied => io::ErrorKind::PermissionDenied,
+        _ => io::ErrorKind::Interrupted,
+    };
+
+    io::Error::new(kind, err)
+}

data/core/src/raw/tests/mod.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Utilities for opendal testing.
+
+mod read;
+pub use read::ReadAction;
+pub use read::ReadChecker;
+
+mod write;
+pub use write::WriteAction;
+pub use write::WriteChecker;
+
+mod utils;
+pub use utils::init_test_service;
+pub use utils::TEST_RUNTIME;

data/core/src/raw/tests/read.rs

@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Bytes;
+use rand::thread_rng;
+use rand::RngCore;
+use sha2::Digest;
+use sha2::Sha256;
+
+use crate::*;
+
+/// ReadAction represents a read action.
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum ReadAction {
+    /// Read represents a read action with given input buf size.
+    ///
+    /// # NOTE
+    ///
+    /// The size is the input buf size, it's possible that the actual read size is smaller.
+    Read(usize, usize),
+}
+
+/// ReadChecker is used to check the correctness of the read process.
+pub struct ReadChecker {
+    /// Raw Data is the data we write to the storage.
+    raw_data: Bytes,
+}
+
+impl ReadChecker {
+    /// Create a new read checker by given size and range.
+    ///
+    /// It's by design that we use a random generator to generate the raw data. The content of data
+    /// is not important, we only care about the correctness of the read process.
+    pub fn new(size: usize) -> Self {
+        let mut rng = thread_rng();
+        let mut data = vec![0; size];
+        rng.fill_bytes(&mut data);
+
+        let raw_data = Bytes::from(data);
+
+        Self { raw_data }
+    }
+
+    /// Return the raw data of this read checker.
+    pub fn data(&self) -> Bytes {
+        self.raw_data.clone()
+    }
+
+    /// check_read checks the correctness of the read process after a read action.
+    ///
+    /// - buf_size is the read action's buf size.
+    /// - output is the output of this read action.
+    fn check_read(&self, offset: usize, size: usize, output: &[u8]) {
+        if size == 0 {
+            assert_eq!(
+                output.len(),
+                0,
+                "check read failed: output must be empty if buf_size is 0"
+            );
+            return;
+        }
+
+        if size > 0 && output.is_empty() {
+            assert!(
+                offset >= self.raw_data.len(),
+                "check read failed: no data read means cur must outsides of ranged_data",
+            );
+            return;
+        }
+
+        assert!(
+            offset + output.len() <= self.raw_data.len(),
+            "check read failed: cur + output length must be less than ranged_data length, offset: {}, output: {}, ranged_data: {}",  offset, output.len(), self.raw_data.len(),
+        );
+
+        let expected = &self.raw_data[offset..offset + output.len()];
+
+        // Check the read result
+        assert_eq!(
+            format!("{:x}", Sha256::digest(output)),
+            format!("{:x}", Sha256::digest(expected)),
+            "check read failed: output bs is different with expected bs",
+        );
+    }
+
+    /// Check will check the correctness of the read process via given actions.
+    ///
+    /// Check will panic if any check failed.
+    pub async fn check(&mut self, r: Reader, actions: &[ReadAction]) {
+        for action in actions {
+            match *action {
+                ReadAction::Read(offset, size) => {
+                    let bs = r
+                        .read(offset as u64..(offset + size) as u64)
+                        .await
+                        .expect("read must success");
+                    self.check_read(offset, size, bs.to_bytes().as_ref());
+                }
+            }
+        }
+    }
+}

data/core/src/raw/tests/utils.rs

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::env;
+use std::str::FromStr;
+use std::sync::LazyLock;
+
+use crate::*;
+
+/// TEST_RUNTIME is the runtime used for running tests.
+pub static TEST_RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+});
+
+/// Init a service with given scheme.
+///
+/// - Load scheme from `OPENDAL_TEST`
+/// - Construct a new Operator with given root.
+/// - Else, returns a `None` to represent no valid config for operator.
+pub fn init_test_service() -> Result<Option<Operator>> {
+    let _ = dotenvy::dotenv();
+
+    let scheme = if let Ok(v) = env::var("OPENDAL_TEST") {
+        v
+    } else {
+        return Ok(None);
+    };
+    let scheme = Scheme::from_str(&scheme).unwrap();
+
+    let prefix = format!("opendal_{scheme}_");
+
+    let mut cfg = env::vars()
+        .filter_map(|(k, v)| {
+            k.to_lowercase()
+                .strip_prefix(&prefix)
+                .map(|k| (k.to_string(), v))
+        })
+        .collect::<HashMap<String, String>>();
+
+    // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
+    let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
+    if !disable_random_root {
+        let root = format!(
+            "{}{}/",
+            cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()),
+            uuid::Uuid::new_v4()
+        );
+        cfg.insert("root".to_string(), root);
+    }
+
+    let op = Operator::via_iter(scheme, cfg).expect("must succeed");
+
+    #[cfg(feature = "layers-chaos")]
+    let op = { op.layer(layers::ChaosLayer::new(0.1)) };
+
+    let op = op
+        .layer(layers::LoggingLayer::default())
+        .layer(layers::TimeoutLayer::new())
+        .layer(layers::RetryLayer::new().with_max_times(4));
+
+    Ok(Some(op))
+}

data/core/src/raw/tests/write.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Bytes;
+use bytes::BytesMut;
+use rand::thread_rng;
+use rand::RngCore;
+use sha2::Digest;
+use sha2::Sha256;
+
+/// WriteAction represents a read action.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum WriteAction {
+    /// Write represents a write action with given input buf size.
+    ///
+    /// # NOTE
+    ///
+    /// The size is the input buf size, it's possible that the actual write size is smaller.
+    Write(usize),
+}
+
+/// WriteAction is used to check the correctness of the write process.
+pub struct WriteChecker {
+    chunks: Vec<Bytes>,
+    data: Bytes,
+}
+
+impl WriteChecker {
+    /// Create a new WriteChecker with given size.
+    pub fn new(size: Vec<usize>) -> Self {
+        let mut rng = thread_rng();
+
+        let mut chunks = Vec::with_capacity(size.len());
+
+        for i in size {
+            let mut bs = vec![0u8; i];
+            rng.fill_bytes(&mut bs);
+            chunks.push(Bytes::from(bs));
+        }
+
+        let data = chunks.iter().fold(BytesMut::new(), |mut acc, x| {
+            acc.extend_from_slice(x);
+            acc
+        });
+
+        WriteChecker {
+            chunks,
+            data: data.freeze(),
+        }
+    }
+
+    /// Get the check's chunks.
+    pub fn chunks(&self) -> &[Bytes] {
+        &self.chunks
+    }
+
+    /// Check the correctness of the write process.
+    pub fn check(&self, actual: &[u8]) {
+        assert_eq!(
+            format!("{:x}", Sha256::digest(actual)),
+            format!("{:x}", Sha256::digest(&self.data)),
+            "check failed: result is not expected"
+        )
+    }
+}

data/core/src/raw/tokio_util.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::Error;
+use crate::ErrorKind;
+
+/// Parse tokio error into opendal::Error.
+pub fn new_task_join_error(e: tokio::task::JoinError) -> Error {
+    Error::new(ErrorKind::Unexpected, "tokio task join failed").set_source(e)
+}

data/core/src/raw/version.rs

@@ -0,0 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// VERSION is the compiled version of OpenDAL.
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");